jody@chefli ~/share/neander $ mpirun -np 1 -host squid_0 -mca plm_rsh_agent "ssh -Y" -mca plm_base_verbose 5 -mca odls_base_verbose 5 --leave-session-attached  --xterm 0 ./HelloMPI 
[chefli:02420] mca:base:select:(  plm) Querying component [rsh]
[chefli:02420] mca:base:select:(  plm) Query of component [rsh] set priority to 10
[chefli:02420] mca:base:select:(  plm) Querying component [slurm]
[chefli:02420] mca:base:select:(  plm) Skipping component [slurm]. Query failed to return a module
[chefli:02420] mca:base:select:(  plm) Selected component [rsh]
[chefli:02420] plm:base:set_hnp_name: initial bias 2420 nodename hash 72192778
[chefli:02420] plm:base:set_hnp_name: final jobfam 40499
[chefli:02420] [[40499,0],0] plm:base:receive start comm
[chefli:02420] mca:base:select:( odls) Querying component [default]
[chefli:02420] mca:base:select:( odls) Query of component [default] set priority to 1
[chefli:02420] mca:base:select:( odls) Selected component [default]
[chefli:02420] [[40499,0],0] plm:rsh: setting up job [40499,1]
[chefli:02420] [[40499,0],0] plm:base:setup_job for job [40499,1]
[chefli:02420] [[40499,0],0] plm:rsh: local shell: 0 (bash)
[chefli:02420] [[40499,0],0] plm:rsh: assuming same remote shell as local shell
[chefli:02420] [[40499,0],0] plm:rsh: remote shell: 0 (bash)
[chefli:02420] [[40499,0],0] plm:rsh: final template argv:
	/usr/bin/ssh -Y -X <template>  orted -mca ess env -mca orte_ess_jobid 2654142464 -mca orte_ess_vpid <template> -mca orte_ess_num_procs 2 --hnp-uri "2654142464.0;tcp://192.168.0.14:39093" -mca plm_base_verbose 5 -mca odls_base_verbose 5 --xterm 0 -mca plm_rsh_agent "ssh -Y"
[chefli:02420] [[40499,0],0] plm:rsh: launching on node squid_0
[chefli:02420] [[40499,0],0] plm:rsh: recording launch of daemon [[40499,0],1]
[chefli:02420] [[40499,0],0] plm:rsh: executing: (//usr/bin/ssh) [/usr/bin/ssh -Y -X squid_0  orted -mca ess env -mca orte_ess_jobid 2654142464 -mca orte_ess_vpid 1 -mca orte_ess_num_procs 2 --hnp-uri "2654142464.0;tcp://192.168.0.14:39093" -mca plm_base_verbose 5 -mca odls_base_verbose 5 --xterm 0 -mca plm_rsh_agent "ssh -Y"]
[squid_0:19442] mca:base:select:( odls) Querying component [default]
[squid_0:19442] mca:base:select:( odls) Query of component [default] set priority to 1
[squid_0:19442] mca:base:select:( odls) Selected component [default]
[chefli:02420] [[40499,0],0] plm:base:daemon_callback
[chefli:02420] [[40499,0],0] plm:base:orted_report_launch from daemon [[40499,0],1]
[chefli:02420] [[40499,0],0] plm:base:orted_report_launch completed for daemon [[40499,0],1]
[chefli:02420] [[40499,0],0] plm:base:daemon_callback completed
[chefli:02420] [[40499,0],0] plm:base:launch_apps for job [40499,1]
[chefli:02420] [[40499,0],0] plm:base:report_launched for job [40499,1]
[chefli:02420] [[40499,0],0] odls:constructing child list
[chefli:02420] [[40499,0],0] odls:construct_child_list unpacking data to launch job [40499,1]
[chefli:02420] [[40499,0],0] odls:construct_child_list adding new jobdat for job [40499,1]
[chefli:02420] [[40499,0],0] odls:construct_child_list unpacking 1 app_contexts
[chefli:02420] [[40499,0],0] odls:constructing child list - checking proc 0 on node 1 with daemon 1
[chefli:02420] [[40499,0],0] odls:construct:child: num_participating 1
[chefli:02420] [[40499,0],0] odls:launch found 12 processors for 0 children and set oversubscribed to false
[chefli:02420] [[40499,0],0] odls:launch reporting job [40499,1] launch status
[chefli:02420] [[40499,0],0] odls:launch setting waitpids
[chefli:02420] [[40499,0],0] plm:base:app_report_launch from daemon [[40499,0],0]
[chefli:02420] [[40499,0],0] plm:base:app_report_launch completed processing
[squid_0:19442] [[40499,0],1] odls:constructing child list
[squid_0:19442] [[40499,0],1] odls:construct_child_list unpacking data to launch job [40499,1]
[squid_0:19442] [[40499,0],1] odls:construct_child_list adding new jobdat for job [40499,1]
[squid_0:19442] [[40499,0],1] odls:construct_child_list unpacking 1 app_contexts
[squid_0:19442] [[40499,0],1] odls:constructing child list - checking proc 0 on node 1 with daemon 1
[squid_0:19442] [[40499,0],1] odls:constructing child list - found proc 0 for me!
[squid_0:19442] [[40499,0],1] odls:construct:child: num_participating 1
[squid_0:19442] [[40499,0],1] odls:launch found 4 processors for 1 children and set oversubscribed to false
[squid_0:19442] [[40499,0],1] odls:launch reporting job [40499,1] launch status
[squid_0:19442] [[40499,0],1] odls:launch setting waitpids
[chefli:02420] [[40499,0],0] plm:base:app_report_launch reissuing non-blocking recv
[chefli:02420] [[40499,0],0] plm:base:app_report_launch from daemon [[40499,0],1]
[chefli:02420] [[40499,0],0] plm:base:app_report_launched for proc [[40499,1],0] from daemon [[40499,0],1]: pid 19446 state 2 exit 0
[chefli:02420] [[40499,0],0] plm:base:app_report_launch completed processing
[chefli:02420] [[40499,0],0] plm:base:report_launched all apps reported
[chefli:02420] [[40499,0],0] plm:base:launch wiring up iof
[chefli:02420] [[40499,0],0] plm:base:launch completed for job [40499,1]
[squid_0:19442] [[40499,0],1] odls: registering sync on child [[40499,1],0]
[squid_0:19442] [[40499,0],1] odls:sync nidmap requested for job [40499,1]
[squid_0:19442] [[40499,0],1] odls: sending sync ack to child [[40499,1],0] with 144 bytes of data
[squid_0:19442] [[40499,0],1] odls: sending contact info to HNP
[squid_0:19442] [[40499,0],1] odls: collecting data from child [[40499,1],0]
[squid_0:19442] [[40499,0],1] odls: executing collective
[squid_0:19442] [[40499,0],1] odls: daemon collective called
[squid_0:19442] [[40499,0],1] odls: daemon collective for job [40499,1] from [[40499,0],1] type 2 num_collected 1 num_participating 1 num_contributors 1
[squid_0:19442] [[40499,0],1] odls: daemon collective not the HNP - sending to parent [[40499,0],0]
[squid_0:19442] [[40499,0],1] odls: collective completed
[chefli:02420] [[40499,0],0] odls: daemon collective called
[chefli:02420] [[40499,0],0] odls: daemon collective for job [40499,1] from [[40499,0],1] type 2 num_collected 1 num_participating 1 num_contributors 1
[chefli:02420] [[40499,0],0] odls: daemon collective HNP - xcasting to job [40499,1]
[squid_0:19442] [[40499,0],1] odls: sending message to tag 15 on child [[40499,1],0]
[squid_0:19442] [[40499,0],1] odls: collecting data from child [[40499,1],0]
[squid_0:19442] [[40499,0],1] odls: executing collective
[squid_0:19442] [[40499,0],1] odls: daemon collective called
[squid_0:19442] [[40499,0],1] odls: daemon collective for job [40499,1] from [[40499,0],1] type 1 num_collected 1 num_participating 1 num_contributors 1
[squid_0:19442] [[40499,0],1] odls: daemon collective not the HNP - sending to parent [[40499,0],0]
[squid_0:19442] [[40499,0],1] odls: collective completed
[chefli:02420] [[40499,0],0] odls: daemon collective called
[chefli:02420] [[40499,0],0] odls: daemon collective for job [40499,1] from [[40499,0],1] type 1 num_collected 1 num_participating 1 num_contributors 1
[chefli:02420] [[40499,0],0] odls: daemon collective HNP - xcasting to job [40499,1]
[squid_0:19442] [[40499,0],1] odls: sending message to tag 17 on child [[40499,1],0]
[squid_0:19442] [[40499,0],1] odls: collecting data from child [[40499,1],0]
[squid_0:19442] [[40499,0],1] odls: executing collective
[squid_0:19442] [[40499,0],1] odls: daemon collective called
[squid_0:19442] [[40499,0],1] odls: daemon collective for job [40499,1] from [[40499,0],1] type 1 num_collected 1 num_participating 1 num_contributors 1
[squid_0:19442] [[40499,0],1] odls: daemon collective not the HNP - sending to parent [[40499,0],0]
[squid_0:19442] [[40499,0],1] odls: collective completed
[chefli:02420] [[40499,0],0] odls: daemon collective called
[chefli:02420] [[40499,0],0] odls: daemon collective for job [40499,1] from [[40499,0],1] type 1 num_collected 1 num_participating 1 num_contributors 1
[chefli:02420] [[40499,0],0] odls: daemon collective HNP - xcasting to job [40499,1]
[squid_0:19442] [[40499,0],1] odls: sending message to tag 17 on child [[40499,1],0]
[squid_0:19442] [[40499,0],1] odls: registering sync on child [[40499,1],0]
[squid_0:19442] [[40499,0],1] odls: sending sync ack to child [[40499,1],0] with 0 bytes of data
[chefli:02420] [[40499,0],0] plm:base:receive got message from [[40499,0],1]
[chefli:02420] [[40499,0],0] plm:base:receive got update_proc_state for job [40499,1]
[chefli:02420] [[40499,0],0] plm:base:receive got update_proc_state for proc [[40499,1],0] curnt state 4 new state 80 exit_code 0
[chefli:02420] [[40499,0],0] plm:base:check_job_completed for job [40499,1] - num_terminated 1  num_procs 1
[chefli:02420] [[40499,0],0] plm:base:check_job_completed declared job [40499,1] normally terminated - checking all jobs
[chefli:02420] [[40499,0],0] plm:base:check_job_completed all jobs terminated - waking up
[chefli:02420] [[40499,0],0] plm:base:orted_cmd sending orted_exit commands
[chefli:02420] [[40499,0],0] odls:kill_local_proc working on job [WILDCARD]
[chefli:02420] [[40499,0],0] plm:base:check_job_completed for job [40499,0] - num_terminated 1  num_procs 2
[squid_0:19442] [[40499,0],1] odls:wait_local_proc child process 19446 terminated
[squid_0:19442] [[40499,0],1] odls:notify_iof_complete for child [[40499,1],0]
[squid_0:19442] [[40499,0],1] odls:waitpid_fired checking abort file /tmp/openmpi-sessions-jody@squid_0_0/2654142465/0/abort
[chefli:02420] [[40499,0],0] plm:base:receive got message from [[40499,0],1]
[chefli:02420] [[40499,0],0] plm:base:receive got update_proc_state for job [40499,0]
[chefli:02420] [[40499,0],0] plm:base:receive got update_proc_state for proc [[40499,0],1] curnt state 4 new state 80 exit_code 0
[chefli:02420] [[40499,0],0] plm:base:check_job_completed for job [40499,0] - num_terminated 2  num_procs 2
[chefli:02420] [[40499,0],0] plm:base:check_job_completed declared job [40499,0] normally terminated - checking all jobs
[chefli:02420] [[40499,0],0] plm:base:receive stop comm
[squid_0:19442] [[40499,0],1] odls:waitpid_fired child process [[40499,1],0] terminated normally
[squid_0:19442] [[40499,0],1] odls:proc_complete reporting all procs in [40499,1] terminated
[squid_0:19442] [[40499,0],1] odls:kill_local_proc working on job [WILDCARD]