[0] func:./p1 [0x81d362d]
[0] func:./p1 [0x81d362d]
[1] func:/lib/i686/libpthread.so.0 [0x400c40ba]
[2] func:/lib/i686/libc.so.6 [0x40131ee0]
[3] func:./p1(mpiPi_init+0x5c) [0x808b2a6]
[0] func:./p1 [0x81d362d]
[1] func:/lib/i686/libpthread.so.0 [0x400c40ba]
[1] func:/lib/i686/libpthread.so.0 [0x400c40ba]
[2] func:/lib/i686/libc.so.6 [0x40131ee0]
[2] func:/lib/i686/libc.so.6 [0x40131ee0]
[3] func:./p1(mpiPi_init+0x5c) [0x808b2a6]
[3] func:./p1(mpiPi_init+0x5c) [0x808b2a6]
mpiexec noticed that job rank 0 with PID 17721 on node "piv110" exited on signal 11.
[piv110:17719] [0,0,0]-[0,0,3] mca_oob_tcp_msg_send_handler: writev failed with errno=32
[piv110:17719] [0,0,0] ORTE_ERROR_LOG: Connection failed in file base/pls_base_proxy.c at line 145
[0] func:/home/osu4005/openmpi_nightly/openmpi/bin/mpiexec [0x80c584d]
[1] func:/lib/i686/libpthread.so.0 [0x400740ba]
[2] func:/lib/i686/libc.so.6 [0x400e1ee0]
[3] func:/home/osu4005/openmpi_nightly/openmpi/bin/mpiexec [0x8083c57]
[4] func:/home/osu4005/openmpi_nightly/openmpi/bin/mpiexec(mca_oob_tcp_msg_complete+0x6f) [0x809cfcb]
[5] func:/home/osu4005/openmpi_nightly/openmpi/bin/mpiexec(mca_oob_tcp_peer_send+0x118) [0x809dea8]
[6] func:/home/osu4005/openmpi_nightly/openmpi/bin/mpiexec(mca_oob_tcp_send_nb+0x240) [0x80a17cc]
[7] func:/home/osu4005/openmpi_nightly/openmpi/bin/mpiexec(mca_oob_send_packed_nb+0x8a) [0x8083bf2]
[8] func:/home/osu4005/openmpi_nightly/openmpi/bin/mpiexec(orte_pls_base_proxy_terminate_job+0x30f) [0x80b0653]
[9] func:/home/osu4005/openmpi_nightly/openmpi/bin/mpiexec [0x80ab3e5]
[10] func:/home/osu4005/openmpi_nightly/openmpi/bin/mpiexec(orte_errmgr_base_incomplete_start+0xf) [0x80639b7]
[11] func:/home/osu4005/openmpi_nightly/openmpi/bin/mpiexec(orte_rmgr_base_proc_stage_gate_mgr_abort+0x62) [0x80876ee]
[12] func:/home/osu4005/openmpi_nightly/openmpi/bin/mpiexec(orte_gpr_replica_deliver_notify_msg+0x216) [0x806a1fa]
[13] func:/home/osu4005/openmpi_nightly/openmpi/bin/mpiexec(orte_gpr_replica_process_callbacks+0x6d) [0x806f031]
[14] func:/home/osu4005/openmpi_nightly/openmpi/bin/mpiexec(orte_gpr_replica_recv+0xa5) [0x807532d]
[15] func:/home/osu4005/openmpi_nightly/openmpi/bin/mpiexec [0x8099bb2]
[16] func:/home/osu4005/openmpi_nightly/openmpi/bin/mpiexec [0x809d7f4]
[17] func:/home/osu4005/openmpi_nightly/openmpi/bin/mpiexec(opal_event_loop+0x243) [0x80bb1ab]
[18] func:/home/osu4005/openmpi_nightly/openmpi/bin/mpiexec(opal_progress+0x67) [0x80ba4ff]
[19] func:/home/osu4005/openmpi_nightly/openmpi/bin/mpiexec(vfprintf+0x2bad) [0x805efd9]
[20] func:/home/osu4005/openmpi_nightly/openmpi/bin/mpiexec(orterun+0x638) [0x805d0b4]
[21] func:/home/osu4005/openmpi_nightly/openmpi/bin/mpiexec(main+0x1e) [0x805ca76]
[22] func:/lib/i686/libc.so.6(__libc_start_main+0xaa) [0x400cebba]
[23] func:/home/osu4005/openmpi_nightly/openmpi/bin/mpiexec(dlopen+0x41) [0x805c9cd]
[piv107:31667] [0,0,1]-[0,0,0] mca_oob_tcp_msg_recv: readv failed with errno=104
[piv109:08029] [0,0,2]-[0,0,0] mca_oob_tcp_msg_recv: readv failed with errno=104
/var/spool/batch/pbs-piv/mom_priv/jobs/481512.nfs4.SC: line 7: 17719 Segmentation fault      /home/osu4005/openmpi_nightly/openmpi/bin/mpiexec --prefix /home/osu4005/openmpi_nightly/openmpi --mca btl mvapi --mca mpi_leave_pinned 0 -n 3 ./p1 >my_results
rcp: core*: No such file or directory
rcp: core*: No such file or directory
rcp: core*: No such file or directory