[hpdcnln001:13670] crcp:bkmrk: req_complete: Matched an iRecv: total = 5507957 [hpdcnln001:13670] crcp:bkmrk: req_complete: Matched an iRecv: total = 5507958 [hpdcnln001:13670] crcp:bkmrk: req_complete: Matched an iRecv: total = 5507959 [hpdcnln001:13670] crcp:bkmrk: req_complete: Matched an iRecv: total = 5507960 [hpdcnln001:13670] crcp:bkmrk: req_complete: Matched an iRecv: total = 5507961 [hpdcnln001:13670] crcp:bkmrk: req_complete: Matched an iRecv: total = 5507962 [hpdcnln001:13670] crcp:bkmrk: req_complete: Matched an iRecv: total = 5507963 [hpdcnln001:13670] crcp:bkmrk: req_complete: Matched an iRecv: total = 5507964 [hpdcnln001:13670] crcp:bkmrk: req_complete: Matched an iRecv: total = 5507965 [hpdcnln001:13670] crcp:bkmrk: req_complete: Matched an iRecv: total = 5507966 [hpdcnln001:13670] crcp:bkmrk: req_complete: Matched an iRecv: total = 5507967 [hpdcnln001:13670] crcp:bkmrk: req_complete: Matched an iRecv: total = 5507968 [hpdcnln001:13670] crcp:bkmrk: req_complete: Matched an iRecv: total = 5507969 [hpdcnln001:13670] crcp:bkmrk: req_complete: Matched an iRecv: total = 5507970 d: total = 5508031 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 5508032 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 5508033 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 5508034 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 5508035 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 5508036 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 5508037 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 5508038 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 5508039 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 5508040 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 5508041 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 5508042 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 5508043 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 5508044 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 5508045 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 5508046 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 5508047 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 5508048 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 5508049 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 5508050 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 5508051 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 5508052 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 5508053 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 5508054 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 5508055 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 5508056 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 5508057 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 5508058 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 5508059 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 5508060 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 5508061 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 5508062 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 5508063 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 5508064 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 5508065 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 5508066 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 5508067 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 5508068 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 5508069 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 5508070 [hpdcnln001:13669] crcp:bkmrk: [[40230,1],0] --> [[40230,1],1] Sending bookmark (S[5508070] R[ 5502]) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] --> [[40230,1],0] Sending bookmark (S[ 5502] R[5507970]) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Received bookmark (S[5508070] R[ 5502]) vs. (S[ 5502] R[5507970]) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Matched Sent (5508070) = Total Recv. (5507970). I need 100. [hpdcnln001:13669] crcp:bkmrk: [[40230,1],0] <-- [[40230,1],1] Received bookmark (S[ 5502] R[5507970]) vs. (S[5508070] R[ 5502]) [hpdcnln001:13669] crcp:bkmrk: [[40230,1],0] --> [[40230,1],1] Total Sent (5508070) = Matched Recv. (5507970). Peer needs 100. [hpdcnln001:13669] crcp:bkmrk: send_msg_details: Stage 1: [M/A/D/AD] [ 0/ 0/5502/ 0] (iSend) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Stage 0: Ck.Drain: [TR 5507970/MS 5508070] sent 5502, unres 5502, res 0 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Stage 1: Ck.Drain: [TR 5507970/MS 5508070] sent 5502, unres 0, res 0 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Recv Detail: Stage --: [100 / 0] [ 0, 0, F] [hpdcnln001:13669] ************************** [hpdcnln001:13669] send_msg_details: 0, 0 = Continue... [5502 / 0] [hpdcnln001:13669] ************************** [hpdcnln001:13669] crcp:bkmrk: send_msg_details: Stage 2: [M/A/D/AD] [ 0/ 0/5502/ 0] (iSend) [ 0, 0, F] [ 0, 0] [hpdcnln001:13669] crcp:bkmrk: send_msg_details: Stage 1: [M/A/D/AD] [ 0/ 0/2751282/ 0] (iSend) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Stage 0: Ck.Drain: [TR 5507970/MS 5508070] sent 2751282, unres 2751282, res 0 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Stage 1: Ck.Drain: [TR 5507970/MS 5508070] sent 2751282, unres 50, res 0 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Stage 2: Ck.Drain: [TR 5507970/MS 5508070] sent 2751282, unres 50, res 0, active 0 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Stage 3: Ck.Drain: [TR 5507970/MS 5508070] sent 2751282, unres 50, res 0, active 0 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Create Drain Msg: Posting 50 = min( 0 / 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 0 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 1 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 2 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 3 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 4 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 5 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 6 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 7 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 8 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 9 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 10 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 11 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 12 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 13 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 14 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 15 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 16 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 17 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 18 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 19 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 20 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 21 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 22 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 23 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 24 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 25 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 26 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 27 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 28 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 29 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 30 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 31 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 32 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 33 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 34 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 35 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 36 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 37 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 38 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 39 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 40 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 41 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 42 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 43 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 44 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 45 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 46 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 47 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 48 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 49 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Added 50 messages to the drained list (size = 1) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Stage 4: Ck.Drain: [TR 5508020/MS 5508070] sent 2751282, unres 0, res 50, active 0 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Recv Detail: Stage --: [100 / 50] [ 50, 50, F] [hpdcnln001:13669] ************************** [hpdcnln001:13669] send_msg_details: 50, 50 = Continue... [2751282 / 0] [hpdcnln001:13669] ************************** [hpdcnln001:13669] crcp:bkmrk: send_msg_details: Stage 2: [M/A/D/AD] [ 50/ 0/2751282/ 0] (iSend) [ 50, 50, F] [ 0, 0] [hpdcnln001:13669] crcp:bkmrk: send_msg_details: Stage 1: [M/A/D/AD] [ 0/ 0/2751286/ 0] (iSend) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Stage 0: Ck.Drain: [TR 5508020/MS 5508070] sent 2751286, unres 2751286, res 0 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Stage 1: Ck.Drain: [TR 5508020/MS 5508070] sent 2751286, unres 50, res 0 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Stage 2: Ck.Drain: [TR 5508020/MS 5508070] sent 2751286, unres 50, res 0, active 0 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Stage 3: Ck.Drain: [TR 5508020/MS 5508070] sent 2751286, unres 50, res 0, active 0 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Create Drain Msg: Posting 50 = min( 0 / 50) [hpdcnln001:13670] crcp:bkmrk: find_drain_msg(): Compare [4, 1, -17, 0] to [8, 1, -17, 0] [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 0 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 1 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 2 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 3 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 4 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 5 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 6 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 7 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 8 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 9 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 10 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 11 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 12 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 13 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 14 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 15 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 16 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 17 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 18 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 19 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 20 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 21 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 22 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 23 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 24 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 25 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 26 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 27 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 28 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 29 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 30 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 31 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 32 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 33 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 34 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 35 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 36 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 37 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 38 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 39 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 40 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 41 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 42 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 43 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 44 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 45 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 46 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 47 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 48 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] --> Find Content: Posting ( 49 of 50) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Added 50 messages to the drained list (size = 2) [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Stage 4: Ck.Drain: [TR 5508070/MS 5508070] sent 2751286, unres 0, res 50, active 0 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Recv Detail: Stage --: [100 / 100] [ 50, 100, T] [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Posting a message to be drained from rank 0. [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] <-- [[40230,1],0] Will be draining 100 messages from this peer. Total 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Waiting on 100 messages to drain [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 0 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 1 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 2 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 3 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 4 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 5 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 6 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 7 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 8 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 9 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 10 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 11 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 12 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 13 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 14 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 15 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 16 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 17 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 18 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 19 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 20 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 21 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 22 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 23 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 24 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 25 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 26 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 27 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 28 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 29 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 30 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 31 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 32 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 33 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 34 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 35 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 36 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 37 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 38 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 39 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 40 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 41 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 42 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 43 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 44 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 45 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 46 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 47 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 48 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 49 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 50 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 51 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 52 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 53 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 54 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 55 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 56 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 57 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 58 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 59 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 60 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 61 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 62 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 63 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 64 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 65 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 66 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 67 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 68 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 69 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 70 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 71 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 72 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 73 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 74 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 75 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 76 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 77 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 78 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 79 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 80 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 81 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 82 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 83 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 84 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 85 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 86 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 87 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 88 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 89 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 90 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 91 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 92 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 93 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 94 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 95 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 96 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 97 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 98 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Request Wait: Done with idx 99 of 100 [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] --> [[40230,1],0] Send ACKs to Peer [hpdcnln001:13670] crcp:bkmrk: [[40230,1],1] Coordination Finished... [hpdcnln001:13669] ************************** [hpdcnln001:13669] send_msg_details: 50, 100 = Done [2751286 / 0] [hpdcnln001:13669] ************************** [hpdcnln001:13669] crcp:bkmrk: send_msg_details: Stage 2: [M/A/D/AD] [ 50/ 0/2751286/ 0] (iSend) [ 50, 100, T] [ 50, 50] [hpdcnln001:13669] crcp:bkmrk: [[40230,1],0] <-> [[40230,1],1] Message Inflight! Will wait on ACK from this peer. [hpdcnln001:13669] crcp:bkmrk: [[40230,1],0] Wait on 1 Drain ACK Messages. [hpdcnln001:13669] crcp:bkmrk: [[40230,1],0] --> [[40230,1],1] Received ACK of FLUSH from peer [hpdcnln001:13669] crcp:bkmrk: [[40230,1],0] Waiting on 0 messages to drain [hpdcnln001:13669] crcp:bkmrk: [[40230,1],0] Waiting on 1 Drain ACK messages [hpdcnln001:13669] crcp:bkmrk: [[40230,1],0] Coordination Finished... [hpdcnln001:13668] [[40230,0],0] orted_recv_cmd: received message from [[40230,1],0] [hpdcnln001:13668] defining message event: ../../orte/orted/orted_comm.c 159 [hpdcnln001:13668] [[40230,0],0] orted_recv_cmd: reissued recv [hpdcnln001:13668] [[40230,0],0] orte:daemon:cmd:processor called by [[40230,1],0] for tag 1 [hpdcnln001:13668] [[40230,0],0] orte:daemon:cmd:processor: processing commands completed [hpdcnln001:13669] progressed_wait: ../../../../../orte/mca/grpcomm/bad/grpcomm_bad_module.c 270 [hpdcnln001:13668] [[40230,0],0] orted_recv_cmd: received message from [[40230,1],1] [hpdcnln001:13668] defining message event: ../../orte/orted/orted_comm.c 159 [hpdcnln001:13668] [[40230,0],0] orted_recv_cmd: reissued recv [hpdcnln001:13668] [[40230,0],0] orte:daemon:cmd:processor called by [[40230,1],1] for tag 1 [hpdcnln001:13668] defining message event: ../../../../../orte/mca/grpcomm/bad/grpcomm_bad_module.c 183 [hpdcnln001:13668] [[40230,0],0] orte:daemon:cmd:processor: processing commands completed [hpdcnln001:13668] [[40230,0],0] orte:daemon:cmd:processor called by [[40230,0],0] for tag 1 [hpdcnln001:13668] [[40230,0],0] orted:comm:message_local_procs delivering message to job [40230,1] tag 17 [hpdcnln001:13668] [[40230,0],0] orte:daemon:send_relay [hpdcnln001:13668] [[40230,0],0] orte:daemon:send_relay - recipient list is empty! [hpdcnln001:13670] progressed_wait: ../../../../../orte/mca/grpcomm/bad/grpcomm_bad_module.c 270 [hpdcnln001:13668] [[40230,0],0] orted_recv_cmd: received message from [[40230,1],1] [hpdcnln001:13668] defining message event: ../../orte/orted/orted_comm.c 159 [hpdcnln001:13668] [[40230,0],0] orted_recv_cmd: reissued recv [hpdcnln001:13670] progressed_wait: ../../../../../orte/mca/grpcomm/bad/grpcomm_bad_module.c 394 [hpdcnln001:13668] [[40230,0],0] orte:daemon:cmd:processor called by [[40230,1],1] for tag 1 [hpdcnln001:13668] [[40230,0],0] orte:daemon:cmd:processor: processing commands completed [hpdcnln001:13668] [[40230,0],0] orted_recv_cmd: received message from [[40230,1],0] [hpdcnln001:13668] defining message event: ../../orte/orted/orted_comm.c 159 [hpdcnln001:13668] [[40230,0],0] orted_recv_cmd: reissued recv [hpdcnln001:13668] [[40230,0],0] orte:daemon:cmd:processor called by [[40230,1],0] for tag 1 [hpdcnln001:13668] defining message event: ../../../../../orte/mca/grpcomm/bad/grpcomm_bad_module.c 183 [hpdcnln001:13668] [[40230,0],0] orte:daemon:cmd:processor: processing commands completed [hpdcnln001:13668] [[40230,0],0] orte:daemon:cmd:processor called by [[40230,0],0] for tag 1 [hpdcnln001:13668] [[40230,0],0] orted:comm:message_local_procs delivering message to job [40230,1] tag 15 [hpdcnln001:13668] [[40230,0],0] orte:daemon:send_relay [hpdcnln001:13668] [[40230,0],0] orte:daemon:send_relay - recipient list is empty! [hpdcnln001:13669] progressed_wait: ../../../../../orte/mca/grpcomm/bad/grpcomm_bad_module.c 394 [hpdcnln001:13668] [[40230,0],0] orted_recv_cmd: received message from [[40230,1],1] [hpdcnln001:13668] defining message event: ../../orte/orted/orted_comm.c 159 [hpdcnln001:13668] [[40230,0],0] orted_recv_cmd: reissued recv [hpdcnln001:13670] progressed_wait: ../../../../../orte/mca/grpcomm/bad/grpcomm_bad_module.c 270 [hpdcnln001:13668] [[40230,0],0] orte:daemon:cmd:processor called by [[40230,1],1] for tag 1 [hpdcnln001:13668] [[40230,0],0] orte:daemon:cmd:processor: processing commands completed [hpdcnln001:13668] [[40230,0],0] orted_recv_cmd: received message from [[40230,1],0] [hpdcnln001:13668] defining message event: ../../orte/orted/orted_comm.c 159 [hpdcnln001:13668] [[40230,0],0] orted_recv_cmd: reissued recv [hpdcnln001:13668] [[40230,0],0] orte:daemon:cmd:processor called by [[40230,1],0] for tag 1 [hpdcnln001:13668] defining message event: ../../../../../orte/mca/grpcomm/bad/grpcomm_bad_module.c 183 [hpdcnln001:13668] [[40230,0],0] orte:daemon:cmd:processor: processing commands completed [hpdcnln001:13669] progressed_wait: ../../../../../orte/mca/grpcomm/bad/grpcomm_bad_module.c 270 [hpdcnln001:13668] [[40230,0],0] orte:daemon:cmd:processor called by [[40230,0],0] for tag 1 [hpdcnln001:13668] [[40230,0],0] orted:comm:message_local_procs delivering message to job [40230,1] tag 17 [hpdcnln001:13668] [[40230,0],0] orte:daemon:send_relay [hpdcnln001:13668] [[40230,0],0] orte:daemon:send_relay - recipient list is empty! [hpdcnln001:13668] defining message event: ../../../../../orte/mca/grpcomm/bad/grpcomm_bad_module.c 183 [hpdcnln001:13668] defining message event: ../../../../../orte/mca/grpcomm/bad/grpcomm_bad_module.c 183 [hpdcnln001:13668] defining message event: ../../../../orte/mca/odls/base/odls_base_default_fns.c 2171 [hpdcnln001:13668] [[40230,0],0] got show_help from [[40230,1],1] [hpdcnln001:13670] *** An error occurred in MPI_Bcast [hpdcnln001:13670] *** on communicator MPI_COMM_WORLD [hpdcnln001:13670] *** MPI_ERR_TRUNCATE: message truncated [hpdcnln001:13670] *** MPI_ERRORS_ARE_FATAL (your MPI job will now abort) [hpdcnln001:13668] [[40230,0],0] orte:daemon:cmd:processor called by [[40230,0],0] for tag 1 [hpdcnln001:13668] [[40230,0],0] orted:comm:message_local_procs delivering message to job [40230,0] tag 25 [hpdcnln001:13668] [[40230,0],0] orte:daemon:cmd:processor called by [[40230,0],0] for tag 1 [hpdcnln001:13668] [[40230,0],0] orted:comm:message_local_procs delivering message to job [40230,0] tag 25 [hpdcnln001:13668] defining message event: ../../../../../orte/mca/iof/hnp/iof_hnp_read.c 281 [hpdcnln001:13670] crcp:bkmrk: find_drain_msg(): Compare [8, 1, -17, 0] to [8, 1, -17, 0] [hpdcnln001:13670] crcp:bkmrk: drain_check_recv(): Matched a drained message... [hpdcnln001:13670] crcp:bkmrk: request_complete: No match found for this request :( 0, 0 ): [0/0,0] [hpdcnln001:13670] crcp:bkmrk: find_drain_msg(): Compare [4, 1, -17, 0] to [8, 1, -17, 0] [hpdcnln001:13670] crcp:bkmrk: find_drain_msg(): Compare [4, 1, -17, 0] to [4, 1, -17, 0] [hpdcnln001:13670] crcp:bkmrk: drain_check_recv(): Matched a drained message... [hpdcnln001:13670] crcp:bkmrk: request_complete: No match found for this request :( 0, 0 ): [0/0,0] [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 1 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 2 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 3 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 4 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 5 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 6 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 7 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 8 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 9 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 10 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 11 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 12 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 13 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 14 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 15 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 16 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 17 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 18 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 19 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 20 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 21 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 22 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 23 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 24 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 25 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 26 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 27 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 28 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 29 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 30 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 31 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 32 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 33 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 34 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 35 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 36 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 37 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 38 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 39 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 40 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 41 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 42 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 43 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 44 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 45 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 46 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 47 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 48 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 49 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 50 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 51 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 52 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 53 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 54 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 55 [hpdcnln001:13669] crcp:bkmrk: req_Process 0 of 2 on hpdcnln001, number =3 Process 0, new_number = 4 [hpdcnln001:13668] [[40230,0],0] orte:daemon:cmd:processor called by [[40230,0],0] for tag 1 [hpdcnln001:13668] [[40230,0],0] orte:daemon:cmd:processor: processing commands completed [hpdcnln001:13668] [[40230,0],0] orte:daemon:send_relay [hpdcnln001:13668] [[40230,0],0] orte:daemon:send_relay - recipient list is empty! [hpdcnln001:13668] [[40230,0],0] orte:daemon:send_relay [hpdcnln001:13668] [[40230,0],0] orte:daemon:send_relay - recipient list is empty! complete: Matched an iSend: total = 56 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 57 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 58 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 59 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 60 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 61 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 62 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 63 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 64 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 65 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 66 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 67 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 68 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 69 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 70 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 71 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 72 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 73 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 74 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 75 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 76 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 77 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 78 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 79 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 80 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 81 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 82 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 83 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 84 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 85 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 86 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 87 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 88 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 89 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 90 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 91 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 92 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 93 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 94 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 95 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 96 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 97 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 98 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 99 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 100 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 101 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 102 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 103 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 104 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 105 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 106 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 107 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 108 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 109 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 110 [hpdcnln001:13669] crcp:bkmrk: req_complete: Match[hpdcnln001:13668] [[40230,0],0] orte:daemon:cmd:processor called by [[40230,0],0] for tag 1 [hpdcnln001:13668] defining message event: ../../../../orte/mca/odls/base/odls_base_default_fns.c 1820 [hpdcnln001:13668] [[40230,0],0] orte:daemon:cmd:processor: processing commands completed ed an iSend: total = 111 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 112 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 113 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 114 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 115 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 116 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 117 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 118 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 119 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 120 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 121 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 122 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 123 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 124 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 125 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 126 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 127 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 128 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 129 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 130 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 131 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 132 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 133 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 134 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 135 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 136 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 137 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 138 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 139 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 140 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 141 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 142 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 143 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 144 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 145 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 146 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 147 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 148 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 149 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 150 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 151 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 152 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 153 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 154 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 155 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 156 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 157 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 158 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 159 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 160 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 161 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 162 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 163 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 164 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 165 [hpdcnln001:13669] cr[hpdcnln001:13668] [[40230,0],0]:../../../../orte/mca/plm/base/plm_base_receive.c(254) updating exit status to 15 [hpdcnln001:13668] defining message event: ../../../../../orte/mca/grpcomm/bad/grpcomm_bad_module.c 183 [hpdcnln001:13668] [[40230,0],0] calling job_complete trigger -------------------------------------------------------------------------- mpirun has exited due to process rank 1 with PID 13670 on node hpdcnln001 exiting without calling "finalize". This may have caused other processes in the application to be terminated by signals sent by mpirun (as reported here). -------------------------------------------------------------------------- [hpdcnln001:13668] defining message event: ../../../../../orte/mca/grpcomm/bad/grpcomm_bad_module.c 183 cp:bkmrk: req_complete: Matched an iSend: total = 166 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 167 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 168 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 169 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 170 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 171 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 172 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 173 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 174 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 175 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 176 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 177 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 178 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 179 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 180 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 181 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 182 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 183 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 184 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 185 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 186 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 187 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 188 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 189 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 190 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 191 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 192 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 193 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 194 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 195 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 196 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 197 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 198 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 199 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 200 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 201 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 202 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 203 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 204 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 205 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 206 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 207 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 208 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 209 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 210 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 211 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 212 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 213 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 214 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 215 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 216 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 217 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 218 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 219 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: tota[hpdcnln001:13668] [[40230,0],0] orte:daemon:cmd:processor called by [[40230,0],0] for tag 1 [hpdcnln001:13668] defining message event: ../../../../orte/mca/odls/base/odls_base_default_fns.c 2383 [hpdcnln001:13668] [[40230,0],0] orte:daemon:send_relay [hpdcnln001:13668] [[40230,0],0] orte:daemon:send_relay - recipient list is empty! [hpdcnln001:13668] [[40230,0],0] orte:daemon:cmd:processor called by [[40230,0],0] for tag 1 [hpdcnln001:13668] [[40230,0],0] calling orted_exit trigger l = 220 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 221 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 222 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 223 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 224 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 225 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 226 [hpdcnln001:13669] crcp:bkmrk: req_complete: Matched an iSend: total = 227