This source file includes following definitions.
- append_frag_to_list
- append_frag_to_umq
- append_frag_to_ordered_list
- remove_head_from_ordered_list
- check_cantmatch_for_match
- mca_pml_ob1_recv_frag_callback_match
- mca_pml_ob1_recv_frag_callback_rndv
- mca_pml_ob1_recv_frag_callback_rget
- mca_pml_ob1_recv_frag_callback_ack
- mca_pml_ob1_recv_frag_callback_frag
- mca_pml_ob1_recv_frag_callback_put
- mca_pml_ob1_recv_frag_callback_fin
- get_posted_recv
- get_next_posted_recv
- match_incomming
- match_incomming_no_any_source
- match_one
- mca_pml_ob1_recv_frag_match
- mca_pml_ob1_recv_frag_match_proc
   1 
   2 
   3 
   4 
   5 
   6 
   7 
   8 
   9 
  10 
  11 
  12 
  13 
  14 
  15 
  16 
  17 
  18 
  19 
  20 
  21 
  22 
  23 
  24 
  25 
  26 
  27 
  28 
  29 
  30 
  31 
  32 
  33 #include "ompi_config.h"
  34 
  35 #include "opal/class/opal_list.h"
  36 #include "opal/threads/mutex.h"
  37 #include "opal/prefetch.h"
  38 
  39 #include "ompi/constants.h"
  40 #include "ompi/communicator/communicator.h"
  41 #include "ompi/mca/pml/pml.h"
  42 #include "ompi/peruse/peruse-internal.h"
  43 #include "ompi/memchecker.h"
  44 #include "ompi/runtime/ompi_spc.h"
  45 
  46 #include "pml_ob1.h"
  47 #include "pml_ob1_comm.h"
  48 #include "pml_ob1_recvfrag.h"
  49 #include "pml_ob1_recvreq.h"
  50 #include "pml_ob1_sendreq.h"
  51 #include "pml_ob1_hdr.h"
  52 #if OPAL_CUDA_SUPPORT
  53 #include "opal/datatype/opal_datatype_cuda.h"
  54 #include "opal/mca/common/cuda/common_cuda.h"
  55 #endif 
  56 
  57 OBJ_CLASS_INSTANCE( mca_pml_ob1_buffer_t,
  58                     opal_free_list_item_t,
  59                     NULL,
  60                     NULL );
  61 
  62 OBJ_CLASS_INSTANCE( mca_pml_ob1_recv_frag_t,
  63                     opal_list_item_t,
  64                     NULL,
  65                     NULL );
  66 
  67 
  68 
  69 
  70 
  71 
  72 
  73 
  74 
  75 
  76 
  77 static void
  78 append_frag_to_list(opal_list_t *queue, mca_btl_base_module_t *btl,
  79                     mca_pml_ob1_match_hdr_t *hdr, mca_btl_base_segment_t* segments,
  80                     size_t num_segments, mca_pml_ob1_recv_frag_t* frag)
  81 {
  82     if(NULL == frag) {
  83         MCA_PML_OB1_RECV_FRAG_ALLOC(frag);
  84         MCA_PML_OB1_RECV_FRAG_INIT(frag, hdr, segments, num_segments, btl);
  85     }
  86     opal_list_append(queue, (opal_list_item_t*)frag);
  87 }
  88 
  89 #if MCA_PML_OB1_CUSTOM_MATCH
  90 
  91 static void
  92 append_frag_to_umq(custom_match_umq *queue, mca_btl_base_module_t *btl,
  93                    mca_pml_ob1_match_hdr_t *hdr, mca_btl_base_segment_t* segments,
  94                    size_t num_segments, mca_pml_ob1_recv_frag_t* frag)
  95 {
  96   if(NULL == frag) {
  97     MCA_PML_OB1_RECV_FRAG_ALLOC(frag);
  98     MCA_PML_OB1_RECV_FRAG_INIT(frag, hdr, segments, num_segments, btl);
  99   }
 100   custom_match_umq_append(queue, hdr->hdr_tag, hdr->hdr_src, frag);
 101 }
 102 
 103 #endif
 104 
 105 
 106 
 107 
 108 
 109 
 110 
 111 
 112 
 113 
 114 
 115 
 116 
 117 void
 118 append_frag_to_ordered_list(mca_pml_ob1_recv_frag_t** queue,
 119                             mca_pml_ob1_recv_frag_t *frag,
 120                             uint16_t seq)
 121 {
 122     mca_pml_ob1_recv_frag_t  *prior, *next;
 123     mca_pml_ob1_match_hdr_t *hdr;
 124 
 125     frag->super.super.opal_list_next = (opal_list_item_t*)frag;
 126     frag->super.super.opal_list_prev = (opal_list_item_t*)frag;
 127     frag->range = NULL;
 128     hdr = &frag->hdr.hdr_match;
 129 
 130     if( NULL == *queue ) {  
 131         *queue = frag;
 132         return;
 133     }
 134 
 135     prior = *queue;
 136     assert(hdr->hdr_seq != prior->hdr.hdr_match.hdr_seq);
 137 
 138     
 139 
 140 
 141     if( hdr->hdr_seq < prior->hdr.hdr_match.hdr_seq ) {
 142         uint16_t d1, d2 = prior->hdr.hdr_match.hdr_seq - hdr->hdr_seq;
 143         do {
 144             d1 = d2;
 145             prior = (mca_pml_ob1_recv_frag_t*)(prior->super.super.opal_list_prev);
 146             d2 = prior->hdr.hdr_match.hdr_seq - hdr->hdr_seq;
 147         } while( (hdr->hdr_seq < prior->hdr.hdr_match.hdr_seq) &&
 148                  (d1 > d2) && (prior != *queue) );
 149     } else {
 150         uint16_t prior_seq = prior->hdr.hdr_match.hdr_seq,
 151         next_seq = ((mca_pml_ob1_recv_frag_t*)(prior->super.super.opal_list_next))->hdr.hdr_match.hdr_seq;
 152         
 153         while( (hdr->hdr_seq > prior_seq) && (hdr->hdr_seq > next_seq) && (prior_seq < next_seq) ) {
 154             prior_seq = next_seq;
 155             prior = (mca_pml_ob1_recv_frag_t*)(prior->super.super.opal_list_next);
 156             next_seq = ((mca_pml_ob1_recv_frag_t*)(prior->super.super.opal_list_next))->hdr.hdr_match.hdr_seq;
 157         }
 158     }
 159 
 160     
 161     mca_pml_ob1_recv_frag_t* parent = prior;
 162 
 163     
 164     if( NULL == parent->range ) {
 165         if( (parent->hdr.hdr_match.hdr_seq + 1) == hdr->hdr_seq ) {
 166             parent->range = (mca_pml_ob1_recv_frag_t*)frag;
 167             goto merge_ranges;
 168         }
 169         
 170     } else {
 171         
 172         mca_pml_ob1_recv_frag_t* largest = (mca_pml_ob1_recv_frag_t*)parent->range->super.super.opal_list_prev;
 173         if( (largest->hdr.hdr_match.hdr_seq + 1) == hdr->hdr_seq ) {
 174             
 175             frag->super.super.opal_list_prev = (opal_list_item_t*)largest;
 176             frag->super.super.opal_list_next = largest->super.super.opal_list_next;
 177             frag->super.super.opal_list_prev->opal_list_next = (opal_list_item_t*)frag;
 178             frag->super.super.opal_list_next->opal_list_prev = (opal_list_item_t*)frag;
 179             goto merge_ranges;
 180         }
 181         
 182     }
 183 
 184     frag->super.super.opal_list_prev = (opal_list_item_t*)prior;
 185     frag->super.super.opal_list_next = (opal_list_item_t*)prior->super.super.opal_list_next;
 186     frag->super.super.opal_list_prev->opal_list_next = (opal_list_item_t*)frag;
 187     frag->super.super.opal_list_next->opal_list_prev = (opal_list_item_t*)frag;
 188     parent = frag;  
 189 
 190     
 191     if( parent->hdr.hdr_match.hdr_seq >= seq )
 192         if( abs(parent->hdr.hdr_match.hdr_seq - seq) < abs((*queue)->hdr.hdr_match.hdr_seq - seq))
 193             *queue = parent;
 194 
 195  merge_ranges:
 196     
 197     next = (mca_pml_ob1_recv_frag_t*)parent->super.super.opal_list_next;
 198     uint16_t upper = parent->hdr.hdr_match.hdr_seq;
 199     if( NULL != parent->range ) {
 200         upper = ((mca_pml_ob1_recv_frag_t*)parent->range->super.super.opal_list_prev)->hdr.hdr_match.hdr_seq;
 201     }
 202     if( (upper + 1) == next->hdr.hdr_match.hdr_seq ) {
 203         
 204         next->super.super.opal_list_next->opal_list_prev = (opal_list_item_t*)parent;
 205         parent->super.super.opal_list_next = next->super.super.opal_list_next;
 206         
 207         if( NULL != next->range ) {
 208             next->super.super.opal_list_next = (opal_list_item_t*)next->range;
 209             next->super.super.opal_list_prev = next->range->super.super.opal_list_prev;
 210             next->super.super.opal_list_next->opal_list_prev = (opal_list_item_t*)next;
 211             next->super.super.opal_list_prev->opal_list_next = (opal_list_item_t*)next;
 212             next->range = NULL;
 213         } else {
 214             next->super.super.opal_list_prev = (opal_list_item_t*)next;
 215             next->super.super.opal_list_next = (opal_list_item_t*)next;
 216         }
 217         if( NULL == parent->range ) {
 218             parent->range = next;
 219         } else {
 220             
 221             frag = (mca_pml_ob1_recv_frag_t*)parent->range->super.super.opal_list_prev;
 222             
 223             frag->super.super.opal_list_next = (opal_list_item_t*)next;
 224             parent->range->super.super.opal_list_prev = next->super.super.opal_list_prev;
 225             next->super.super.opal_list_prev->opal_list_next = (opal_list_item_t*)parent->range;
 226             next->super.super.opal_list_prev = (opal_list_item_t*)frag;
 227         }
 228         if( next == *queue )
 229             *queue = parent;
 230     }
 231 }
 232 
 233 
 234 
 235 
 236 static mca_pml_ob1_recv_frag_t*
 237 remove_head_from_ordered_list(mca_pml_ob1_recv_frag_t** queue)
 238 {
 239     mca_pml_ob1_recv_frag_t* frag = *queue;
 240     
 241     if( NULL == *queue )
 242         return NULL;
 243     if( NULL == frag->range ) {
 244         
 245         if( frag->super.super.opal_list_next == (opal_list_item_t*)frag ) {
 246             
 247 
 248             *queue = NULL;
 249         } else {
 250             
 251             *queue = (mca_pml_ob1_recv_frag_t*)frag->super.super.opal_list_next;
 252             frag->super.super.opal_list_next->opal_list_prev = frag->super.super.opal_list_prev;
 253             frag->super.super.opal_list_prev->opal_list_next = frag->super.super.opal_list_next;
 254         }
 255     } else {
 256         
 257         mca_pml_ob1_recv_frag_t* range = frag->range;
 258         frag->range = NULL;
 259         *queue = (mca_pml_ob1_recv_frag_t*)range;
 260         if( range->super.super.opal_list_next == (opal_list_item_t*)range ) {
 261             
 262             assert( range->super.super.opal_list_prev == (opal_list_item_t*)range );
 263             range->range = NULL;
 264         } else {
 265             range->range = (mca_pml_ob1_recv_frag_t*)range->super.super.opal_list_next;
 266             
 267             range->super.super.opal_list_next->opal_list_prev = range->super.super.opal_list_prev;
 268             range->super.super.opal_list_prev->opal_list_next = range->super.super.opal_list_next;
 269         }
 270         
 271         if( frag->super.super.opal_list_next == (opal_list_item_t*)frag ) {
 272             range->super.super.opal_list_next = (opal_list_item_t*)range;
 273             range->super.super.opal_list_prev = (opal_list_item_t*)range;
 274         } else {
 275             range->super.super.opal_list_next = frag->super.super.opal_list_next;
 276             range->super.super.opal_list_prev = frag->super.super.opal_list_prev;
 277             range->super.super.opal_list_next->opal_list_prev = (opal_list_item_t*)range;
 278             range->super.super.opal_list_prev->opal_list_next = (opal_list_item_t*)range;
 279         }
 280     }
 281     frag->super.super.opal_list_next = NULL;
 282     frag->super.super.opal_list_prev = NULL;
 283     return frag;
 284 }
 285 
 286 
 287 
 288 
 289 
 290 
 291 
 292 
 293 
 294 
 295 
 296 static int mca_pml_ob1_recv_frag_match( mca_btl_base_module_t *btl,
 297                                         mca_pml_ob1_match_hdr_t *hdr,
 298                                         mca_btl_base_segment_t* segments,
 299                                         size_t num_segments,
 300                                         int type );
 301 
 302 
 303 
 304 
 305 
 306 
 307 
 308 
 309 
 310 
 311 
 312 
 313 
 314 
 315 static int
 316 mca_pml_ob1_recv_frag_match_proc( mca_btl_base_module_t *btl,
 317                                   ompi_communicator_t* comm_ptr,
 318                                   mca_pml_ob1_comm_proc_t *proc,
 319                                   mca_pml_ob1_match_hdr_t *hdr,
 320                                   mca_btl_base_segment_t* segments,
 321                                   size_t num_segments,
 322                                   int type,
 323                                   mca_pml_ob1_recv_frag_t* frag );
 324 
 325 static mca_pml_ob1_recv_request_t*
 326 match_one(mca_btl_base_module_t *btl,
 327           mca_pml_ob1_match_hdr_t *hdr, mca_btl_base_segment_t* segments,
 328           size_t num_segments, ompi_communicator_t *comm_ptr,
 329           mca_pml_ob1_comm_proc_t *proc,
 330           mca_pml_ob1_recv_frag_t* frag);
 331 
 332 mca_pml_ob1_recv_frag_t*
 333 check_cantmatch_for_match(mca_pml_ob1_comm_proc_t *proc)
 334 {
 335     mca_pml_ob1_recv_frag_t *frag = proc->frags_cant_match;
 336 
 337     if( (NULL != frag) && (frag->hdr.hdr_match.hdr_seq == proc->expected_sequence) ) {
 338         return remove_head_from_ordered_list(&proc->frags_cant_match);
 339     }
 340     return NULL;
 341 }
 342 
 343 void mca_pml_ob1_recv_frag_callback_match(mca_btl_base_module_t* btl,
 344                                           mca_btl_base_tag_t tag,
 345                                           mca_btl_base_descriptor_t* des,
 346                                           void* cbdata )
 347 {
 348     mca_btl_base_segment_t* segments = des->des_segments;
 349     mca_pml_ob1_match_hdr_t* hdr = (mca_pml_ob1_match_hdr_t*)segments->seg_addr.pval;
 350     ompi_communicator_t *comm_ptr;
 351     mca_pml_ob1_recv_request_t *match = NULL;
 352     mca_pml_ob1_comm_t *comm;
 353     mca_pml_ob1_comm_proc_t *proc;
 354     size_t num_segments = des->des_segment_count;
 355     size_t bytes_received = 0;
 356 
 357     assert(num_segments <= MCA_BTL_DES_MAX_SEGMENTS);
 358 
 359     if( OPAL_UNLIKELY(segments->seg_len < OMPI_PML_OB1_MATCH_HDR_LEN) ) {
 360         return;
 361     }
 362     ob1_hdr_ntoh(((mca_pml_ob1_hdr_t*) hdr), MCA_PML_OB1_HDR_TYPE_MATCH);
 363 
 364     
 365     comm_ptr = ompi_comm_lookup(hdr->hdr_ctx);
 366     if(OPAL_UNLIKELY(NULL == comm_ptr)) {
 367         
 368 
 369 
 370 
 371 
 372 
 373 
 374         append_frag_to_list( &mca_pml_ob1.non_existing_communicator_pending,
 375                              btl, hdr, segments, num_segments, NULL );
 376         return;
 377     }
 378     comm = (mca_pml_ob1_comm_t *)comm_ptr->c_pml_comm;
 379 
 380     
 381     proc = mca_pml_ob1_peer_lookup (comm_ptr, hdr->hdr_src);
 382 
 383     
 384 
 385 
 386 
 387 
 388 
 389     PERUSE_TRACE_MSG_EVENT(PERUSE_COMM_MSG_ARRIVED, comm_ptr,
 390                            hdr->hdr_src, hdr->hdr_tag, PERUSE_RECV);
 391 
 392     
 393 
 394 
 395 
 396 
 397 
 398 
 399     OB1_MATCHING_LOCK(&comm->matching_lock);
 400 
 401     if (!OMPI_COMM_CHECK_ASSERT_ALLOW_OVERTAKE(comm_ptr)) {
 402         
 403 
 404 
 405 
 406         if(OPAL_UNLIKELY(((uint16_t) hdr->hdr_seq) != ((uint16_t) proc->expected_sequence))) {
 407             mca_pml_ob1_recv_frag_t* frag;
 408             MCA_PML_OB1_RECV_FRAG_ALLOC(frag);
 409             MCA_PML_OB1_RECV_FRAG_INIT(frag, hdr, segments, num_segments, btl);
 410             append_frag_to_ordered_list(&proc->frags_cant_match, frag, proc->expected_sequence);
 411             SPC_RECORD(OMPI_SPC_OUT_OF_SEQUENCE, 1);
 412             OB1_MATCHING_UNLOCK(&comm->matching_lock);
 413             return;
 414         }
 415 
 416         
 417         proc->expected_sequence++;
 418     }
 419 
 420     
 421 
 422 
 423 
 424     PERUSE_TRACE_MSG_EVENT(PERUSE_COMM_SEARCH_POSTED_Q_BEGIN, comm_ptr,
 425                            hdr->hdr_src, hdr->hdr_tag, PERUSE_RECV);
 426 
 427     match = match_one(btl, hdr, segments, num_segments, comm_ptr, proc, NULL);
 428 
 429     
 430 
 431 
 432 
 433     PERUSE_TRACE_MSG_EVENT(PERUSE_COMM_SEARCH_POSTED_Q_END, comm_ptr,
 434                            hdr->hdr_src, hdr->hdr_tag, PERUSE_RECV);
 435 
 436     
 437     OB1_MATCHING_UNLOCK(&comm->matching_lock);
 438 
 439     if(OPAL_LIKELY(match)) {
 440         bytes_received = segments->seg_len - OMPI_PML_OB1_MATCH_HDR_LEN;
 441         
 442 
 443 
 444 
 445 
 446         match->req_recv.req_bytes_packed = bytes_received + (num_segments-1);
 447 
 448         MCA_PML_OB1_RECV_REQUEST_MATCHED(match, hdr);
 449         if(match->req_bytes_expected > 0) {
 450             struct iovec iov[MCA_BTL_DES_MAX_SEGMENTS];
 451             uint32_t iov_count = 1;
 452 
 453             
 454 
 455 
 456             MEMCHECKER(
 457                        memchecker_call(&opal_memchecker_base_mem_defined,
 458                                        match->req_recv.req_base.req_addr,
 459                                        match->req_recv.req_base.req_count,
 460                                        match->req_recv.req_base.req_datatype);
 461                        );
 462 
 463             iov[0].iov_len = bytes_received;
 464             iov[0].iov_base = (IOVBASE_TYPE*)((unsigned char*)segments->seg_addr.pval +
 465                                               OMPI_PML_OB1_MATCH_HDR_LEN);
 466             while (iov_count < num_segments) {
 467                 bytes_received += segments[iov_count].seg_len;
 468                 iov[iov_count].iov_len = segments[iov_count].seg_len;
 469                 iov[iov_count].iov_base = (IOVBASE_TYPE*)((unsigned char*)segments[iov_count].seg_addr.pval);
 470                 iov_count++;
 471             }
 472             opal_convertor_unpack( &match->req_recv.req_base.req_convertor,
 473                                    iov,
 474                                    &iov_count,
 475                                    &bytes_received );
 476             match->req_bytes_received = bytes_received;
 477             SPC_USER_OR_MPI(match->req_recv.req_base.req_ompi.req_status.MPI_TAG, (ompi_spc_value_t)bytes_received,
 478                             OMPI_SPC_BYTES_RECEIVED_USER, OMPI_SPC_BYTES_RECEIVED_MPI);
 479             
 480 
 481 
 482             MEMCHECKER(
 483                        memchecker_call(&opal_memchecker_base_mem_noaccess,
 484                                        match->req_recv.req_base.req_addr,
 485                                        match->req_recv.req_base.req_count,
 486                                        match->req_recv.req_base.req_datatype);
 487                        );
 488         }
 489 
 490         
 491         
 492         recv_request_pml_complete(match);
 493     }
 494 
 495     
 496 
 497 
 498 
 499 
 500 
 501 
 502 
 503     if(NULL != proc->frags_cant_match) {
 504         mca_pml_ob1_recv_frag_t* frag;
 505 
 506         OB1_MATCHING_LOCK(&comm->matching_lock);
 507         if((frag = check_cantmatch_for_match(proc))) {
 508             
 509             mca_pml_ob1_recv_frag_match_proc(frag->btl, comm_ptr, proc,
 510                                              &frag->hdr.hdr_match,
 511                                              frag->segments, frag->num_segments,
 512                                              frag->hdr.hdr_match.hdr_common.hdr_type, frag);
 513         } else {
 514             OB1_MATCHING_UNLOCK(&comm->matching_lock);
 515         }
 516     }
 517 
 518     return;
 519 }
 520 
 521 
 522 void mca_pml_ob1_recv_frag_callback_rndv(mca_btl_base_module_t* btl,
 523                                          mca_btl_base_tag_t tag,
 524                                          mca_btl_base_descriptor_t* des,
 525                                          void* cbdata )
 526 {
 527     mca_btl_base_segment_t* segments = des->des_segments;
 528     mca_pml_ob1_hdr_t* hdr = (mca_pml_ob1_hdr_t*)segments->seg_addr.pval;
 529 
 530     if( OPAL_UNLIKELY(segments->seg_len < sizeof(mca_pml_ob1_common_hdr_t)) ) {
 531         return;
 532     }
 533     ob1_hdr_ntoh(hdr, MCA_PML_OB1_HDR_TYPE_RNDV);
 534     mca_pml_ob1_recv_frag_match(btl, &hdr->hdr_match, segments,
 535                                 des->des_segment_count, MCA_PML_OB1_HDR_TYPE_RNDV);
 536     return;
 537 }
 538 
 539 void mca_pml_ob1_recv_frag_callback_rget(mca_btl_base_module_t* btl,
 540                                          mca_btl_base_tag_t tag,
 541                                          mca_btl_base_descriptor_t* des,
 542                                          void* cbdata )
 543 {
 544     mca_btl_base_segment_t* segments = des->des_segments;
 545     mca_pml_ob1_hdr_t* hdr = (mca_pml_ob1_hdr_t*)segments->seg_addr.pval;
 546 
 547     if( OPAL_UNLIKELY(segments->seg_len < sizeof(mca_pml_ob1_common_hdr_t)) ) {
 548         return;
 549     }
 550     ob1_hdr_ntoh(hdr, MCA_PML_OB1_HDR_TYPE_RGET);
 551     mca_pml_ob1_recv_frag_match(btl, &hdr->hdr_match, segments,
 552                                 des->des_segment_count, MCA_PML_OB1_HDR_TYPE_RGET);
 553     return;
 554 }
 555 
 556 
 557 
 558 void mca_pml_ob1_recv_frag_callback_ack(mca_btl_base_module_t* btl,
 559                                         mca_btl_base_tag_t tag,
 560                                         mca_btl_base_descriptor_t* des,
 561                                         void* cbdata )
 562 {
 563     mca_btl_base_segment_t* segments = des->des_segments;
 564     mca_pml_ob1_hdr_t* hdr = (mca_pml_ob1_hdr_t*)segments->seg_addr.pval;
 565     mca_pml_ob1_send_request_t* sendreq;
 566     size_t size;
 567 
 568     if( OPAL_UNLIKELY(segments->seg_len < sizeof(mca_pml_ob1_common_hdr_t)) ) {
 569          return;
 570     }
 571 
 572     ob1_hdr_ntoh(hdr, MCA_PML_OB1_HDR_TYPE_ACK);
 573     sendreq = (mca_pml_ob1_send_request_t*)hdr->hdr_ack.hdr_src_req.pval;
 574     sendreq->req_recv = hdr->hdr_ack.hdr_dst_req;
 575 
 576     
 577 
 578     if(hdr->hdr_common.hdr_flags & MCA_PML_OB1_HDR_FLAGS_NORDMA) {
 579         if (NULL != sendreq->rdma_frag) {
 580             MCA_PML_OB1_RDMA_FRAG_RETURN(sendreq->rdma_frag);
 581             sendreq->rdma_frag = NULL;
 582         }
 583 
 584         sendreq->req_throttle_sends = true;
 585     }
 586 
 587     if (hdr->hdr_ack.hdr_send_size) {
 588         size = hdr->hdr_ack.hdr_send_size;
 589     } else {
 590         size = sendreq->req_send.req_bytes_packed - hdr->hdr_ack.hdr_send_offset;
 591     }
 592 
 593     mca_pml_ob1_send_request_copy_in_out(sendreq, hdr->hdr_ack.hdr_send_offset, size);
 594 
 595     if (sendreq->req_state != 0) {
 596         
 597 
 598 
 599 
 600 
 601 
 602 
 603 
 604 
 605         OPAL_THREAD_ADD_FETCH32(&sendreq->req_state, -1);
 606     }
 607 
 608 #if OPAL_CUDA_SUPPORT 
 609     if ((sendreq->req_send.req_base.req_convertor.flags & CONVERTOR_CUDA) &&
 610         (btl->btl_flags & MCA_BTL_FLAGS_CUDA_COPY_ASYNC_SEND)) {
 611         
 612 
 613 
 614         void *strm = mca_common_cuda_get_dtoh_stream();
 615         opal_cuda_set_copy_function_async(&sendreq->req_send.req_base.req_convertor, strm);
 616     }
 617 #endif 
 618 
 619     if(send_request_pml_complete_check(sendreq) == false)
 620         mca_pml_ob1_send_request_schedule(sendreq);
 621 
 622     return;
 623 }
 624 
 625 void mca_pml_ob1_recv_frag_callback_frag(mca_btl_base_module_t* btl,
 626                                          mca_btl_base_tag_t tag,
 627                                          mca_btl_base_descriptor_t* des,
 628                                          void* cbdata ) {
 629     mca_btl_base_segment_t* segments = des->des_segments;
 630     mca_pml_ob1_hdr_t* hdr = (mca_pml_ob1_hdr_t*)segments->seg_addr.pval;
 631     mca_pml_ob1_recv_request_t* recvreq;
 632 
 633     if( OPAL_UNLIKELY(segments->seg_len < sizeof(mca_pml_ob1_common_hdr_t)) ) {
 634         return;
 635     }
 636 
 637     ob1_hdr_ntoh(hdr, MCA_PML_OB1_HDR_TYPE_FRAG);
 638     recvreq = (mca_pml_ob1_recv_request_t*)hdr->hdr_frag.hdr_dst_req.pval;
 639 #if OPAL_CUDA_SUPPORT 
 640     
 641 
 642 
 643     if (recvreq->req_recv.req_base.req_convertor.flags & CONVERTOR_CUDA_ASYNC) {
 644         assert(btl->btl_flags & MCA_BTL_FLAGS_CUDA_COPY_ASYNC_RECV);
 645 
 646         
 647         mca_pml_ob1_recv_request_frag_copy_start(recvreq,btl,segments,des->des_segment_count,des);
 648 
 649         
 650         des->des_flags |= MCA_BTL_DES_FLAGS_CUDA_COPY_ASYNC;
 651 
 652         return;
 653     }
 654 #endif 
 655 
 656     mca_pml_ob1_recv_request_progress_frag(recvreq,btl,segments,des->des_segment_count);
 657 
 658     return;
 659 }
 660 
 661 
 662 void mca_pml_ob1_recv_frag_callback_put(mca_btl_base_module_t* btl,
 663                                         mca_btl_base_tag_t tag,
 664                                         mca_btl_base_descriptor_t* des,
 665                                         void* cbdata ) {
 666     mca_btl_base_segment_t* segments = des->des_segments;
 667     mca_pml_ob1_hdr_t* hdr = (mca_pml_ob1_hdr_t*)segments->seg_addr.pval;
 668     mca_pml_ob1_send_request_t* sendreq;
 669 
 670     if( OPAL_UNLIKELY(segments->seg_len < sizeof(mca_pml_ob1_common_hdr_t)) ) {
 671         return;
 672     }
 673 
 674     ob1_hdr_ntoh(hdr, MCA_PML_OB1_HDR_TYPE_PUT);
 675     sendreq = (mca_pml_ob1_send_request_t*)hdr->hdr_rdma.hdr_req.pval;
 676     mca_pml_ob1_send_request_put(sendreq,btl,&hdr->hdr_rdma);
 677 
 678     return;
 679 }
 680 
 681 
 682 void mca_pml_ob1_recv_frag_callback_fin(mca_btl_base_module_t* btl,
 683                                         mca_btl_base_tag_t tag,
 684                                         mca_btl_base_descriptor_t* des,
 685                                         void* cbdata ) {
 686     mca_btl_base_segment_t* segments = des->des_segments;
 687     mca_pml_ob1_fin_hdr_t* hdr = (mca_pml_ob1_fin_hdr_t *) segments->seg_addr.pval;
 688     mca_pml_ob1_rdma_frag_t *frag;
 689 
 690     if( OPAL_UNLIKELY(segments->seg_len < sizeof(mca_pml_ob1_fin_hdr_t)) ) {
 691         return;
 692     }
 693 
 694     ob1_hdr_ntoh((union mca_pml_ob1_hdr_t *)hdr, MCA_PML_OB1_HDR_TYPE_FIN);
 695     frag = (mca_pml_ob1_rdma_frag_t *) hdr->hdr_frag.pval;
 696     frag->cbfunc (frag, hdr->hdr_size);
 697 }
 698 
 699 
 700 
 701 #define PML_MAX_SEQ ~((mca_pml_sequence_t)0);
 702 
 703 static inline mca_pml_ob1_recv_request_t* get_posted_recv(opal_list_t *queue)
 704 {
 705     if(opal_list_get_size(queue) == 0)
 706         return NULL;
 707 
 708     return (mca_pml_ob1_recv_request_t*)opal_list_get_first(queue);
 709 }
 710 
 711 static inline mca_pml_ob1_recv_request_t* get_next_posted_recv(
 712         opal_list_t *queue,
 713         mca_pml_ob1_recv_request_t* req)
 714 {
 715     opal_list_item_t *i = opal_list_get_next((opal_list_item_t*)req);
 716 
 717     if(opal_list_get_end(queue) == i)
 718         return NULL;
 719 
 720     return (mca_pml_ob1_recv_request_t*)i;
 721 }
 722 
 723 static mca_pml_ob1_recv_request_t *match_incomming(
 724         mca_pml_ob1_match_hdr_t *hdr, mca_pml_ob1_comm_t *comm,
 725         mca_pml_ob1_comm_proc_t *proc)
 726 {
 727 #if !MCA_PML_OB1_CUSTOM_MATCH
 728     mca_pml_ob1_recv_request_t *specific_recv, *wild_recv;
 729     mca_pml_sequence_t wild_recv_seq, specific_recv_seq;
 730     int tag = hdr->hdr_tag;
 731 
 732     specific_recv = get_posted_recv(&proc->specific_receives);
 733     wild_recv = get_posted_recv(&comm->wild_receives);
 734 
 735     wild_recv_seq = wild_recv ?
 736         wild_recv->req_recv.req_base.req_sequence : PML_MAX_SEQ;
 737     specific_recv_seq = specific_recv ?
 738         specific_recv->req_recv.req_base.req_sequence : PML_MAX_SEQ;
 739 
 740     
 741     while(wild_recv_seq != specific_recv_seq) {
 742         mca_pml_ob1_recv_request_t **match;
 743         opal_list_t *queue;
 744         int req_tag;
 745         mca_pml_sequence_t *seq;
 746 
 747         if (OPAL_UNLIKELY(wild_recv_seq < specific_recv_seq)) {
 748             match = &wild_recv;
 749             queue = &comm->wild_receives;
 750             seq = &wild_recv_seq;
 751         } else {
 752             match = &specific_recv;
 753             queue = &proc->specific_receives;
 754             seq = &specific_recv_seq;
 755         }
 756 
 757         req_tag = (*match)->req_recv.req_base.req_tag;
 758         if(req_tag == tag || (req_tag == OMPI_ANY_TAG && tag >= 0)) {
 759             opal_list_remove_item(queue, (opal_list_item_t*)(*match));
 760             PERUSE_TRACE_COMM_EVENT(PERUSE_COMM_REQ_REMOVE_FROM_POSTED_Q,
 761                     &((*match)->req_recv.req_base), PERUSE_RECV);
 762             return *match;
 763         }
 764 
 765         *match = get_next_posted_recv(queue, *match);
 766         *seq = (*match) ? (*match)->req_recv.req_base.req_sequence : PML_MAX_SEQ;
 767     }
 768 
 769     return NULL;
 770 #else
 771     return custom_match_prq_find_dequeue_verify(comm->prq, hdr->hdr_tag, hdr->hdr_src);
 772 #endif
 773 }
 774 
 775 #if !MCA_PML_OB1_CUSTOM_MATCH
 776 static mca_pml_ob1_recv_request_t *match_incomming_no_any_source (
 777         mca_pml_ob1_match_hdr_t *hdr, mca_pml_ob1_comm_t *comm,
 778         mca_pml_ob1_comm_proc_t *proc)
 779 {
 780     mca_pml_ob1_recv_request_t *recv_req;
 781     int tag = hdr->hdr_tag;
 782 
 783     OPAL_LIST_FOREACH(recv_req, &proc->specific_receives, mca_pml_ob1_recv_request_t) {
 784         int req_tag = recv_req->req_recv.req_base.req_tag;
 785 
 786         if (req_tag == tag || (req_tag == OMPI_ANY_TAG && tag >= 0)) {
 787             opal_list_remove_item (&proc->specific_receives, (opal_list_item_t *) recv_req);
 788             PERUSE_TRACE_COMM_EVENT(PERUSE_COMM_REQ_REMOVE_FROM_POSTED_Q,
 789                     &(recv_req->req_recv.req_base), PERUSE_RECV);
 790             return recv_req;
 791         }
 792     }
 793 
 794     return NULL;
 795 }
 796 #endif
 797 
 798 static mca_pml_ob1_recv_request_t*
 799 match_one(mca_btl_base_module_t *btl,
 800           mca_pml_ob1_match_hdr_t *hdr, mca_btl_base_segment_t* segments,
 801           size_t num_segments, ompi_communicator_t *comm_ptr,
 802           mca_pml_ob1_comm_proc_t *proc,
 803           mca_pml_ob1_recv_frag_t* frag)
 804 {
 805 #if SPC_ENABLE == 1
 806     opal_timer_t timer = 0;
 807 #endif
 808     SPC_TIMER_START(OMPI_SPC_MATCH_TIME, &timer);
 809 
 810     mca_pml_ob1_recv_request_t *match;
 811     mca_pml_ob1_comm_t *comm = (mca_pml_ob1_comm_t *)comm_ptr->c_pml_comm;
 812 
 813     do {
 814 #if MCA_PML_OB1_CUSTOM_MATCH
 815         match = match_incomming(hdr, comm, proc);
 816 #else
 817         if (!OMPI_COMM_CHECK_ASSERT_NO_ANY_SOURCE (comm_ptr)) {
 818             match = match_incomming(hdr, comm, proc);
 819         } else {
 820             match = match_incomming_no_any_source (hdr, comm, proc);
 821         }
 822 #endif
 823 
 824         
 825         if(OPAL_LIKELY(NULL != match)) {
 826             match->req_recv.req_base.req_proc = proc->ompi_proc;
 827 
 828             if(OPAL_UNLIKELY(MCA_PML_REQUEST_PROBE == match->req_recv.req_base.req_type)) {
 829                 
 830                 mca_pml_ob1_recv_request_matched_probe(match, btl, segments,
 831                                                        num_segments);
 832                 
 833                 continue;
 834             } else if (MCA_PML_REQUEST_MPROBE == match->req_recv.req_base.req_type) {
 835                 
 836 
 837 
 838                 mca_pml_ob1_recv_frag_t *tmp;
 839                 if(NULL == frag) {
 840                     MCA_PML_OB1_RECV_FRAG_ALLOC(tmp);
 841                     MCA_PML_OB1_RECV_FRAG_INIT(tmp, hdr, segments, num_segments, btl);
 842                 } else {
 843                     tmp = frag;
 844                 }
 845 
 846                 match->req_recv.req_base.req_addr = tmp;
 847                 mca_pml_ob1_recv_request_matched_probe(match, btl, segments,
 848                                                        num_segments);
 849                 
 850 
 851                 SPC_TIMER_STOP(OMPI_SPC_MATCH_TIME, &timer);
 852                 return NULL;
 853             }
 854 
 855             PERUSE_TRACE_COMM_EVENT(PERUSE_COMM_MSG_MATCH_POSTED_REQ,
 856                                     &(match->req_recv.req_base), PERUSE_RECV);
 857             SPC_TIMER_STOP(OMPI_SPC_MATCH_TIME, &timer);
 858             return match;
 859         }
 860 
 861         
 862 #if MCA_PML_OB1_CUSTOM_MATCH
 863         append_frag_to_umq(comm->umq, btl, hdr, segments,
 864                             num_segments, frag);
 865 #else
 866         append_frag_to_list(&proc->unexpected_frags, btl, hdr, segments,
 867                             num_segments, frag);
 868 #endif
 869         SPC_RECORD(OMPI_SPC_UNEXPECTED, 1);
 870         SPC_RECORD(OMPI_SPC_UNEXPECTED_IN_QUEUE, 1);
 871         SPC_UPDATE_WATERMARK(OMPI_SPC_MAX_UNEXPECTED_IN_QUEUE, OMPI_SPC_UNEXPECTED_IN_QUEUE);
 872         PERUSE_TRACE_MSG_EVENT(PERUSE_COMM_MSG_INSERT_IN_UNEX_Q, comm_ptr,
 873                                hdr->hdr_src, hdr->hdr_tag, PERUSE_RECV);
 874         SPC_TIMER_STOP(OMPI_SPC_MATCH_TIME, &timer);
 875         return NULL;
 876     } while(true);
 877 }
 878 
 879 
 880 
 881 
 882 
 883 
 884 
 885 
 886 
 887 
 888 
 889 
 890 
 891 
 892 
 893 
 894 
 895 
 896 
 897 
 898 
 899 
 900 
 901 
 902 
 903 
 904 
 905 
 906 
 907 
 908 
 909 static int mca_pml_ob1_recv_frag_match( mca_btl_base_module_t *btl,
 910                                         mca_pml_ob1_match_hdr_t *hdr,
 911                                         mca_btl_base_segment_t* segments,
 912                                         size_t num_segments,
 913                                         int type)
 914 {
 915     
 916     uint16_t frag_msg_seq;
 917     uint16_t next_msg_seq_expected;
 918     ompi_communicator_t *comm_ptr;
 919     mca_pml_ob1_comm_t *comm;
 920     mca_pml_ob1_comm_proc_t *proc;
 921 
 922     
 923     comm_ptr = ompi_comm_lookup(hdr->hdr_ctx);
 924     if(OPAL_UNLIKELY(NULL == comm_ptr)) {
 925         
 926 
 927 
 928 
 929 
 930 
 931 
 932         append_frag_to_list( &mca_pml_ob1.non_existing_communicator_pending,
 933                              btl, hdr, segments, num_segments, NULL );
 934         return OMPI_SUCCESS;
 935     }
 936     comm = (mca_pml_ob1_comm_t *)comm_ptr->c_pml_comm;
 937 
 938     
 939     proc = mca_pml_ob1_peer_lookup (comm_ptr, hdr->hdr_src);
 940 
 941     
 942 
 943 
 944 
 945 
 946 
 947     PERUSE_TRACE_MSG_EVENT(PERUSE_COMM_MSG_ARRIVED, comm_ptr,
 948                            hdr->hdr_src, hdr->hdr_tag, PERUSE_RECV);
 949 
 950     
 951 
 952 
 953 
 954 
 955 
 956 
 957     OB1_MATCHING_LOCK(&comm->matching_lock);
 958 
 959     frag_msg_seq = hdr->hdr_seq;
 960     next_msg_seq_expected = (uint16_t)proc->expected_sequence;
 961 
 962     if (!OMPI_COMM_CHECK_ASSERT_ALLOW_OVERTAKE(comm_ptr)) {
 963         
 964         if(OPAL_UNLIKELY(frag_msg_seq != next_msg_seq_expected)) {
 965             mca_pml_ob1_recv_frag_t* frag;
 966             MCA_PML_OB1_RECV_FRAG_ALLOC(frag);
 967             MCA_PML_OB1_RECV_FRAG_INIT(frag, hdr, segments, num_segments, btl);
 968             append_frag_to_ordered_list(&proc->frags_cant_match, frag, next_msg_seq_expected);
 969 
 970             SPC_RECORD(OMPI_SPC_OUT_OF_SEQUENCE, 1);
 971             SPC_RECORD(OMPI_SPC_OOS_IN_QUEUE, 1);
 972             SPC_UPDATE_WATERMARK(OMPI_SPC_MAX_OOS_IN_QUEUE, OMPI_SPC_OOS_IN_QUEUE);
 973 
 974             OB1_MATCHING_UNLOCK(&comm->matching_lock);
 975             return OMPI_SUCCESS;
 976         }
 977     }
 978 
 979     
 980     return mca_pml_ob1_recv_frag_match_proc(btl, comm_ptr, proc, hdr,
 981                                             segments, num_segments,
 982                                             type, NULL);
 983 }
 984 
 985 
 986 
 987 
 988 
 989 
 990 
 991 
 992 static int
 993 mca_pml_ob1_recv_frag_match_proc( mca_btl_base_module_t *btl,
 994                                   ompi_communicator_t* comm_ptr,
 995                                   mca_pml_ob1_comm_proc_t *proc,
 996                                   mca_pml_ob1_match_hdr_t *hdr,
 997                                   mca_btl_base_segment_t* segments,
 998                                   size_t num_segments,
 999                                   int type,
1000                                   mca_pml_ob1_recv_frag_t* frag )
1001 {
1002     
1003     mca_pml_ob1_comm_t* comm = (mca_pml_ob1_comm_t *)comm_ptr->c_pml_comm;
1004     mca_pml_ob1_recv_request_t *match = NULL;
1005 
1006     
1007 
1008 
1009 
1010  match_this_frag:
1011     
1012     
1013 
1014 
1015 
1016     proc->expected_sequence++;
1017 
1018     
1019 
1020 
1021 
1022     PERUSE_TRACE_MSG_EVENT(PERUSE_COMM_SEARCH_POSTED_Q_BEGIN, comm_ptr,
1023                            hdr->hdr_src, hdr->hdr_tag, PERUSE_RECV);
1024 
1025     match = match_one(btl, hdr, segments, num_segments, comm_ptr, proc, frag);
1026 
1027     
1028 
1029 
1030 
1031     PERUSE_TRACE_MSG_EVENT(PERUSE_COMM_SEARCH_POSTED_Q_END, comm_ptr,
1032                            hdr->hdr_src, hdr->hdr_tag, PERUSE_RECV);
1033 
1034     
1035     OB1_MATCHING_UNLOCK(&comm->matching_lock);
1036 
1037     if(OPAL_LIKELY(match)) {
1038         switch(type) {
1039         case MCA_PML_OB1_HDR_TYPE_MATCH:
1040             mca_pml_ob1_recv_request_progress_match(match, btl, segments, num_segments);
1041             break;
1042         case MCA_PML_OB1_HDR_TYPE_RNDV:
1043             mca_pml_ob1_recv_request_progress_rndv(match, btl, segments, num_segments);
1044             break;
1045         case MCA_PML_OB1_HDR_TYPE_RGET:
1046             mca_pml_ob1_recv_request_progress_rget(match, btl, segments, num_segments);
1047             break;
1048         }
1049 
1050         if(OPAL_UNLIKELY(frag))
1051             MCA_PML_OB1_RECV_FRAG_RETURN(frag);
1052     }
1053 
1054     
1055 
1056 
1057 
1058 
1059     if(OPAL_UNLIKELY(NULL != proc->frags_cant_match)) {
1060         OB1_MATCHING_LOCK(&comm->matching_lock);
1061         if((frag = check_cantmatch_for_match(proc))) {
1062             hdr = &frag->hdr.hdr_match;
1063             segments = frag->segments;
1064             num_segments = frag->num_segments;
1065             btl = frag->btl;
1066             type = hdr->hdr_common.hdr_type;
1067             goto match_this_frag;
1068         }
1069         OB1_MATCHING_UNLOCK(&comm->matching_lock);
1070     }
1071 
1072     return OMPI_SUCCESS;
1073 }
1074