root/ompi/mca/mtl/portals4/mtl_portals4_send.c

DEFINITIONS

1. ompi_mtl_portals4_callback
2. ompi_mtl_portals4_send_callback
3. ompi_mtl_portals4_isend_callback
4. ompi_mtl_portals4_short_isend
5. ompi_mtl_portals4_long_isend
6. ompi_mtl_portals4_pending_list_progress
7. ompi_mtl_portals4_send_start
8. ompi_mtl_portals4_send
9. ompi_mtl_portals4_isend

   1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
   2 /*
   3  * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
   4  *                         University Research and Technology
   5  *                         Corporation.  All rights reserved.
   6  * Copyright (c) 2004-2005 The University of Tennessee and The University
   7  *                         of Tennessee Research Foundation.  All rights
   8  *                         reserved.
   9  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
  10  *                         University of Stuttgart.  All rights reserved.
  11  * Copyright (c) 2004-2005 The Regents of the University of California.
  12  *                         All rights reserved.
  13  * Copyright (c) 2010      Sandia National Laboratories.  All rights reserved.
  14  * Copyright (c) 2015      Los Alamos National Security, LLC.  All rights
  15  *                         reserved.
  16  * $COPYRIGHT$
  17  *
  18  * Additional copyrights may follow
  19  *
  20  * $HEADER$
  21  */
  22 
  23 #include "ompi_config.h"
  24 
  25 #include "ompi/communicator/communicator.h"
  26 #include "opal/datatype/opal_convertor.h"
  27 #include "ompi/mca/mtl/base/base.h"
  28 #include "ompi/mca/mtl/base/mtl_base_datatype.h"
  29 
  30 #include "mtl_portals4.h"
  31 #include "mtl_portals4_endpoint.h"
  32 #include "mtl_portals4_request.h"
  33 #if OMPI_MTL_PORTALS4_FLOW_CONTROL
  34 #include "mtl_portals4_flowctl.h"
  35 #endif
  36 
  37 
  38 static inline int
  39 ompi_mtl_portals4_callback(ptl_event_t *ev,
  40                            ompi_mtl_portals4_base_request_t* ptl_base_request,
  41                            bool *complete)
  42 {
  43     int retval = OMPI_SUCCESS, ret, val, add = 1;
  44     ompi_mtl_portals4_isend_request_t* ptl_request =
  45         (ompi_mtl_portals4_isend_request_t*) ptl_base_request;
  46 
  47     if (PTL_EVENT_GET == ev->type) {
  48         ret = OPAL_THREAD_ADD_FETCH32(&(ptl_request->pending_get), -1);
  49         if (ret > 0) {
  50             /* wait for other gets */
  51             OPAL_OUTPUT_VERBOSE((90, ompi_mtl_base_framework.framework_output, "PTL_EVENT_GET received now pending_get=%d",ret));
  52             return retval;
  53         }
  54         assert(ptl_request->pending_get == 0);
  55 
  56         /* last get received */
  57         OPAL_OUTPUT_VERBOSE((90, ompi_mtl_base_framework.framework_output, "PTL_EVENT_GET: PtlMEUnlink is called ptl_request->me_h=%d (pending get=%d)", ptl_request->me_h, ret));
  58 
  59         if (!PtlHandleIsEqual(ptl_request->me_h, PTL_INVALID_HANDLE)) {
  60             ret = PtlMEUnlink(ptl_request->me_h);
  61             if (PTL_OK != ret) {
  62                 opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
  63                                 "%s:%d: send callback PtlMEUnlink returned %d",
  64                                 __FILE__, __LINE__, ret);
  65             }
  66             ptl_request->me_h = PTL_INVALID_HANDLE;
  67         }
  68     }
  69 
  70 #if OMPI_MTL_PORTALS4_FLOW_CONTROL
  71     if (OPAL_UNLIKELY(ev->ni_fail_type == PTL_NI_PT_DISABLED)) {
  72         ompi_mtl_portals4_pending_request_t *pending =
  73             ptl_request->pending;
  74 
  75         OPAL_OUTPUT_VERBOSE((10, ompi_mtl_base_framework.framework_output,
  76                              "send %lu hit flow control (%d)",
  77                              ptl_request->opcount, ev->type));
  78 
  79         /* BWB: FIX ME: this is a hack.. */
  80         if (pending->fc_notified) {
  81             return OMPI_SUCCESS;
  82         }
  83         pending->fc_notified = 1;
  84 
  85         if (!PtlHandleIsEqual(ptl_request->me_h, PTL_INVALID_HANDLE)) {
  86             ret = PtlMEUnlink(ptl_request->me_h);
  87             if (PTL_OK != ret) {
  88                 opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
  89                                     "%s:%d: send callback PtlMEUnlink returned %d",
  90                                     __FILE__, __LINE__, ret);
  91             }
  92             ptl_request->me_h = PTL_INVALID_HANDLE;
  93         }
  94 
  95         opal_list_append(&ompi_mtl_portals4.flowctl.pending_sends,
  96                          &pending->super.super);
  97         OPAL_THREAD_ADD_FETCH32(&ompi_mtl_portals4.flowctl.send_slots, 1);
  98         ompi_mtl_portals4_flowctl_trigger();
  99 
 100         return OMPI_SUCCESS;
 101     }
 102 #endif
 103 
 104     if (OPAL_UNLIKELY(ev->ni_fail_type != PTL_NI_OK)) {
 105         opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
 106                             "%s:%d: send callback ni_fail_type: %d",
 107                             __FILE__, __LINE__, ev->ni_fail_type);
 108         *complete = true;
 109         return OMPI_ERROR;
 110     }
 111 
 112     OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_framework.framework_output,
 113                          "send %lu got event of type %d",
 114                          ptl_request->opcount, ev->type));
 115 
 116     /* First put achieved successfully (In the Priority List), so it may be necessary to decrement the number of pending get
 117      * If the protocol is eager, just decrement pending_get 
 118      * Else (the protocol is rndv), decrement pending_get only if length % max_msg_size <= eager_limit
 119      * (This is the case where the eager part allows to save one get)
 120      */
 121     if ((PTL_EVENT_ACK == ev->type) &&
 122         (PTL_PRIORITY_LIST == ev->ptl_list) &&
 123         (0 <  ptl_request->pending_get)) {
 124 
 125         if ((eager == ompi_mtl_portals4.protocol) ||
 126              (ptl_request->length % ompi_mtl_portals4.max_msg_size_mtl <= ompi_mtl_portals4.eager_limit)) {
 127            val = OPAL_THREAD_ADD_FETCH32(&(ptl_request->pending_get), -1);
 128         }
 129         if (0 == val) {
 130             add = 2; /* We haven't to wait for any get, so we have to add an extra count to cause the message to complete */
 131             if (!PtlHandleIsEqual(ptl_request->me_h, PTL_INVALID_HANDLE)) {
 132                 ret = PtlMEUnlink(ptl_request->me_h);
 133                 if (PTL_OK != ret) {
 134                     opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
 135                                 "%s:%d: send callback PtlMEUnlink returned %d",
 136                                 __FILE__, __LINE__, ret);
 137                 }
 138                 ptl_request->me_h = PTL_INVALID_HANDLE;
 139             }
 140         }
 141     }
 142 
 143     if ((PTL_EVENT_ACK == ev->type) &&
 144         (PTL_PRIORITY_LIST == ev->ptl_list) &&
 145         (ev->mlength == ptl_request->length) &&
 146         (!PtlHandleIsEqual(ptl_request->me_h, PTL_INVALID_HANDLE))) {
 147         /* long expected messages with the eager protocol
 148            (and also with the rndv protocol if the length
 149            is less or egal to eager_limit) won't see a
 150            get event to complete the message.  Give them an extra
 151            count to cause the message to complete with just the SEND
 152            and ACK events and remove the ME. (we wait for the counter
 153            to reach 3 events, but short messages start the counter at
 154            1, so they don't need to enter this path) */
 155         ret = PtlMEUnlink(ptl_request->me_h);
 156         if (PTL_OK != ret) {
 157             opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
 158                                 "%s:%d: send callback PtlMEUnlink returned %d",
 159                                 __FILE__, __LINE__, ret);
 160         }
 161         ptl_request->me_h = PTL_INVALID_HANDLE;
 162         add++;
 163     }
 164     val = OPAL_THREAD_ADD_FETCH32((int32_t*)&ptl_request->event_count, add);
 165     assert(val <= 3);
 166 
 167     if (val == 3) {
 168         if (NULL != ptl_request->buffer_ptr) {
 169             free(ptl_request->buffer_ptr);
 170         }
 171 
 172         OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_framework.framework_output, "send %lu completed",
 173                              ptl_request->opcount));
 174 
 175         *complete = true;
 176 #if OMPI_MTL_PORTALS4_FLOW_CONTROL
 177         OPAL_THREAD_ADD_FETCH32(&ompi_mtl_portals4.flowctl.send_slots, 1);
 178         opal_free_list_return (&ompi_mtl_portals4.flowctl.pending_fl,
 179                                &ptl_request->pending->super);
 180 
 181         if (OPAL_UNLIKELY(0 != opal_list_get_size(&ompi_mtl_portals4.flowctl.pending_sends))) {
 182             ompi_mtl_portals4_pending_list_progress();
 183         }
 184 #endif
 185     }
 186 
 187     return retval;
 188 }
 189 
 190 
 191 static int
 192 ompi_mtl_portals4_send_callback(ptl_event_t *ev,
 193                                 ompi_mtl_portals4_base_request_t* ptl_base_request)
 194 {
 195     bool complete = false;
 196     int ret;
 197     ompi_mtl_portals4_send_request_t* ptl_request =
 198         (ompi_mtl_portals4_send_request_t*) ptl_base_request;
 199 
 200     ret = ompi_mtl_portals4_callback(ev, ptl_base_request, &complete);
 201     if (complete) {
 202         ptl_request->retval = ret;
 203         opal_atomic_wmb();
 204         ptl_request->complete = true;
 205     }
 206 
 207     return OMPI_SUCCESS;
 208 }
 209 
 210 
 211 static int
 212 ompi_mtl_portals4_isend_callback(ptl_event_t *ev,
 213                                  ompi_mtl_portals4_base_request_t* ptl_base_request)
 214 {
 215     bool complete = false;
 216     int ret;
 217     ompi_mtl_portals4_isend_request_t* ptl_request =
 218         (ompi_mtl_portals4_isend_request_t*) ptl_base_request;
 219 
 220     ret = ompi_mtl_portals4_callback(ev, ptl_base_request, &complete);
 221     if (complete) {
 222         ptl_request->super.super.ompi_req->req_status.MPI_ERROR = ret;
 223         ptl_request->super.super.completion_callback(&ptl_request->super.super);
 224     }
 225 
 226     return OMPI_SUCCESS;
 227 }
 228 
 229 
 230 static inline int
 231 ompi_mtl_portals4_short_isend(mca_pml_base_send_mode_t mode,
 232                               void *start, int length, int contextid, int tag,
 233                               int localrank,
 234                               ptl_process_t ptl_proc,
 235                               ompi_mtl_portals4_isend_request_t *ptl_request)
 236 {
 237     int ret;
 238     ptl_match_bits_t match_bits;
 239     ptl_me_t me;
 240     ptl_hdr_data_t hdr_data;
 241 
 242     MTL_PORTALS4_SET_SEND_BITS(match_bits, contextid, localrank, tag,
 243                                MTL_PORTALS4_SHORT_MSG);
 244 
 245     MTL_PORTALS4_SET_HDR_DATA(hdr_data, ptl_request->opcount, length,
 246                               (MCA_PML_BASE_SEND_SYNCHRONOUS == mode) ? 1 : 0);
 247     ptl_request->me_h = PTL_INVALID_HANDLE;
 248 
 249     if (MCA_PML_BASE_SEND_SYNCHRONOUS == mode) {
 250         me.start = NULL;
 251         me.length = 0;
 252         me.ct_handle = PTL_CT_NONE;
 253         me.min_free = 0;
 254         me.uid = ompi_mtl_portals4.uid;
 255         me.options =
 256             PTL_ME_OP_PUT |
 257             PTL_ME_USE_ONCE |
 258             PTL_ME_EVENT_LINK_DISABLE |
 259             PTL_ME_EVENT_UNLINK_DISABLE;
 260         me.match_id = ptl_proc;
 261         me.match_bits = hdr_data;
 262         me.ignore_bits = 0;
 263 
 264         ret = PtlMEAppend(ompi_mtl_portals4.ni_h,
 265                           ompi_mtl_portals4.read_idx,
 266                           &me,
 267                           PTL_PRIORITY_LIST,
 268                           ptl_request,
 269                           &ptl_request->me_h);
 270         if (OPAL_UNLIKELY(PTL_OK != ret)) {
 271             opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
 272                                 "%s:%d: PtlMEAppend failed: %d",
 273                                 __FILE__, __LINE__, ret);
 274             ptl_request->me_h = PTL_INVALID_HANDLE;
 275             return ompi_mtl_portals4_get_error(ret);
 276         }
 277 
 278         OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_framework.framework_output,
 279                              "Send %lu short sync send with hdr_data 0x%lx (0x%lx)",
 280                              ptl_request->opcount, hdr_data, match_bits));
 281     } else {
 282         ptl_request->event_count = 1;
 283 
 284         OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_framework.framework_output,
 285                              "Send %lu short send with hdr_data 0x%lx (0x%lx)",
 286                              ptl_request->opcount, hdr_data, match_bits));
 287     }
 288 
 289     OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_framework.framework_output,
 290                          "Send %lu, start: %p",
 291                          ptl_request->opcount, start));
 292 
 293     ptl_request->pending_get = 0;
 294     ret = PtlPut(ompi_mtl_portals4.send_md_h,
 295                  (ptl_size_t) start,
 296                  length,
 297                  PTL_ACK_REQ,
 298                  ptl_proc,
 299                  ompi_mtl_portals4.recv_idx,
 300                  match_bits,
 301                  0,
 302                  ptl_request,
 303                  hdr_data);
 304     if (OPAL_UNLIKELY(PTL_OK != ret)) {
 305         opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
 306                             "%s:%d: PtlPut failed: %d",
 307                             __FILE__, __LINE__, ret);
 308         if (MCA_PML_BASE_SEND_SYNCHRONOUS == mode) {
 309             PtlMEUnlink(ptl_request->me_h);
 310             ptl_request->me_h = PTL_INVALID_HANDLE;
 311         }
 312         return ompi_mtl_portals4_get_error(ret);
 313     }
 314 
 315     return OMPI_SUCCESS;
 316 }
 317 
 318 static inline int
 319 ompi_mtl_portals4_long_isend(void *start, size_t length, int contextid, int tag,
 320                              int localrank,
 321                              ptl_process_t ptl_proc,
 322                              ompi_mtl_portals4_isend_request_t *ptl_request)
 323 {
 324     int ret;
 325     ptl_match_bits_t match_bits;
 326     ptl_me_t me;
 327     ptl_hdr_data_t hdr_data;
 328     ptl_size_t put_length;
 329 
 330     MTL_PORTALS4_SET_SEND_BITS(match_bits, contextid, localrank, tag,
 331                                MTL_PORTALS4_LONG_MSG);
 332 
 333     MTL_PORTALS4_SET_HDR_DATA(hdr_data, ptl_request->opcount, length, 0);
 334 
 335     me.start = start;
 336     me.length = length;
 337     me.ct_handle = PTL_CT_NONE;
 338     me.min_free = 0;
 339     me.uid = ompi_mtl_portals4.uid;
 340     me.options =
 341         PTL_ME_OP_GET |
 342         PTL_ME_EVENT_LINK_DISABLE |
 343         PTL_ME_EVENT_UNLINK_DISABLE;
 344     me.match_id = ptl_proc;
 345     me.match_bits = hdr_data;
 346     me.ignore_bits = 0;
 347 
 348     ret = PtlMEAppend(ompi_mtl_portals4.ni_h,
 349                       ompi_mtl_portals4.read_idx,
 350                       &me,
 351                       PTL_PRIORITY_LIST,
 352                       ptl_request,
 353                       &ptl_request->me_h);
 354     if (OPAL_UNLIKELY(PTL_OK != ret)) {
 355         opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
 356                             "%s:%d: PtlMEAppend failed: %d",
 357                             __FILE__, __LINE__, ret);
 358         return ompi_mtl_portals4_get_error(ret);
 359     }
 360 
 361     OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_framework.framework_output,
 362                          "Send %lu long send with hdr_data 0x%lx (0x%lx)",
 363                          ptl_request->opcount, hdr_data, match_bits));
 364 
 365     if (rndv == ompi_mtl_portals4.protocol) {
 366         ptl_size_t min = (OPAL_LIKELY (ompi_mtl_portals4.eager_limit < ompi_mtl_portals4.max_msg_size_mtl)) ?
 367             ompi_mtl_portals4.eager_limit :
 368             ompi_mtl_portals4.max_msg_size_mtl;
 369         if ((ptl_size_t) length > (ptl_size_t) min) {
 370             OPAL_OUTPUT_VERBOSE((90, ompi_mtl_base_framework.framework_output,
 371                                  "msg truncated by %ld", length - min));
 372             put_length = (ptl_size_t) min;
 373         }
 374         else
 375             put_length = (ptl_size_t) length;
 376     } else { // eager protocol
 377         if (length >  ompi_mtl_portals4.max_msg_size_mtl)
 378             put_length = (ptl_size_t) ompi_mtl_portals4.max_msg_size_mtl;
 379         else
 380             put_length = (ptl_size_t) length;
 381     }
 382 
 383     /* We have to wait for some GET events.
 384        If the first put falls in overflow list, the number of GET event is egal to:
 385            (length - 1) / ompi_mtl_portals4.max_msg_size_mtl + 1
 386        else we will re-calculate this number when we received the first ACK event (with remote overflow list)
 387      */
 388 
 389     ptl_request->pending_get = (length - 1) / ompi_mtl_portals4.max_msg_size_mtl + 1;
 390     OPAL_OUTPUT_VERBOSE((90, ompi_mtl_base_framework.framework_output, "pending_get=%d", ptl_request->pending_get));
 391 
 392     ret = PtlPut(ompi_mtl_portals4.send_md_h,
 393                  (ptl_size_t) start,
 394                  put_length,
 395                  PTL_ACK_REQ,
 396                  ptl_proc,
 397                  ompi_mtl_portals4.recv_idx,
 398                  match_bits,
 399                  0,
 400                  ptl_request,
 401                  hdr_data);
 402     if (OPAL_UNLIKELY(PTL_OK != ret)) {
 403         opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
 404                             "%s:%d: PtlPut failed: %d",
 405                             __FILE__, __LINE__, ret);
 406         PtlMEUnlink(ptl_request->me_h);
 407         ptl_request->me_h = PTL_INVALID_HANDLE;
 408         return ompi_mtl_portals4_get_error(ret);
 409     }
 410 
 411     return OMPI_SUCCESS;
 412 }
 413 
 414 
 415 #if OMPI_MTL_PORTALS4_FLOW_CONTROL
 416 void
 417 ompi_mtl_portals4_pending_list_progress()
 418 {
 419     int ret, val;
 420     opal_list_item_t *item;
 421     ompi_mtl_portals4_pending_request_t *pending;
 422 
 423     while ((!ompi_mtl_portals4.flowctl.flowctl_active) &&
 424            (0 != opal_list_get_size(&ompi_mtl_portals4.flowctl.pending_sends))) {
 425         val = OPAL_THREAD_ADD_FETCH32(&ompi_mtl_portals4.flowctl.send_slots, -1);
 426         if (val < 0) {
 427             OPAL_THREAD_ADD_FETCH32(&ompi_mtl_portals4.flowctl.send_slots, 1);
 428             return;
 429         }
 430 
 431         item = opal_list_remove_first(&ompi_mtl_portals4.flowctl.pending_sends);
 432         if (OPAL_UNLIKELY(NULL == item)) {
 433             OPAL_THREAD_ADD_FETCH32(&ompi_mtl_portals4.flowctl.send_slots, 1);
 434             return;
 435         }
 436 
 437         pending = (ompi_mtl_portals4_pending_request_t*) item;
 438         if (pending->length <= ompi_mtl_portals4.short_limit) {
 439             ret = ompi_mtl_portals4_short_isend(pending->mode,
 440                                                 pending->start,
 441                                                 pending->length,
 442                                                 pending->contextid,
 443                                                 pending->tag,
 444                                                 pending->my_rank,
 445                                                 pending->ptl_proc,
 446                                                 pending->ptl_request);
 447         } else {
 448             ret = ompi_mtl_portals4_long_isend(pending->start,
 449                                                pending->length,
 450                                                pending->contextid,
 451                                                pending->tag,
 452                                                pending->my_rank,
 453                                                pending->ptl_proc,
 454                                                pending->ptl_request);
 455         }
 456         if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
 457             opal_list_prepend(&ompi_mtl_portals4.flowctl.pending_sends,
 458                               &pending->super.super);
 459             OPAL_THREAD_ADD_FETCH32(&ompi_mtl_portals4.flowctl.send_slots, 1);
 460         }
 461     }
 462 }
 463 #endif
 464 
 465 
 466 static inline int
 467 ompi_mtl_portals4_send_start(struct mca_mtl_base_module_t* mtl,
 468                              struct ompi_communicator_t* comm,
 469                              int dest,
 470                              int tag,
 471                              struct opal_convertor_t *convertor,
 472                              mca_pml_base_send_mode_t mode,
 473                              ompi_mtl_portals4_isend_request_t* ptl_request)
 474 {
 475     int ret= OMPI_SUCCESS;
 476     void *start;
 477     size_t length;
 478     bool free_after;
 479     ptl_process_t ptl_proc;
 480 #if OMPI_MTL_PORTALS4_FLOW_CONTROL
 481     opal_free_list_item_t *item;
 482     ompi_mtl_portals4_pending_request_t *pending;
 483 #endif
 484 
 485     if ((ompi_mtl_portals4.use_logical) && (MPI_COMM_WORLD == comm)) {
 486         ptl_proc.rank = dest;
 487     } else {
 488         ompi_proc_t *ompi_proc = ompi_comm_peer_lookup(comm, dest);
 489         ptl_proc = *((ptl_process_t*) ompi_mtl_portals4_get_endpoint (mtl, ompi_proc));
 490     }
 491 
 492     ret = ompi_mtl_datatype_pack(convertor, &start, &length, &free_after);
 493     if (OMPI_SUCCESS != ret) return ret;
 494 
 495     ptl_request->opcount = OPAL_THREAD_ADD_FETCH64((int64_t*)&ompi_mtl_portals4.opcount, 1);
 496     ptl_request->buffer_ptr = (free_after) ? start : NULL;
 497     ptl_request->length = length;
 498     ptl_request->event_count = 0;
 499 
 500     OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_framework.framework_output,
 501                          "Send %lu to %x,%x of length %ld\n",
 502                          ptl_request->opcount,
 503                          ptl_proc.phys.nid,
 504                          ptl_proc.phys.pid,
 505                          (int64_t)length));
 506 
 507 #if OMPI_MTL_PORTALS4_FLOW_CONTROL
 508     item = opal_free_list_get (&ompi_mtl_portals4.flowctl.pending_fl);
 509     if (NULL == item) return OMPI_ERR_OUT_OF_RESOURCE;
 510 
 511     pending = (ompi_mtl_portals4_pending_request_t*) item;
 512     ptl_request->pending = pending;
 513     pending->mode = mode;
 514     pending->start = start;
 515     pending->length = length;
 516     pending->contextid = comm->c_contextid;
 517     pending->tag = tag;
 518     pending->my_rank = comm->c_my_rank;
 519     pending->fc_notified = 0;
 520     pending->ptl_proc = ptl_proc;
 521     pending->ptl_request = ptl_request;
 522 
 523     if (OPAL_UNLIKELY(OPAL_THREAD_ADD_FETCH32(&ompi_mtl_portals4.flowctl.send_slots, -1) < 0)) {
 524         OPAL_THREAD_ADD_FETCH32(&ompi_mtl_portals4.flowctl.send_slots, 1);
 525         opal_list_append(&ompi_mtl_portals4.flowctl.pending_sends,
 526                          &pending->super.super);
 527         return OMPI_SUCCESS;
 528     }
 529 
 530     if (OPAL_UNLIKELY(0 != opal_list_get_size(&ompi_mtl_portals4.flowctl.pending_sends))) {
 531         OPAL_THREAD_ADD_FETCH32(&ompi_mtl_portals4.flowctl.send_slots, 1);
 532         opal_list_append(&ompi_mtl_portals4.flowctl.pending_sends,
 533                          &pending->super.super);
 534         ompi_mtl_portals4_pending_list_progress();
 535         return OMPI_SUCCESS;
 536     }
 537 
 538     if (OPAL_UNLIKELY(ompi_mtl_portals4.flowctl.flowctl_active)) {
 539         OPAL_THREAD_ADD_FETCH32(&ompi_mtl_portals4.flowctl.send_slots, 1);
 540         opal_list_append(&ompi_mtl_portals4.flowctl.pending_sends,
 541                          &pending->super.super);
 542         return OMPI_SUCCESS;
 543     }
 544 #endif
 545     if (length <= ompi_mtl_portals4.short_limit) {
 546         ret = ompi_mtl_portals4_short_isend(mode,
 547                                             start,
 548                                             length,
 549                                             comm->c_contextid,
 550                                             tag,
 551                                             comm->c_my_rank,
 552                                             ptl_proc,
 553                                             ptl_request);
 554     } else {
 555         ret = ompi_mtl_portals4_long_isend(start,
 556                                            length,
 557                                            comm->c_contextid,
 558                                            tag,
 559                                            comm->c_my_rank,
 560                                            ptl_proc,
 561                                            ptl_request);
 562     }
 563 
 564     return ret;
 565 }
 566 
 567 
 568 int
 569 ompi_mtl_portals4_send(struct mca_mtl_base_module_t* mtl,
 570                        struct ompi_communicator_t* comm,
 571                        int dest,
 572                        int tag,
 573                        struct opal_convertor_t *convertor,
 574                        mca_pml_base_send_mode_t mode)
 575 {
 576     int ret = OMPI_SUCCESS;
 577     ompi_mtl_portals4_send_request_t ptl_request;
 578 
 579     ptl_request.complete = false;
 580     ptl_request.retval = OMPI_SUCCESS;
 581     ptl_request.super.super.type = portals4_req_send;
 582     ptl_request.super.super.event_callback = ompi_mtl_portals4_send_callback;
 583 
 584     ret = ompi_mtl_portals4_send_start(mtl, comm, dest, tag,
 585                                        convertor, mode, &ptl_request.super);
 586     if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
 587         if (NULL != ptl_request.super.buffer_ptr) {
 588             free(ptl_request.super.buffer_ptr);
 589         }
 590         return ret;
 591     }
 592 
 593     while (false == ptl_request.complete) {
 594         ompi_mtl_portals4_progress();
 595     }
 596     ret = ptl_request.retval;
 597 
 598     return ret;
 599 }
 600 
 601 
 602 int
 603 ompi_mtl_portals4_isend(struct mca_mtl_base_module_t* mtl,
 604                         struct ompi_communicator_t* comm,
 605                         int dest,
 606                         int tag,
 607                         struct opal_convertor_t *convertor,
 608                         mca_pml_base_send_mode_t mode,
 609                         bool blocking,
 610                         mca_mtl_request_t *mtl_request)
 611 {
 612     int ret = OMPI_SUCCESS;
 613     ompi_mtl_portals4_isend_request_t *ptl_request =
 614         (ompi_mtl_portals4_isend_request_t*) mtl_request;
 615 
 616     ptl_request->super.type = portals4_req_isend;
 617     ptl_request->super.event_callback = ompi_mtl_portals4_isend_callback;
 618 
 619     ret = ompi_mtl_portals4_send_start(mtl, comm, dest, tag,
 620                                        convertor, mode, ptl_request);
 621 
 622     if (OPAL_UNLIKELY(OMPI_SUCCESS != ret && NULL != ptl_request->buffer_ptr)) {
 623         free(ptl_request->buffer_ptr);
 624     }
 625 
 626     return ret;
 627 }