root/ompi/mca/pml/ob1/pml_ob1_rdma.c

DEFINITIONS

1. mca_pml_ob1_rdma_btls
2. mca_pml_ob1_rdma_pipeline_btls_count
3. mca_pml_ob1_rdma_pipeline_btls

   1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
   2 /*
   3  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
   4  *                         University Research and Technology
   5  *                         Corporation.  All rights reserved.
   6  * Copyright (c) 2004-2006 The University of Tennessee and The University
   7  *                         of Tennessee Research Foundation.  All rights
   8  *                         reserved.
   9  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
  10  *                         University of Stuttgart.  All rights reserved.
  11  * Copyright (c) 2004-2005 The Regents of the University of California.
  12  *                         All rights reserved.
  13  * Copyright (c) 2014-2017 Los Alamos National Security, LLC. All rights
  14  *                         reserved.
  15  * $COPYRIGHT$
  16  *
  17  * Additional copyrights may follow
  18  *
  19  * $HEADER$
  20  */
  21 
  22 
  23 /*%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%*/
  24 
  25 #include "ompi_config.h"
  26 #include "ompi/constants.h"
  27 #include "ompi/mca/pml/pml.h"
  28 #include "ompi/mca/bml/bml.h"
  29 #include "opal/mca/mpool/mpool.h"
  30 #include "opal/runtime/opal_params.h"
  31 #include "pml_ob1.h"
  32 #include "pml_ob1_rdma.h"
  33 
  34 /*
  35  * Check to see if memory is registered or can be registered. Build a
  36  * set of registrations on the request.
  37  */
  38 
  39 size_t mca_pml_ob1_rdma_btls(
  40     mca_bml_base_endpoint_t* bml_endpoint,
  41     unsigned char* base,
  42     size_t size,
  43     mca_pml_ob1_com_btl_t* rdma_btls)
  44 {
  45     int num_btls = mca_bml_base_btl_array_get_size(&bml_endpoint->btl_rdma);
  46     int num_eager_btls = mca_bml_base_btl_array_get_size (&bml_endpoint->btl_eager);
  47     double weight_total = 0;
  48     int num_btls_used = 0;
  49 
  50     /* shortcut when there are no rdma capable btls */
  51     if(num_btls == 0) {
  52         return 0;
  53     }
  54 
  55     /* check to see if memory is registered */
  56     for (int n = 0; n < num_btls && num_btls_used < mca_pml_ob1.max_rdma_per_request; n++) {
  57         mca_bml_base_btl_t* bml_btl =
  58             mca_bml_base_btl_array_get_index(&bml_endpoint->btl_rdma,
  59                     (bml_endpoint->btl_rdma_index + n) % num_btls);
  60         mca_btl_base_registration_handle_t *reg_handle = NULL;
  61         mca_btl_base_module_t *btl = bml_btl->btl;
  62         /* NTH: go ahead and use an rdma btl if is the only one */
  63         bool ignore = !mca_pml_ob1.use_all_rdma;
  64 
  65         /* do not use rdma btls that are not in the eager list. this is necessary to avoid using
  66          * btls that exist on the endpoint only to support RMA. */
  67         for (int i = 0 ; i < num_eager_btls && ignore ; ++i) {
  68             mca_bml_base_btl_t *eager_btl = mca_bml_base_btl_array_get_index (&bml_endpoint->btl_eager, i);
  69             if (eager_btl->btl_endpoint == bml_btl->btl_endpoint) {
  70                 ignore = false;
  71                 break;
  72             }
  73         }
  74 
  75         if (ignore) {
  76             continue;
  77         }
  78 
  79         if (btl->btl_register_mem) {
  80             /* do not use the RDMA protocol with this btl if 1) leave pinned is disabled,
  81              * 2) the btl supports put, and 3) the fragment is larger than the minimum
  82              * pipeline size specified by the BTL */
  83             if (!opal_leave_pinned && (btl->btl_flags & MCA_BTL_FLAGS_PUT) &&
  84                   size > btl->btl_min_rdma_pipeline_size) {
  85                 continue;
  86             }
  87 
  88             /* try to register the memory region with the btl */
  89             reg_handle = btl->btl_register_mem (btl, bml_btl->btl_endpoint, base,
  90                                                 size, MCA_BTL_REG_FLAG_REMOTE_READ);
  91             if (NULL == reg_handle) {
  92                 /* btl requires registration but the registration failed */
  93                 continue;
  94             }
  95         } /* else no registration is needed with this btl */
  96 
  97         rdma_btls[num_btls_used].bml_btl = bml_btl;
  98         rdma_btls[num_btls_used].btl_reg = reg_handle;
  99         weight_total += bml_btl->btl_weight;
 100         num_btls_used++;
 101     }
 102 
 103     /* if we don't use leave_pinned and all BTLs that already have this memory
 104      * registered amount to less then half of available bandwidth - fall back to
 105      * pipeline protocol */
 106     if (0 == num_btls_used || (!opal_leave_pinned && weight_total < 0.5))
 107         return 0;
 108 
 109     mca_pml_ob1_calc_weighted_length(rdma_btls, num_btls_used, size,
 110                                      weight_total);
 111 
 112     bml_endpoint->btl_rdma_index = (bml_endpoint->btl_rdma_index + 1) % num_btls;
 113     return num_btls_used;
 114 }
 115 
 116 size_t mca_pml_ob1_rdma_pipeline_btls_count (mca_bml_base_endpoint_t* bml_endpoint)
 117 {
 118     int num_btls = mca_bml_base_btl_array_get_size (&bml_endpoint->btl_rdma);
 119     int num_eager_btls = mca_bml_base_btl_array_get_size (&bml_endpoint->btl_eager);
 120     int rdma_count = 0;
 121 
 122     for(int i = 0; i < num_btls && i < mca_pml_ob1.max_rdma_per_request; ++i) {
 123         mca_bml_base_btl_t *bml_btl = mca_bml_base_btl_array_get_next(&bml_endpoint->btl_rdma);
 124         /* NTH: go ahead and use an rdma btl if is the only one */
 125         bool ignore = !mca_pml_ob1.use_all_rdma;
 126 
 127         for (int i = 0 ; i < num_eager_btls && ignore ; ++i) {
 128             mca_bml_base_btl_t *eager_btl = mca_bml_base_btl_array_get_index (&bml_endpoint->btl_eager, i);
 129             if (eager_btl->btl_endpoint == bml_btl->btl_endpoint) {
 130                 ignore = false;
 131                 break;
 132             }
 133         }
 134 
 135         if (!ignore) {
 136             ++rdma_count;
 137         }
 138     }
 139 
 140     return rdma_count;
 141 }
 142 
 143 size_t mca_pml_ob1_rdma_pipeline_btls( mca_bml_base_endpoint_t* bml_endpoint,
 144                                        size_t size,
 145                                        mca_pml_ob1_com_btl_t* rdma_btls )
 146 {
 147     int num_btls = mca_bml_base_btl_array_get_size (&bml_endpoint->btl_rdma);
 148     int num_eager_btls = mca_bml_base_btl_array_get_size (&bml_endpoint->btl_eager);
 149     double weight_total = 0;
 150     int rdma_count = 0;
 151 
 152     for(int i = 0; i < num_btls && i < mca_pml_ob1.max_rdma_per_request; i++) {
 153         mca_bml_base_btl_t *bml_btl = mca_bml_base_btl_array_get_next(&bml_endpoint->btl_rdma);
 154         /* NTH: go ahead and use an rdma btl if is the only one */
 155         bool ignore = !mca_pml_ob1.use_all_rdma;
 156 
 157         for (int i = 0 ; i < num_eager_btls && ignore ; ++i) {
 158             mca_bml_base_btl_t *eager_btl = mca_bml_base_btl_array_get_index (&bml_endpoint->btl_eager, i);
 159             if (eager_btl->btl_endpoint == bml_btl->btl_endpoint) {
 160                 ignore = false;
 161                 break;
 162             }
 163         }
 164 
 165         if (ignore) {
 166             continue;
 167         }
 168 
 169         rdma_btls[rdma_count].bml_btl = bml_btl;
 170         rdma_btls[rdma_count++].btl_reg = NULL;
 171 
 172         weight_total += bml_btl->btl_weight;
 173     }
 174 
 175     mca_pml_ob1_calc_weighted_length (rdma_btls, rdma_count, size, weight_total);
 176 
 177     return rdma_count;
 178 }