root/opal/mca/btl/ugni/btl_ugni_add_procs.c

/* [<][>][^][v][top][bottom][index][help] */

DEFINITIONS

This source file includes following definitions.
  1. mca_btl_ugni_add_procs
  2. mca_btl_ugni_del_procs
  3. mca_btl_ugni_get_ep
  4. ugni_reg_mem
  5. ugni_dereg_mem
  6. mca_btl_ugni_setup_mpools
  7. mca_btl_ugni_module_set_max_reg
  8. mca_btl_ugni_smsg_setup

   1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
   2 /*
   3  * Copyright (c) 2011-2017 Los Alamos National Security, LLC. All rights
   4  *                         reserved.
   5  * Copyright (c) 2011      UT-Battelle, LLC. All rights reserved.
   6  * Copyright (c) 2014-2017 Intel, Inc. All rights reserved.
   7  * $COPYRIGHT$
   8  *
   9  * Additional copyrights may follow
  10  *
  11  * $HEADER$
  12  */
  13 
  14 #include "opal_config.h"
  15 
  16 #include "btl_ugni.h"
  17 #include "btl_ugni_frag.h"
  18 #include "btl_ugni_smsg.h"
  19 
  20 #include "opal/include/opal/align.h"
  21 #include "opal/mca/pmix/pmix.h"
  22 
  23 #define INITIAL_GNI_EPS 1024
  24 
  25 static int
  26 mca_btl_ugni_setup_mpools (mca_btl_ugni_module_t *ugni_module);
  27 static void
  28 mca_btl_ugni_module_set_max_reg (mca_btl_ugni_module_t *ugni_module, int nlocal_procs);
  29 static int mca_btl_ugni_smsg_setup (int nprocs);
  30 
  31 int mca_btl_ugni_add_procs (struct mca_btl_base_module_t* btl, size_t nprocs,
  32                             struct opal_proc_t **procs,
  33                             struct mca_btl_base_endpoint_t **peers,
  34                             opal_bitmap_t *reachable) {
  35     mca_btl_ugni_module_t *ugni_module = (mca_btl_ugni_module_t *) btl;
  36     int rc;
  37     void *mmap_start_addr;
  38     struct timeval tv = {.tv_sec = 0, .tv_usec = MCA_BTL_UGNI_CONNECT_USEC};
  39 
  40     if (false == ugni_module->initialized) {
  41 
  42         /* TODO: need to think of something more elegant than this max array */
  43 
  44         rc = opal_pointer_array_init (&ugni_module->endpoints, INITIAL_GNI_EPS, 1 << 24, 512);
  45         if (OPAL_SUCCESS != rc) {
  46             BTL_ERROR(("error inializing the endpoint array. rc = %d", rc));
  47             return rc;
  48         }
  49 
  50 
  51         /* NTH: might want to vary this size based off the universe size (if
  52          * one exists). the table is only used for connection lookup and
  53          * endpoint removal. */
  54         rc = opal_hash_table_init (&ugni_module->id_to_endpoint, INITIAL_GNI_EPS);
  55         if (OPAL_SUCCESS != rc) {
  56             BTL_ERROR(("error initializing the endpoint hash. rc = %d", rc));
  57             return rc;
  58         }
  59     }
  60 
  61     for (size_t i = 0 ; i < nprocs ; ++i) {
  62         peers[i] = mca_btl_ugni_get_ep (btl, procs[i]);
  63         if (NULL == peers[i]) {
  64             continue;
  65         }
  66 
  67         if (procs[i] == opal_proc_local_get ()) {
  68             ugni_module->local_ep = peers[i];
  69         }
  70 
  71         /* Set the reachable bit if necessary */
  72         if (reachable) {
  73             (void) opal_bitmap_set_bit (reachable, i);
  74         }
  75     }
  76 
  77     mca_btl_ugni_module_set_max_reg (ugni_module, ugni_module->nlocal_procs);
  78 
  79     if (false == ugni_module->initialized) {
  80         for (int i = 0 ; i < mca_btl_ugni_component.virtual_device_count ; ++i) {
  81             mca_btl_ugni_device_t *device = ugni_module->devices + i;
  82             rc = GNI_CqCreate (device->dev_handle, mca_btl_ugni_component.local_rdma_cq_size, 0,
  83                                GNI_CQ_NOBLOCK, NULL, NULL, &device->dev_rdma_local_cq.gni_handle);
  84             if (GNI_RC_SUCCESS != rc) {
  85                 BTL_ERROR(("error creating local BTE/FMA CQ"));
  86                 return mca_btl_rc_ugni_to_opal (rc);
  87             }
  88 
  89             rc = GNI_CqCreate (device->dev_handle, mca_btl_ugni_component.local_cq_size,
  90                                0, GNI_CQ_NOBLOCK, NULL, NULL, &device->dev_smsg_local_cq.gni_handle);
  91             if (GNI_RC_SUCCESS != rc) {
  92                 BTL_ERROR(("error creating local SMSG CQ"));
  93                 return mca_btl_rc_ugni_to_opal (rc);
  94             }
  95 
  96             if (mca_btl_ugni_component.progress_thread_enabled) {
  97                 rc = GNI_CqCreate (device->dev_handle, mca_btl_ugni_component.local_rdma_cq_size,
  98                                    0, GNI_CQ_BLOCKING, NULL, NULL, &device->dev_rdma_local_irq_cq.gni_handle);
  99                 if (GNI_RC_SUCCESS != rc) {
 100                     BTL_ERROR(("error creating local BTE/FMA CQ"));
 101                     return mca_btl_rc_ugni_to_opal (rc);
 102                 }
 103             }
 104         }
 105 
 106         rc = GNI_CqCreate (ugni_module->devices[0].dev_handle, mca_btl_ugni_component.remote_cq_size,
 107                            0, GNI_CQ_NOBLOCK, NULL, NULL, &ugni_module->smsg_remote_cq);
 108         if (GNI_RC_SUCCESS != rc) {
 109             BTL_ERROR(("error creating remote SMSG CQ"));
 110             return mca_btl_rc_ugni_to_opal (rc);
 111         }
 112 
 113         if (mca_btl_ugni_component.progress_thread_enabled) {
 114             rc = GNI_CqCreate (ugni_module->devices[0].dev_handle, mca_btl_ugni_component.remote_cq_size,
 115                                0, GNI_CQ_BLOCKING, NULL, NULL, &ugni_module->smsg_remote_irq_cq);
 116             if (GNI_RC_SUCCESS != rc) {
 117                 BTL_ERROR(("error creating remote SMSG CQ"));
 118                 return mca_btl_rc_ugni_to_opal (rc);
 119             }
 120         }
 121 
 122         rc = mca_btl_ugni_setup_mpools (ugni_module);
 123         if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
 124             BTL_ERROR(("btl/ugni error setting up mpools/free lists"));
 125             return rc;
 126         }
 127 
 128         rc = mca_btl_ugni_smsg_init (ugni_module);
 129         if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
 130             BTL_ERROR(("btl/ugni error initializing SMSG"));
 131             return rc;
 132         }
 133 
 134         /*
 135          * If progress thread enabled, registered a page of memory
 136          * with the smsg_remote_irq_cq.  This memory handle is passed
 137          * to ranks which want to communicate with this rank. A rank which
 138          * posts a GNI_PostCqWrite targeting this memory handle generates
 139          * an IRQ at the target node, which ultimately causes the progress
 140          * thread in the target rank to become schedulable.
 141          */
 142         if (mca_btl_ugni_component.progress_thread_enabled) {
 143             mmap_start_addr = mmap(NULL, 4096, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);
 144             if (NULL == mmap_start_addr) {
 145                 BTL_ERROR(("btl/ugni mmap returned error"));
 146                 return OPAL_ERR_OUT_OF_RESOURCE;
 147             }
 148 
 149             rc = GNI_MemRegister(ugni_module->devices[0].dev_handle,
 150                                      (unsigned long)mmap_start_addr,
 151                                      4096,
 152                                      ugni_module->smsg_remote_irq_cq,
 153                                      GNI_MEM_READWRITE,
 154                                      -1,
 155                                      &ugni_module->devices[0].smsg_irq_mhndl);
 156 
 157             mca_btl_ugni_spawn_progress_thread(btl);
 158         }
 159 
 160         opal_event_evtimer_add (&ugni_module->connection_event, &tv);
 161 
 162         ugni_module->initialized = true;
 163     }
 164 
 165     return OPAL_SUCCESS;
 166 }
 167 
 168 int mca_btl_ugni_del_procs (struct mca_btl_base_module_t *btl,
 169                             size_t nprocs, struct opal_proc_t **procs,
 170                             struct mca_btl_base_endpoint_t **peers) {
 171     mca_btl_ugni_module_t *ugni_module = (mca_btl_ugni_module_t *) btl;
 172 
 173     OPAL_THREAD_LOCK(&ugni_module->endpoint_lock);
 174 
 175     for (size_t i = 0 ; i < nprocs ; ++i) {
 176         struct opal_proc_t *opal_proc = procs[i];
 177         uint64_t proc_id = mca_btl_ugni_proc_name_to_id(opal_proc->proc_name);
 178         mca_btl_base_endpoint_t *ep = NULL;
 179 
 180         /* lookup this proc in the hash table */
 181         (void) opal_hash_table_get_value_uint64 (&ugni_module->id_to_endpoint, proc_id, (void **) &ep);
 182 
 183         BTL_VERBOSE(("deleting endpoint with proc id 0x%" PRIx64 ", ptr: %p", proc_id, (void *) ep));
 184 
 185         if (NULL != ep) {
 186             mca_btl_ugni_release_ep (ep);
 187             --ugni_module->endpoint_count;
 188         }
 189 
 190         if (OPAL_PROC_ON_LOCAL_NODE(opal_proc->proc_flags)) {
 191             --ugni_module->nlocal_procs;
 192         }
 193 
 194         /* remote the endpoint from the hash table */
 195         opal_hash_table_set_value_uint64 (&ugni_module->id_to_endpoint, proc_id, NULL);
 196     }
 197 
 198     OPAL_THREAD_UNLOCK(&ugni_module->endpoint_lock);
 199 
 200     mca_btl_ugni_module_set_max_reg (ugni_module, ugni_module->nlocal_procs);
 201 
 202     return OPAL_SUCCESS;
 203 }
 204 
 205 
 206 struct mca_btl_base_endpoint_t *mca_btl_ugni_get_ep (struct mca_btl_base_module_t *module, opal_proc_t *proc)
 207 {
 208     mca_btl_ugni_module_t *ugni_module = (mca_btl_ugni_module_t *) module;
 209     uint64_t proc_id = mca_btl_ugni_proc_name_to_id(proc->proc_name);
 210     mca_btl_base_endpoint_t *ep;
 211     int rc;
 212 
 213     OPAL_THREAD_LOCK(&ugni_module->endpoint_lock);
 214 
 215     do {
 216         rc = opal_hash_table_get_value_uint64 (&ugni_module->id_to_endpoint, proc_id, (void **) &ep);
 217         if (OPAL_SUCCESS == rc) {
 218             BTL_VERBOSE(("returning existing endpoint for proc %s", OPAL_NAME_PRINT(proc->proc_name)));
 219             break;
 220         }
 221 
 222         BTL_VERBOSE(("initialized uGNI endpoint for proc id: 0x%" PRIx64 " ptr: %p", proc_id, (void *) proc));
 223 
 224         /*  Create and Init endpoints */
 225         rc = mca_btl_ugni_init_ep (ugni_module, &ep, ugni_module, proc);
 226         if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
 227             BTL_ERROR(("btl/ugni error initializing endpoint"));
 228             break;
 229         }
 230 
 231         /* ugni is allowed on local processes to provide support for network atomic operations */
 232         if (OPAL_PROC_ON_LOCAL_NODE(proc->proc_flags)) {
 233             ++ugni_module->nlocal_procs;
 234         }
 235         ++ugni_module->endpoint_count;
 236 
 237         /* add this endpoint to the connection lookup table */
 238         opal_hash_table_set_value_uint64 (&ugni_module->id_to_endpoint, proc_id, ep);
 239     } while (0);
 240 
 241     OPAL_THREAD_UNLOCK(&ugni_module->endpoint_lock);
 242 
 243     return ep;
 244 }
 245 
 246 
 247 static int ugni_reg_mem (void *reg_data, void *base, size_t size,
 248                          mca_rcache_base_registration_t *reg)
 249 {
 250     mca_btl_ugni_module_t *ugni_module = (mca_btl_ugni_module_t *) reg_data;
 251     gni_cq_handle_t cq = 0;
 252     int flags, rc;
 253 
 254     if (ugni_module->reg_count >= ugni_module->reg_max) {
 255         return OPAL_ERR_OUT_OF_RESOURCE;
 256     }
 257 
 258     if (reg->access_flags & (MCA_RCACHE_ACCESS_REMOTE_WRITE | MCA_RCACHE_ACCESS_LOCAL_WRITE |
 259                              MCA_RCACHE_ACCESS_REMOTE_ATOMIC)) {
 260         flags = GNI_MEM_READWRITE;
 261     } else {
 262         flags = GNI_MEM_READ_ONLY;
 263     }
 264 
 265     if (!(reg->flags & MCA_RCACHE_FLAGS_SO_MEM)) {
 266         flags |= GNI_MEM_RELAXED_PI_ORDERING;
 267     }
 268 
 269     if (reg->flags & MCA_RCACHE_FLAGS_RESV0) {
 270         cq = ugni_module->smsg_remote_cq;
 271     }
 272 
 273     rc = mca_btl_ugni_reg_mem (ugni_module, base, size, (mca_btl_ugni_reg_t *) reg, cq, flags);
 274     if (OPAL_LIKELY(OPAL_SUCCESS == rc)) {
 275         opal_atomic_add_fetch_32(&ugni_module->reg_count,1);
 276     }
 277 
 278     return rc;
 279 }
 280 
 281 static int
 282 ugni_dereg_mem (void *reg_data, mca_rcache_base_registration_t *reg)
 283 {
 284     mca_btl_ugni_module_t *ugni_module = (mca_btl_ugni_module_t *) reg_data;
 285     int rc;
 286 
 287     rc = mca_btl_ugni_dereg_mem (ugni_module, (mca_btl_ugni_reg_t *) reg);
 288     if (OPAL_LIKELY(OPAL_SUCCESS == rc)) {
 289         opal_atomic_add_fetch_32(&ugni_module->reg_count,-1);
 290     }
 291 
 292     return rc;
 293 }
 294 
 295 static int
 296 mca_btl_ugni_setup_mpools (mca_btl_ugni_module_t *ugni_module)
 297 {
 298     mca_rcache_udreg_resources_t rcache_resources;
 299     unsigned int mbox_increment;
 300     uint32_t nprocs, *u32;
 301     char *rcache_name;
 302     int rc;
 303 
 304     rc = opal_pointer_array_init (&ugni_module->pending_smsg_frags_bb, 0,
 305                                   1 << 30, 32768);
 306     if (OPAL_SUCCESS != rc) {
 307         return rc;
 308     }
 309 
 310     /* determine how many procs are in the job (might want to check universe size here) */
 311     u32 = &nprocs;
 312     OPAL_MODEX_RECV_VALUE(rc, OPAL_PMIX_UNIV_SIZE, &OPAL_PROC_MY_NAME,
 313                           &u32, OPAL_UINT32);
 314     if (OPAL_SUCCESS != rc) {
 315         /* take a wild conservative guess */
 316         nprocs = 512;
 317     }
 318 
 319     rc = mca_btl_ugni_smsg_setup (nprocs);
 320     if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
 321         BTL_ERROR(("error setting up smsg"));
 322         return rc;
 323     }
 324 
 325     rc = opal_free_list_init (ugni_module->frags_lists + MCA_BTL_UGNI_LIST_SMSG,
 326                               sizeof (mca_btl_ugni_smsg_frag_t),
 327                               opal_cache_line_size, OBJ_CLASS(mca_btl_ugni_smsg_frag_t),
 328                               mca_btl_ugni_component.ugni_smsg_limit,
 329                               opal_cache_line_size,
 330                               mca_btl_ugni_component.ugni_free_list_num,
 331                               mca_btl_ugni_component.ugni_free_list_max,
 332                               mca_btl_ugni_component.ugni_free_list_inc,
 333                               NULL, 0, NULL, (opal_free_list_item_init_fn_t) mca_btl_ugni_frag_init,
 334                               (void *) (intptr_t) MCA_BTL_UGNI_LIST_SMSG);
 335     if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
 336         BTL_ERROR(("error creating smsg fragment free list"));
 337         return rc;
 338     }
 339 
 340     rc = opal_free_list_init (ugni_module->frags_lists + MCA_BTL_UGNI_LIST_RDMA,
 341                               sizeof (mca_btl_ugni_rdma_frag_t), 64,
 342                               OBJ_CLASS(mca_btl_ugni_rdma_frag_t),
 343                               0, opal_cache_line_size,
 344                               mca_btl_ugni_component.ugni_free_list_num,
 345                               mca_btl_ugni_component.ugni_free_list_max,
 346                               mca_btl_ugni_component.ugni_free_list_inc,
 347                               NULL, 0, NULL, (opal_free_list_item_init_fn_t) mca_btl_ugni_frag_init,
 348                               (void *) (intptr_t) MCA_BTL_UGNI_LIST_RDMA);
 349     if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
 350         return rc;
 351     }
 352 
 353     rc = opal_free_list_init (ugni_module->frags_lists + MCA_BTL_UGNI_LIST_RDMA_INT,
 354                               sizeof (mca_btl_ugni_rdma_frag_t), 8,
 355                               OBJ_CLASS(mca_btl_ugni_rdma_frag_t),
 356                               0, opal_cache_line_size, 0, -1, 64,
 357                               NULL, 0, NULL, (opal_free_list_item_init_fn_t) mca_btl_ugni_frag_init,
 358                               (void *) (intptr_t) MCA_BTL_UGNI_LIST_RDMA_INT);
 359     if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
 360         return rc;
 361     }
 362 
 363     ugni_module->super.btl_mpool = mca_mpool_base_module_lookup (mca_btl_ugni_component.mpool_hints);
 364     if (NULL == ugni_module->super.btl_mpool) {
 365         BTL_ERROR(("could not find mpool matching hints %s", mca_btl_ugni_component.mpool_hints));
 366         return OPAL_ERROR;
 367     }
 368 
 369     rcache_resources.base.cache_name     = "ompi.ugni";
 370     rcache_resources.base.reg_data       = (void *) ugni_module;
 371     rcache_resources.base.sizeof_reg     = sizeof (mca_btl_ugni_reg_t);
 372     rcache_resources.base.register_mem   = ugni_reg_mem;
 373     rcache_resources.base.deregister_mem = ugni_dereg_mem;
 374 
 375     if (MCA_BTL_UGNI_RCACHE_UDREG == mca_btl_ugni_component.rcache_type) {
 376         /* additional settings for the udreg mpool */
 377         /* 4k should be large enough for any Gemini/Ares system */
 378         rcache_resources.max_entries       = 4096;
 379         rcache_resources.use_kernel_cache  = true;
 380 
 381         rcache_resources.use_evict_w_unreg = false;
 382         rcache_name = "udreg";
 383     } else {
 384         rcache_name = "grdma";
 385     }
 386 
 387     ugni_module->rcache =
 388         mca_rcache_base_module_create (rcache_name, ugni_module->devices, &rcache_resources.base);
 389 
 390     if (NULL == ugni_module->rcache) {
 391         BTL_ERROR(("error creating registration cache"));
 392         return OPAL_ERROR;
 393     }
 394 
 395     rc = opal_free_list_init (ugni_module->frags_lists + MCA_BTL_UGNI_LIST_EAGER_SEND,
 396                               sizeof (mca_btl_ugni_eager_frag_t), 8,
 397                               OBJ_CLASS(mca_btl_ugni_eager_frag_t),
 398                               ugni_module->super.btl_eager_limit, 64,
 399                               mca_btl_ugni_component.ugni_eager_num,
 400                               mca_btl_ugni_component.ugni_eager_max,
 401                               mca_btl_ugni_component.ugni_eager_inc,
 402                               ugni_module->super.btl_mpool, 0, ugni_module->rcache,
 403                               (opal_free_list_item_init_fn_t) mca_btl_ugni_frag_init,
 404                               (void *) (intptr_t) MCA_BTL_UGNI_LIST_EAGER_SEND);
 405     if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
 406         BTL_ERROR(("error creating eager send fragment free list"));
 407         return rc;
 408     }
 409 
 410     rc = opal_free_list_init (ugni_module->frags_lists + MCA_BTL_UGNI_LIST_EAGER_RECV,
 411                               sizeof (mca_btl_ugni_eager_frag_t), 8,
 412                               OBJ_CLASS(mca_btl_ugni_eager_frag_t),
 413                               ugni_module->super.btl_eager_limit, 64,
 414                               mca_btl_ugni_component.ugni_eager_num,
 415                               mca_btl_ugni_component.ugni_eager_max,
 416                               mca_btl_ugni_component.ugni_eager_inc,
 417                               ugni_module->super.btl_mpool, 0, ugni_module->rcache,
 418                               (opal_free_list_item_init_fn_t) mca_btl_ugni_frag_init,
 419                               (void *) (intptr_t) MCA_BTL_UGNI_LIST_EAGER_RECV);
 420     if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
 421         BTL_ERROR(("error creating eager receive fragment free list"));
 422         return rc;
 423     }
 424 
 425     if (0 == mca_btl_ugni_component.mbox_increment) {
 426         /* limit mailbox allocations to either 12.5% of available registrations
 427            or 2MiB per allocation */
 428         mbox_increment = (unsigned int) (2097152.0 / (float)mca_btl_ugni_component.smsg_mbox_size);
 429 
 430         /* we may end up using more */
 431         if (nprocs/mbox_increment > (unsigned int) ugni_module->reg_max / 8) {
 432             mbox_increment = nprocs / (ugni_module->reg_max >> 3);
 433         }
 434     } else {
 435         mbox_increment = mca_btl_ugni_component.mbox_increment;
 436     }
 437 
 438     /* use the MCA_RCACHE_FLAGS_RESV0 to signal this is smsg memory */
 439     rc = opal_free_list_init (&ugni_module->smsg_mboxes,
 440                               sizeof (mca_btl_ugni_smsg_mbox_t), 8,
 441                               OBJ_CLASS(mca_btl_ugni_smsg_mbox_t),
 442                               mca_btl_ugni_component.smsg_mbox_size, 128,
 443                               32, -1, mbox_increment, ugni_module->super.btl_mpool,
 444                               MCA_RCACHE_FLAGS_SO_MEM | MCA_RCACHE_FLAGS_RESV0,
 445                               ugni_module->rcache, NULL, NULL);
 446     if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
 447         BTL_ERROR(("error creating smsg mailbox free list"));
 448         return rc;
 449     }
 450 
 451     return OPAL_SUCCESS;
 452 }
 453 
 454 static void
 455 mca_btl_ugni_module_set_max_reg (mca_btl_ugni_module_t *ugni_module, int nlocal_procs)
 456 {
 457     if (0 == mca_btl_ugni_component.max_mem_reg) {
 458 #if defined(HAVE_GNI_GETJOBRESINFO)
 459         gni_job_res_desc_t res_des;
 460         gni_return_t grc;
 461         int fuzz = 20;
 462 
 463         grc = GNI_GetJobResInfo (0, mca_btl_ugni_component.ptag,
 464                                  GNI_JOB_RES_MDD, &res_des);
 465         if (GNI_RC_SUCCESS == grc) {
 466             if (nlocal_procs) {
 467                 ugni_module->reg_max = (res_des.limit - fuzz) / nlocal_procs;
 468             } else {
 469                 ugni_module->reg_max = 0;
 470             }
 471         }
 472 #else
 473         /* no way to determine the maximum registration count */
 474         if (nlocal_procs) {
 475             ugni_module->reg_max = 1200 / nlocal_procs;
 476         } else {
 477             ugni_module->reg_max = 0;
 478         }
 479 #endif
 480     } else if (-1 == mca_btl_ugni_component.max_mem_reg) {
 481         ugni_module->reg_max = INT_MAX;
 482     } else {
 483         ugni_module->reg_max = mca_btl_ugni_component.max_mem_reg;
 484     }
 485 
 486     ugni_module->reg_count = 0;
 487 }
 488 
 489 static int mca_btl_ugni_smsg_setup (int nprocs)
 490 {
 491     gni_smsg_attr_t tmp_smsg_attrib;
 492     unsigned int mbox_size;
 493     gni_return_t grc;
 494 
 495     if (0 == mca_btl_ugni_component.ugni_smsg_limit) {
 496         /* auto-set the smsg limit based on the number of ranks */
 497         if (nprocs <= 512) {
 498             mca_btl_ugni_component.ugni_smsg_limit = 8192;
 499         } else if (nprocs <= 1024) {
 500             mca_btl_ugni_component.ugni_smsg_limit = 2048;
 501         } else if (nprocs <= 8192) {
 502             mca_btl_ugni_component.ugni_smsg_limit = 1024;
 503         } else if (nprocs <= 16384) {
 504             mca_btl_ugni_component.ugni_smsg_limit = 512;
 505         } else {
 506             mca_btl_ugni_component.ugni_smsg_limit = 256;
 507         }
 508     }
 509 
 510     mca_btl_ugni_component.smsg_max_data = mca_btl_ugni_component.ugni_smsg_limit -
 511         sizeof (mca_btl_ugni_send_frag_hdr_t);
 512 
 513     if (mca_btl_ugni_component.ugni_smsg_limit == mca_btl_ugni_module.super.btl_eager_limit) {
 514         mca_btl_ugni_module.super.btl_eager_limit = mca_btl_ugni_component.smsg_max_data;
 515     }
 516 
 517     /* calculate mailbox size */
 518     tmp_smsg_attrib.msg_type       = GNI_SMSG_TYPE_MBOX_AUTO_RETRANSMIT;
 519     tmp_smsg_attrib.msg_maxsize    = mca_btl_ugni_component.ugni_smsg_limit;
 520     tmp_smsg_attrib.mbox_maxcredit = mca_btl_ugni_component.smsg_max_credits;
 521 
 522     grc = GNI_SmsgBufferSizeNeeded (&tmp_smsg_attrib, &mbox_size);
 523     if (OPAL_UNLIKELY(GNI_RC_SUCCESS != grc)) {
 524         BTL_ERROR(("error in GNI_SmsgBufferSizeNeeded"));
 525         return mca_btl_rc_ugni_to_opal (grc);
 526     }
 527 
 528     mca_btl_ugni_component.smsg_mbox_size = OPAL_ALIGN(mbox_size, 64, unsigned int);
 529 
 530     return OPAL_SUCCESS;
 531 }

/* [<][>][^][v][top][bottom][index][help] */