root/ompi/mca/pml/ob1/pml_ob1_component.c

DEFINITIONS

1. mca_pml_ob1_param_register_int
2. mca_pml_ob1_param_register_uint
3. mca_pml_ob1_param_register_sizet
4. mca_pml_ob1_comm_size_notify
5. mca_pml_ob1_get_unex_msgq_size
6. mca_pml_ob1_get_posted_recvq_size
7. mca_pml_ob1_component_register
8. mca_pml_ob1_component_open
9. mca_pml_ob1_component_close
10. mca_pml_ob1_component_init
11. mca_pml_ob1_component_fini
12. mca_pml_ob1_seg_alloc
13. mca_pml_ob1_seg_free

   1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
   2 /*
   3  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
   4  *                         University Research and Technology
   5  *                         Corporation.  All rights reserved.
   6  * Copyright (c) 2004-2016 The University of Tennessee and The University
   7  *                         of Tennessee Research Foundation.  All rights
   8  *                         reserved.
   9  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
  10  *                         University of Stuttgart.  All rights reserved.
  11  * Copyright (c) 2004-2005 The Regents of the University of California.
  12  *                         All rights reserved.
  13  * Copyright (c) 2007-2010 Cisco Systems, Inc.  All rights reserved.
  14  * Copyright (c) 2010      Oracle and/or its affiliates.  All rights reserved
  15  * Copyright (c) 2013-2017 Los Alamos National Security, LLC. All rights
  16  *                         reserved.
  17  * Copyright (c) 2018      Sandia National Laboratories
  18  *                         All rights reserved.
  19  * Copyright (c) 2018      Research Organization for Information Science
  20  *                         and Technology (RIST).  All rights reserved.
  21  * $COPYRIGHT$
  22  *
  23  * Additional copyrights may follow
  24  *
  25  * $HEADER$
  26  */
  27 
  28 #include "ompi_config.h"
  29 #include "opal/mca/event/event.h"
  30 #include "mpi.h"
  31 #include "ompi/runtime/params.h"
  32 #include "ompi/mca/pml/pml.h"
  33 #include "ompi/mca/pml/base/pml_base_bsend.h"
  34 #include "pml_ob1.h"
  35 #include "pml_ob1_hdr.h"
  36 #include "pml_ob1_sendreq.h"
  37 #include "pml_ob1_recvreq.h"
  38 #include "pml_ob1_rdmafrag.h"
  39 #include "pml_ob1_recvfrag.h"
  40 #include "ompi/mca/bml/base/base.h"
  41 #include "pml_ob1_component.h"
  42 #include "opal/mca/allocator/base/base.h"
  43 #include "opal/mca/base/mca_base_pvar.h"
  44 #include "opal/runtime/opal_params.h"
  45 #include "opal/mca/btl/base/base.h"
  46 
  47 OBJ_CLASS_INSTANCE( mca_pml_ob1_pckt_pending_t,
  48                     opal_free_list_item_t,
  49                     NULL,
  50                     NULL );
  51 
  52 static int mca_pml_ob1_component_register(void);
  53 static int mca_pml_ob1_component_open(void);
  54 static int mca_pml_ob1_component_close(void);
  55 static mca_pml_base_module_t*
  56 mca_pml_ob1_component_init( int* priority, bool enable_progress_threads,
  57                             bool enable_mpi_threads );
  58 static int mca_pml_ob1_component_fini(void);
  59 int mca_pml_ob1_output = 0;
  60 static int mca_pml_ob1_verbose = 0;
  61 bool mca_pml_ob1_matching_protection = false;
  62 
  63 mca_pml_base_component_2_0_0_t mca_pml_ob1_component = {
  64     /* First, the mca_base_component_t struct containing meta
  65        information about the component itself */
  66 
  67     .pmlm_version = {
  68         MCA_PML_BASE_VERSION_2_0_0,
  69 
  70         .mca_component_name = "ob1",
  71         .mca_component_major_version = OMPI_MAJOR_VERSION,
  72         .mca_component_minor_version = OMPI_MINOR_VERSION,
  73         .mca_component_release_version = OMPI_RELEASE_VERSION,
  74         .mca_open_component = mca_pml_ob1_component_open,
  75         .mca_close_component = mca_pml_ob1_component_close,
  76         .mca_register_component_params = mca_pml_ob1_component_register
  77     },
  78     .pmlm_data = {
  79         /* The component is checkpoint ready */
  80         MCA_BASE_METADATA_PARAM_CHECKPOINT
  81     },
  82 
  83     .pmlm_init = mca_pml_ob1_component_init,
  84     .pmlm_finalize = mca_pml_ob1_component_fini,
  85 };
  86 
  87 void *mca_pml_ob1_seg_alloc (void *ctx, size_t* size);
  88 
  89 void mca_pml_ob1_seg_free (void *ctx, void *segment);
  90 
  91 static inline int mca_pml_ob1_param_register_int(
  92     const char* param_name,
  93     int default_value,
  94     int *storage)
  95 {
  96     *storage = default_value;
  97     (void) mca_base_component_var_register(&mca_pml_ob1_component.pmlm_version, param_name,
  98                                            NULL, MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
  99                                            OPAL_INFO_LVL_9,
 100                                            MCA_BASE_VAR_SCOPE_READONLY, storage);
 101     return *storage;
 102 }
 103 
 104 static inline unsigned int mca_pml_ob1_param_register_uint(
 105     const char* param_name,
 106     unsigned int default_value,
 107     unsigned int *storage)
 108 {
 109     *storage = default_value;
 110     (void) mca_base_component_var_register(&mca_pml_ob1_component.pmlm_version, param_name,
 111                                            NULL, MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, 0, 0,
 112                                            OPAL_INFO_LVL_9,
 113                                            MCA_BASE_VAR_SCOPE_READONLY, storage);
 114     return *storage;
 115 }
 116 
 117 #if 0
 118 static inline size_t mca_pml_ob1_param_register_sizet(
 119     const char* param_name,
 120     size_t default_value,
 121     size_t *storage)
 122 {
 123     *storage = default_value;
 124     (void) mca_base_component_var_register(&mca_pml_ob1_component.pmlm_version, param_name,
 125                                            NULL, MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0,
 126                                            OPAL_INFO_LVL_9,
 127                                            MCA_BASE_VAR_SCOPE_READONLY, storage);
 128     return *storage;
 129 }
 130 #endif
 131 
 132 static int mca_pml_ob1_comm_size_notify (mca_base_pvar_t *pvar, mca_base_pvar_event_t event, void *obj_handle, int *count)
 133 {
 134     if (MCA_BASE_PVAR_HANDLE_BIND == event) {
 135         /* Return the size of the communicator as the number of values */
 136         *count = ompi_comm_size ((ompi_communicator_t *) obj_handle);
 137     }
 138 
 139     return OMPI_SUCCESS;
 140 }
 141 
 142 static int mca_pml_ob1_get_unex_msgq_size (const struct mca_base_pvar_t *pvar, void *value, void *obj_handle)
 143 {
 144     ompi_communicator_t *comm = (ompi_communicator_t *) obj_handle;
 145     mca_pml_ob1_comm_t *pml_comm = comm->c_pml_comm;
 146     int comm_size = ompi_comm_size (comm);
 147     unsigned *values = (unsigned *) value;
 148     mca_pml_ob1_comm_proc_t *pml_proc;
 149     int i;
 150 
 151     for (i = 0 ; i < comm_size ; ++i) {
 152         pml_proc = pml_comm->procs[i];
 153         if (pml_proc) {
 154 #if MCA_PML_OB1_CUSTOM_MATCH
 155             values[i] = custom_match_umq_size(pml_comm->umq); // TODO: given the structure of custom match this does not make sense,
 156                                                      //       as we only have one set of queues.
 157 #else
 158             values[i] = opal_list_get_size (&pml_proc->unexpected_frags);
 159 #endif
 160         } else {
 161             values[i] = 0;
 162         }
 163     }
 164 
 165     return OMPI_SUCCESS;
 166 }
 167 
 168 static int mca_pml_ob1_get_posted_recvq_size (const struct mca_base_pvar_t *pvar, void *value, void *obj_handle)
 169 {
 170     ompi_communicator_t *comm = (ompi_communicator_t *) obj_handle;
 171     mca_pml_ob1_comm_t *pml_comm = comm->c_pml_comm;
 172     int comm_size = ompi_comm_size (comm);
 173     unsigned *values = (unsigned *) value;
 174     mca_pml_ob1_comm_proc_t *pml_proc;
 175     int i;
 176 
 177     for (i = 0 ; i < comm_size ; ++i) {
 178         pml_proc = pml_comm->procs[i];
 179 
 180         if (pml_proc) {
 181 #if MCA_PML_OB1_CUSTOM_MATCH
 182             values[i] = custom_match_prq_size(pml_comm->prq); // TODO: given the structure of custom match this does not make sense,
 183                                                      //       as we only have one set of queues.
 184 #else
 185             values[i] = opal_list_get_size (&pml_proc->specific_receives);
 186 #endif
 187         } else {
 188             values[i] = 0;
 189         }
 190     }
 191 
 192     return OMPI_SUCCESS;
 193 }
 194 
 195 static int mca_pml_ob1_component_register(void)
 196 {
 197     mca_pml_ob1_param_register_int("verbose", 0, &mca_pml_ob1_verbose);
 198 
 199     mca_pml_ob1_param_register_int("free_list_num", 4, &mca_pml_ob1.free_list_num);
 200     mca_pml_ob1_param_register_int("free_list_max", -1, &mca_pml_ob1.free_list_max);
 201     mca_pml_ob1_param_register_int("free_list_inc", 64, &mca_pml_ob1.free_list_inc);
 202     mca_pml_ob1_param_register_int("priority", 20, &mca_pml_ob1.priority);
 203     mca_pml_ob1_param_register_int("send_pipeline_depth", 3, &mca_pml_ob1.send_pipeline_depth);
 204     mca_pml_ob1_param_register_int("recv_pipeline_depth", 4, &mca_pml_ob1.recv_pipeline_depth);
 205 
 206     /* NTH: we can get into a live-lock situation in the RDMA failure path so disable
 207        RDMA retries for now. Falling back to send may suck but it is better than
 208        hanging */
 209     mca_pml_ob1.rdma_retries_limit = 0;
 210     /*     mca_pml_ob1_param_register_sizet("rdma_retries_limit", 5, &mca_pml_ob1.rdma_retries_limit); */
 211 
 212     mca_pml_ob1_param_register_int("max_rdma_per_request", 4, &mca_pml_ob1.max_rdma_per_request);
 213     mca_pml_ob1_param_register_int("max_send_per_range", 4, &mca_pml_ob1.max_send_per_range);
 214 
 215     mca_pml_ob1_param_register_uint("unexpected_limit", 128, &mca_pml_ob1.unexpected_limit);
 216 
 217     mca_pml_ob1.use_all_rdma = false;
 218     (void) mca_base_component_var_register(&mca_pml_ob1_component.pmlm_version, "use_all_rdma",
 219                                            "Use all available RDMA btls for the RDMA and RDMA pipeline protocols "
 220                                            "(default: false)", MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
 221                                            OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_GROUP, &mca_pml_ob1.use_all_rdma);
 222 
 223     mca_pml_ob1.allocator_name = "bucket";
 224     (void) mca_base_component_var_register(&mca_pml_ob1_component.pmlm_version, "allocator",
 225                                            "Name of allocator component for unexpected messages",
 226                                            MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_9,
 227                                            MCA_BASE_VAR_SCOPE_READONLY, &mca_pml_ob1.allocator_name);
 228     (void)mca_base_component_pvar_register(&mca_pml_ob1_component.pmlm_version,
 229                                            "unexpected_msgq_length", "Number of unexpected messages "
 230                                            "received by each peer in a communicator", OPAL_INFO_LVL_4, MPI_T_PVAR_CLASS_SIZE,
 231                                            MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, MPI_T_BIND_MPI_COMM,
 232                                            MCA_BASE_PVAR_FLAG_READONLY | MCA_BASE_PVAR_FLAG_CONTINUOUS,
 233                                            mca_pml_ob1_get_unex_msgq_size, NULL, mca_pml_ob1_comm_size_notify, NULL);
 234 
 235     (void)mca_base_component_pvar_register(&mca_pml_ob1_component.pmlm_version,
 236                                            "posted_recvq_length", "Number of unmatched receives "
 237                                            "posted for each peer in a communicator", OPAL_INFO_LVL_4, MPI_T_PVAR_CLASS_SIZE,
 238                                            MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, MPI_T_BIND_MPI_COMM,
 239                                            MCA_BASE_PVAR_FLAG_READONLY | MCA_BASE_PVAR_FLAG_CONTINUOUS,
 240                                            mca_pml_ob1_get_posted_recvq_size, NULL, mca_pml_ob1_comm_size_notify, NULL);
 241 
 242     return OMPI_SUCCESS;
 243 }
 244 
 245 static int mca_pml_ob1_component_open(void)
 246 {
 247     mca_pml_ob1_output = opal_output_open(NULL);
 248     opal_output_set_verbosity(mca_pml_ob1_output, mca_pml_ob1_verbose);
 249 
 250     mca_pml_ob1.enabled = false;
 251     return mca_base_framework_open(&ompi_bml_base_framework, 0);
 252 }
 253 
 254 
 255 static int mca_pml_ob1_component_close(void)
 256 {
 257     int rc;
 258 
 259     if (OMPI_SUCCESS != (rc = mca_base_framework_close(&ompi_bml_base_framework))) {
 260          return rc;
 261     }
 262     opal_output_close(mca_pml_ob1_output);
 263 
 264     return OMPI_SUCCESS;
 265 }
 266 
 267 
 268 static mca_pml_base_module_t*
 269 mca_pml_ob1_component_init( int* priority,
 270                             bool enable_progress_threads,
 271                             bool enable_mpi_threads )
 272 {
 273     mca_allocator_base_component_t* allocator_component;
 274 
 275     opal_output_verbose( 10, mca_pml_ob1_output,
 276                          "in ob1, my priority is %d\n", mca_pml_ob1.priority);
 277 
 278     *priority = mca_pml_ob1.priority;
 279 
 280     allocator_component = mca_allocator_component_lookup( mca_pml_ob1.allocator_name );
 281     if(NULL == allocator_component) {
 282         opal_output(0, "mca_pml_ob1_component_init: can't find allocator: %s\n", mca_pml_ob1.allocator_name);
 283         return NULL;
 284     }
 285 
 286     mca_pml_ob1.allocator = allocator_component->allocator_init(true,
 287                                                                 mca_pml_ob1_seg_alloc,
 288                                                                 mca_pml_ob1_seg_free, NULL);
 289     if(NULL == mca_pml_ob1.allocator) {
 290         opal_output(0, "mca_pml_ob1_component_init: unable to initialize allocator\n");
 291         return NULL;
 292     }
 293 
 294     if(OMPI_SUCCESS != mca_bml_base_init( enable_progress_threads,
 295                                           enable_mpi_threads)) {
 296         return NULL;
 297     }
 298 
 299     /* check if any btls do not support dynamic add_procs */
 300     mca_btl_base_selected_module_t* selected_btl;
 301     OPAL_LIST_FOREACH(selected_btl, &mca_btl_base_modules_initialized, mca_btl_base_selected_module_t) {
 302         mca_btl_base_module_t *btl = selected_btl->btl_module;
 303 
 304         if (btl->btl_flags & MCA_BTL_FLAGS_BTL_PROGRESS_THREAD_ENABLED) {
 305             mca_pml_ob1_matching_protection = true;
 306         }
 307 
 308         if (btl->btl_flags & MCA_BTL_FLAGS_SINGLE_ADD_PROCS) {
 309             mca_pml_ob1.super.pml_flags |= MCA_PML_BASE_FLAG_REQUIRE_WORLD;
 310             break;
 311         }
 312 
 313     }
 314 
 315     return &mca_pml_ob1.super;
 316 }
 317 
 318 int mca_pml_ob1_component_fini(void)
 319 {
 320     int rc;
 321 
 322     /* Shutdown BML */
 323     if(OMPI_SUCCESS != (rc = mca_bml.bml_finalize()))
 324         return rc;
 325 
 326     if(!mca_pml_ob1.enabled) {
 327         if( NULL != mca_pml_ob1.allocator ) {
 328             (void)mca_pml_ob1.allocator->alc_finalize(mca_pml_ob1.allocator);
 329             mca_pml_ob1.allocator = NULL;
 330         }
 331 
 332         return OMPI_SUCCESS; /* never selected.. return success.. */
 333     }
 334     mca_pml_ob1.enabled = false;  /* not anymore */
 335 
 336     /* return the static receive/send requests to the respective free list and
 337      * let the free list handle destruction. */
 338     if( NULL != mca_pml_ob1_recvreq ) {
 339         opal_free_list_return (&mca_pml_base_recv_requests, (opal_free_list_item_t *) mca_pml_ob1_recvreq);
 340         mca_pml_ob1_recvreq = NULL;
 341     }
 342 
 343     if( NULL != mca_pml_ob1_sendreq ) {
 344         opal_free_list_return (&mca_pml_base_send_requests, (opal_free_list_item_t *) mca_pml_ob1_sendreq);
 345         mca_pml_ob1_sendreq = NULL;
 346     }
 347 
 348     OBJ_DESTRUCT(&mca_pml_ob1.rdma_pending);
 349     OBJ_DESTRUCT(&mca_pml_ob1.pckt_pending);
 350     OBJ_DESTRUCT(&mca_pml_ob1.recv_pending);
 351     OBJ_DESTRUCT(&mca_pml_ob1.send_pending);
 352     OBJ_DESTRUCT(&mca_pml_ob1.non_existing_communicator_pending);
 353     OBJ_DESTRUCT(&mca_pml_ob1.buffers);
 354     OBJ_DESTRUCT(&mca_pml_ob1.pending_pckts);
 355     OBJ_DESTRUCT(&mca_pml_ob1.recv_frags);
 356     OBJ_DESTRUCT(&mca_pml_ob1.rdma_frags);
 357     OBJ_DESTRUCT(&mca_pml_ob1.lock);
 358     OBJ_DESTRUCT(&mca_pml_ob1.send_ranges);
 359 
 360     if( NULL != mca_pml_ob1.allocator ) {
 361         (void)mca_pml_ob1.allocator->alc_finalize(mca_pml_ob1.allocator);
 362         mca_pml_ob1.allocator = NULL;
 363     }
 364 
 365 #if 0
 366     if (mca_pml_base_send_requests.fl_num_allocated !=
 367         mca_pml_base_send_requests.super.opal_list_length) {
 368         opal_output(0, "ob1 send requests: %d allocated %d returned\n",
 369                     mca_pml_base_send_requests.fl_num_allocated,
 370                     mca_pml_base_send_requests.super.opal_list_length);
 371     }
 372     if (mca_pml_base_recv_requests.fl_num_allocated !=
 373         mca_pml_base_recv_requests.super.opal_list_length) {
 374         opal_output(0, "ob1 recv requests: %d allocated %d returned\n",
 375                     mca_pml_base_recv_requests.fl_num_allocated,
 376                     mca_pml_base_recv_requests.super.opal_list_length);
 377     }
 378 #endif
 379 
 380     return OMPI_SUCCESS;
 381 }
 382 
 383 void *mca_pml_ob1_seg_alloc (void *ctx, size_t *size)
 384 {
 385     return malloc(*size);
 386 }
 387 
 388 void mca_pml_ob1_seg_free (void *ctx, void *segment)
 389 {
 390     free(segment);
 391 }