root/opal/mca/btl/self/btl_self_component.c

DEFINITIONS

1. mca_btl_self_component_register
2. mca_btl_self_component_open
3. mca_btl_self_component_close
4. mca_btl_self_component_init

   1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
   2 /*
   3  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
   4  *                         University Research and Technology
   5  *                         Corporation.  All rights reserved.
   6  * Copyright (c) 2004-2006 The University of Tennessee and The University
   7  *                         of Tennessee Research Foundation.  All rights
   8  *                         reserved.
   9  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
  10  *                         University of Stuttgart.  All rights reserved.
  11  * Copyright (c) 2004-2005 The Regents of the University of California.
  12  *                         All rights reserved.
  13  * Copyright (c) 2010      Cisco Systems, Inc. All rights reserved.
  14  * Copyright (c) 2014-2016 Los Alamos National Security, LLC. All rights
  15  *                         reserved.
  16  * $COPYRIGHT$
  17  *
  18  * Additional copyrights may follow
  19  *
  20  * $HEADER$
  21  */
  22 #include "opal_config.h"
  23 
  24 #include "btl_self.h"
  25 #include "btl_self_frag.h"
  26 
  27 static int mca_btl_self_component_register(void);
  28 static int mca_btl_self_component_open(void);
  29 static int mca_btl_self_component_close(void);
  30 
  31 /**
  32  * SELF module initialization.
  33  *
  34  * @param num_btls (OUT)                  Number of BTLs returned in BTL array.
  35  * @param enable_progress_threads (IN)    Flag indicating whether BTL is allowed to have progress threads
  36  * @param enable_mpi_threads (IN)         Flag indicating whether BTL must support multilple simultaneous invocations from different threads
  37  *
  38  */
  39 static mca_btl_base_module_t **mca_btl_self_component_init (int *num_btls,
  40                                                             bool enable_progress_threads,
  41                                                             bool enable_mpi_threads);
  42 
  43 /*
  44  * Shared Memory (SELF) component instance.
  45  */
  46 
  47 mca_btl_self_component_t mca_btl_self_component = {
  48     .super = {
  49         /* First, the mca_base_component_t struct containing meta information
  50           about the component itself */
  51         .btl_version = {
  52             MCA_BTL_DEFAULT_VERSION("self"),
  53             .mca_open_component = mca_btl_self_component_open,
  54             .mca_close_component = mca_btl_self_component_close,
  55             .mca_register_component_params = mca_btl_self_component_register,
  56         },
  57         .btl_data = {
  58             /* The component is checkpoint ready */
  59             .param_field = MCA_BASE_METADATA_PARAM_CHECKPOINT,
  60         },
  61 
  62         .btl_init = mca_btl_self_component_init,
  63     }  /* end super */
  64 };
  65 
  66 /*
  67  *  Called by MCA framework to open the component, registers
  68  *  component parameters.
  69  */
  70 
  71 static int mca_btl_self_component_register(void)
  72 {
  73     mca_base_var_group_component_register(&mca_btl_self_component.super.btl_version,
  74                                           "BTL for self communication");
  75 
  76     /* register SELF component parameters */
  77     mca_btl_self_component.free_list_num = 0;
  78     (void) mca_base_component_var_register(&mca_btl_self_component.super.btl_version, "free_list_num",
  79                                            "Number of fragments by default",
  80                                            MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
  81                                            OPAL_INFO_LVL_9,
  82                                            MCA_BASE_VAR_SCOPE_READONLY,
  83                                            &mca_btl_self_component.free_list_num);
  84     /* NTH: free list buffers are not released until we tear down so DO NOT make them unlimited here */
  85     mca_btl_self_component.free_list_max = 64;
  86     (void) mca_base_component_var_register(&mca_btl_self_component.super.btl_version, "free_list_max",
  87                                            "Maximum number of fragments",
  88                                            MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
  89                                            OPAL_INFO_LVL_9,
  90                                            MCA_BASE_VAR_SCOPE_READONLY,
  91                                            &mca_btl_self_component.free_list_max);
  92     mca_btl_self_component.free_list_inc = 8;
  93     (void) mca_base_component_var_register(&mca_btl_self_component.super.btl_version, "free_list_inc",
  94                                            "Increment by this number of fragments",
  95                                            MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
  96                                            OPAL_INFO_LVL_9,
  97                                            MCA_BASE_VAR_SCOPE_READONLY,
  98                                            &mca_btl_self_component.free_list_inc);
  99 
 100     mca_btl_self.btl_exclusivity = MCA_BTL_EXCLUSIVITY_HIGH;
 101     mca_btl_self.btl_eager_limit = 1024;
 102     mca_btl_self.btl_rndv_eager_limit = 128 * 1024;
 103     mca_btl_self.btl_max_send_size = 16 * 1024;
 104     mca_btl_self.btl_rdma_pipeline_send_length = INT_MAX;
 105     mca_btl_self.btl_rdma_pipeline_frag_size = INT_MAX;
 106     mca_btl_self.btl_min_rdma_pipeline_size = 0;
 107     mca_btl_self.btl_flags = MCA_BTL_FLAGS_RDMA | MCA_BTL_FLAGS_SEND_INPLACE | MCA_BTL_FLAGS_SEND;
 108     mca_btl_self.btl_bandwidth = 100;
 109     mca_btl_self.btl_latency = 0;
 110     mca_btl_base_param_register (&mca_btl_self_component.super.btl_version, &mca_btl_self);
 111 
 112     return OPAL_SUCCESS;
 113 }
 114 
 115 static int mca_btl_self_component_open(void)
 116 {
 117     /* initialize objects */
 118     OBJ_CONSTRUCT(&mca_btl_self_component.self_frags_eager, opal_free_list_t);
 119     OBJ_CONSTRUCT(&mca_btl_self_component.self_frags_send, opal_free_list_t);
 120     OBJ_CONSTRUCT(&mca_btl_self_component.self_frags_rdma, opal_free_list_t);
 121 
 122     return OPAL_SUCCESS;
 123 }
 124 
 125 
 126 /*
 127  * component cleanup - sanity checking of queue lengths
 128  */
 129 
 130 static int mca_btl_self_component_close(void)
 131 {
 132     OBJ_DESTRUCT(&mca_btl_self_component.self_frags_eager);
 133     OBJ_DESTRUCT(&mca_btl_self_component.self_frags_send);
 134     OBJ_DESTRUCT(&mca_btl_self_component.self_frags_rdma);
 135     return OPAL_SUCCESS;
 136 }
 137 
 138 /*
 139  *  SELF component initialization
 140  */
 141 static mca_btl_base_module_t **mca_btl_self_component_init (int *num_btls,
 142                                                             bool enable_progress_threads,
 143                                                             bool enable_mpi_threads)
 144 {
 145     mca_btl_base_module_t **btls = NULL;
 146     int ret;
 147 
 148     /* initialize free lists */
 149     ret = opal_free_list_init (&mca_btl_self_component.self_frags_eager,
 150                                sizeof (mca_btl_self_frag_eager_t) + mca_btl_self.btl_eager_limit,
 151                                opal_cache_line_size, OBJ_CLASS(mca_btl_self_frag_eager_t), 0,
 152                                opal_cache_line_size, mca_btl_self_component.free_list_num,
 153                                mca_btl_self_component.free_list_max,
 154                                mca_btl_self_component.free_list_inc,
 155                                NULL, 0, NULL, NULL, NULL);
 156     if (OPAL_SUCCESS != ret) {
 157         return NULL;
 158     }
 159 
 160     ret = opal_free_list_init (&mca_btl_self_component.self_frags_send,
 161                                sizeof (mca_btl_self_frag_send_t) + mca_btl_self.btl_max_send_size,
 162                                opal_cache_line_size, OBJ_CLASS(mca_btl_self_frag_send_t), 0,
 163                                opal_cache_line_size, mca_btl_self_component.free_list_num,
 164                                mca_btl_self_component.free_list_max,
 165                                mca_btl_self_component.free_list_inc,
 166                                NULL, 0, NULL, NULL, NULL);
 167     if (OPAL_SUCCESS != ret) {
 168         return NULL;
 169     }
 170 
 171     ret = opal_free_list_init (&mca_btl_self_component.self_frags_rdma,
 172                                sizeof (mca_btl_self_frag_rdma_t) + MCA_BTL_SELF_MAX_INLINE_SIZE,
 173                                opal_cache_line_size, OBJ_CLASS(mca_btl_self_frag_rdma_t), 0,
 174                                opal_cache_line_size, mca_btl_self_component.free_list_num,
 175                                mca_btl_self_component.free_list_max,
 176                                mca_btl_self_component.free_list_inc,
 177                                NULL, 0, NULL, NULL, NULL);
 178     if (OPAL_SUCCESS != ret) {
 179         return NULL;
 180     }
 181 
 182     /* get pointer to the btls */
 183     btls = (mca_btl_base_module_t **) malloc (sizeof (mca_btl_base_module_t *));
 184     if (NULL == btls) {
 185         return NULL;
 186     }
 187 
 188     btls[0] = &mca_btl_self;
 189     *num_btls = 1;
 190 
 191     return btls;
 192 }
 193