root/opal/mca/btl/template/btl_template.c

DEFINITIONS

1. mca_btl_template_add_procs
2. mca_btl_template_del_procs
3. mca_btl_template_register
4. mca_btl_template_alloc
5. mca_btl_template_free
6. mca_btl_template_prepare_src
7. mca_btl_template_send
8. mca_btl_template_put
9. mca_btl_template_get
10. mca_btl_template_register_mem
11. mca_btl_template_deregister_mem
12. mca_btl_template_finalize
13. mca_btl_template_ft_event

   1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
   2 /*
   3  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
   4  *                         University Research and Technology
   5  *                         Corporation.  All rights reserved.
   6  * Copyright (c) 2004-2013 The University of Tennessee and The University
   7  *                         of Tennessee Research Foundation.  All rights
   8  *                         reserved.
   9  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
  10  *                         University of Stuttgart.  All rights reserved.
  11  * Copyright (c) 2004-2005 The Regents of the University of California.
  12  *                         All rights reserved.
  13  * Copyright (c) 2014-2015 Los Alamos National Security, LLC. All rights
  14  *                         reserved.
  15  * $COPYRIGHT$
  16  *
  17  * Additional copyrights may follow
  18  *
  19  * $HEADER$
  20  */
  21 
  22 #include "opal_config.h"
  23 #include <string.h>
  24 #include "opal/class/opal_bitmap.h"
  25 #include "opal/mca/btl/btl.h"
  26 #include "opal/datatype/opal_convertor.h"
  27 #include "opal/mca/mpool/base/base.h"
  28 #include "opal/mca/mpool/mpool.h"
  29 
  30 #include "btl_template.h"
  31 #include "btl_template_frag.h"
  32 #include "btl_template_proc.h"
  33 #include "btl_template_endpoint.h"
  34 
  35 
  36 mca_btl_template_module_t mca_btl_template_module = {
  37     .super = {
  38         .btl_component = &mca_btl_template_component.super,
  39         .btl_add_procs = mca_btl_template_add_procs,
  40         .btl_del_procs = mca_btl_template_del_procs,
  41         .btl_register = mca_btl_template_register,
  42         .btl_finalize = mca_btl_template_finalize,
  43         .btl_alloc = mca_btl_template_alloc,
  44         .btl_free = mca_btl_template_free,
  45         .btl_prepare_src = mca_btl_template_prepare_src,
  46         .btl_send = mca_btl_template_send,
  47         .btl_put = mca_btl_template_put,
  48         .btl_get = mca_btl_template_get,
  49         .btl_register_mem = mca_btl_template_register_mem,
  50         .btl_deregister_mem = mca_btl_template_deregister_mem,
  51         .btl_ft_event = mca_btl_template_ft_event
  52     }
  53 };
  54 
  55 /**
  56  *
  57  */
  58 
  59 int mca_btl_template_add_procs(
  60     struct mca_btl_base_module_t* btl,
  61     size_t nprocs,
  62     struct opal_proc_t **opal_procs,
  63     struct mca_btl_base_endpoint_t** peers,
  64     opal_bitmap_t* reachable)
  65 {
  66     mca_btl_template_module_t* template_btl = (mca_btl_template_module_t*)btl;
  67     int i, rc;
  68 
  69     for(i = 0; i < (int) nprocs; i++) {
  70 
  71         struct opal_proc_t* opal_proc = opal_procs[i];
  72         mca_btl_template_proc_t* template_proc;
  73         mca_btl_base_endpoint_t* template_endpoint;
  74 
  75 #if 0
  76         /* If the BTL doesn't support heterogeneous operations, be
  77            sure to say we can't reach that proc */
  78         if (opal_proc_local()->proc_arch != opal_proc->proc_arch) {
  79             continue;
  80         }
  81 #endif
  82 
  83         if(NULL == (template_proc = mca_btl_template_proc_create(opal_proc))) {
  84             return OPAL_ERR_OUT_OF_RESOURCE;
  85         }
  86 
  87         /*
  88          * Check to make sure that the peer has at least as many interface
  89          * addresses exported as we are trying to use. If not, then
  90          * don't bind this BTL instance to the proc.
  91          */
  92 
  93         OPAL_THREAD_LOCK(&template_proc->proc_lock);
  94 
  95         /* The btl_proc datastructure is shared by all TEMPLATE BTL
  96          * instances that are trying to reach this destination.
  97          * Cache the peer instance on the btl_proc.
  98          */
  99         template_endpoint = OBJ_NEW(mca_btl_template_endpoint_t);
 100         if(NULL == template_endpoint) {
 101             OPAL_THREAD_UNLOCK(&template_proc->proc_lock);
 102             return OPAL_ERR_OUT_OF_RESOURCE;
 103         }
 104 
 105         template_endpoint->endpoint_btl = template_btl;
 106         rc = mca_btl_template_proc_insert(template_proc, template_endpoint);
 107         if(rc != OPAL_SUCCESS) {
 108             OBJ_RELEASE(template_endpoint);
 109             OPAL_THREAD_UNLOCK(&template_proc->proc_lock);
 110             continue;
 111         }
 112 
 113         opal_bitmap_set_bit(reachable, i);
 114         OPAL_THREAD_UNLOCK(&template_proc->proc_lock);
 115         peers[i] = template_endpoint;
 116     }
 117 
 118     return OPAL_SUCCESS;
 119 }
 120 
 121 int mca_btl_template_del_procs(struct mca_btl_base_module_t* btl,
 122         size_t nprocs,
 123         struct opal_proc_t **procs,
 124         struct mca_btl_base_endpoint_t ** peers)
 125 {
 126     /* TODO */
 127     return OPAL_SUCCESS;
 128 }
 129 
 130 
 131 /**
 132  * Register callback function to support send/recv semantics
 133  */
 134 
 135 int mca_btl_template_register(
 136                         struct mca_btl_base_module_t* btl,
 137                         mca_btl_base_tag_t tag,
 138                         mca_btl_base_module_recv_cb_fn_t cbfunc,
 139                         void* cbdata)
 140 {
 141     return OPAL_SUCCESS;
 142 }
 143 
 144 
 145 /**
 146  * Allocate a segment.
 147  *
 148  * @param btl (IN)      BTL module
 149  * @param size (IN)     Request segment size.
 150  */
 151 
 152 mca_btl_base_descriptor_t* mca_btl_template_alloc(
 153     struct mca_btl_base_module_t* btl,
 154     struct mca_btl_base_endpoint_t* endpoint,
 155     uint8_t order,
 156     size_t size,
 157     uint32_t flags)
 158 {
 159     mca_btl_template_module_t* template_btl = (mca_btl_template_module_t*) btl;
 160     mca_btl_template_frag_t* frag = NULL;
 161 
 162     if(size <= btl->btl_eager_limit){
 163         MCA_BTL_TEMPLATE_FRAG_ALLOC_EAGER(template_btl, frag);
 164     } else {
 165         MCA_BTL_TEMPLATE_FRAG_ALLOC_MAX(template_btl, frag);
 166     }
 167     if( OPAL_UNLIKELY(NULL != frag) ) {
 168         return NULL;
 169     }
 170 
 171     frag->segment.seg_len = size;
 172     frag->base.des_flags = 0;
 173     return (mca_btl_base_descriptor_t*)frag;
 174 }
 175 
 176 
 177 /**
 178  * Return a segment
 179  */
 180 
 181 int mca_btl_template_free(
 182     struct mca_btl_base_module_t* btl,
 183     mca_btl_base_descriptor_t* des)
 184 {
 185     mca_btl_template_frag_t* frag = (mca_btl_template_frag_t*)des;
 186     if(frag->size == 0) {
 187 #if MCA_BTL_HAS_MPOOL
 188         OBJ_RELEASE(frag->registration);
 189 #endif
 190         MCA_BTL_TEMPLATE_FRAG_RETURN_USER(btl, frag);
 191     } else if(frag->size == btl->btl_eager_limit){
 192         MCA_BTL_TEMPLATE_FRAG_RETURN_EAGER(btl, frag);
 193     } else if(frag->size == btl->btl_max_send_size) {
 194         MCA_BTL_TEMPLATE_FRAG_RETURN_EAGER(btl, frag);
 195     }  else {
 196         return OPAL_ERR_BAD_PARAM;
 197     }
 198     return OPAL_SUCCESS;
 199 }
 200 
 201 /**
 202  * Pack data and return a descriptor that can be
 203  * used for send/put.
 204  *
 205  * @param btl (IN)      BTL module
 206  * @param peer (IN)     BTL peer addressing
 207  */
 208 mca_btl_base_descriptor_t* mca_btl_template_prepare_src(
 209     struct mca_btl_base_module_t* btl,
 210     struct mca_btl_base_endpoint_t* endpoint,
 211     struct opal_convertor_t* convertor,
 212     uint8_t order,
 213     size_t reserve,
 214     size_t* size,
 215     uint32_t flags
 216 )
 217 {
 218     mca_btl_template_frag_t* frag;
 219     struct iovec iov;
 220     uint32_t iov_count = 1;
 221     size_t max_data = *size;
 222     int rc;
 223 
 224 
 225     /*
 226      * if we aren't pinning the data and the requested size is less
 227      * than the eager limit pack into a fragment from the eager pool
 228     */
 229     if (max_data+reserve <= btl->btl_eager_limit) {
 230 
 231         MCA_BTL_TEMPLATE_FRAG_ALLOC_EAGER(btl, frag);
 232         if(OPAL_UNLIKELY(NULL == frag)) {
 233             return NULL;
 234         }
 235 
 236         iov.iov_len = max_data;
 237         iov.iov_base = (unsigned char*) frag->segment.seg_addr.pval + reserve;
 238 
 239         rc = opal_convertor_pack(convertor, &iov, &iov_count, &max_data );
 240         *size  = max_data;
 241         if( rc < 0 ) {
 242             MCA_BTL_TEMPLATE_FRAG_RETURN_EAGER(btl, frag);
 243             return NULL;
 244         }
 245         frag->segment.seg_len = max_data + reserve;
 246     }
 247 
 248     /*
 249      * otherwise pack as much data as we can into a fragment
 250      * that is the max send size.
 251      */
 252     else {
 253 
 254         MCA_BTL_TEMPLATE_FRAG_ALLOC_MAX(btl, frag);
 255         if(OPAL_UNLIKELY(NULL == frag)) {
 256             return NULL;
 257         }
 258         if(max_data + reserve > frag->size){
 259             max_data = frag->size - reserve;
 260         }
 261         iov.iov_len = max_data;
 262         iov.iov_base = (unsigned char*) frag->segment.seg_addr.pval + reserve;
 263 
 264         rc = opal_convertor_pack(convertor, &iov, &iov_count, &max_data );
 265         *size  = max_data;
 266 
 267         if( rc < 0 ) {
 268             MCA_BTL_TEMPLATE_FRAG_RETURN_MAX(btl, frag);
 269             return NULL;
 270         }
 271         frag->segment.seg_len = max_data + reserve;
 272     }
 273 
 274     frag->base.des_segments = &frag->segment;
 275     frag->base.des_segment_count = 1;
 276     frag->base.des_flags = 0;
 277     return &frag->base;
 278 }
 279 
 280 
 281 /**
 282  * Initiate an asynchronous send.
 283  *
 284  * @param btl (IN)         BTL module
 285  * @param endpoint (IN)    BTL addressing information
 286  * @param descriptor (IN)  Description of the data to be transfered
 287  * @param tag (IN)         The tag value used to notify the peer.
 288  */
 289 
 290 int mca_btl_template_send(
 291     struct mca_btl_base_module_t* btl,
 292     struct mca_btl_base_endpoint_t* endpoint,
 293     struct mca_btl_base_descriptor_t* descriptor,
 294     mca_btl_base_tag_t tag)
 295 
 296 {
 297     /* mca_btl_template_module_t* template_btl = (mca_btl_template_module_t*) btl; */
 298     mca_btl_template_frag_t* frag = (mca_btl_template_frag_t*)descriptor;
 299     frag->endpoint = endpoint;
 300     /* TODO */
 301     return OPAL_ERR_NOT_IMPLEMENTED;
 302 }
 303 
 304 
 305 /**
 306  * Initiate an asynchronous put.
 307  *
 308  * @param btl (IN)         BTL module
 309  * @param endpoint (IN)    BTL addressing information
 310  * @param descriptor (IN)  Description of the data to be transferred
 311  */
 312 
 313 int mca_btl_template_put (struct mca_btl_base_module_t *btl,
 314     struct mca_btl_base_endpoint_t *endpoint, void *local_address,
 315     uint64_t remote_address, struct mca_btl_base_registration_handle_t *local_handle,
 316     struct mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
 317     int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
 318 {
 319     /* mca_btl_template_module_t* template_btl = (mca_btl_template_module_t*) btl; */
 320     /* TODO */
 321     return OPAL_ERR_NOT_IMPLEMENTED;
 322 }
 323 
 324 
 325 /**
 326  * Initiate an asynchronous get.
 327  *
 328  * @param btl (IN)         BTL module
 329  * @param endpoint (IN)    BTL addressing information
 330  * @param descriptor (IN)  Description of the data to be transferred
 331  *
 332  */
 333 
 334 int mca_btl_template_get (struct mca_btl_base_module_t *btl,
 335     struct mca_btl_base_endpoint_t *endpoint, void *local_address,
 336     uint64_t remote_address, struct mca_btl_base_registration_handle_t *local_handle,
 337     struct mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
 338     int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
 339 {
 340     /* mca_btl_template_module_t* template_btl = (mca_btl_template_module_t*) btl; */
 341     /* TODO */
 342     return OPAL_ERR_NOT_IMPLEMENTED;
 343 }
 344 
 345 /**
 346  * @brief Register a memory region for put/get/atomic operations.
 347  *
 348  * @param btl (IN)         BTL module
 349  * @param endpoint(IN)     BTL addressing information (or NULL for all endpoints)
 350  * @param base (IN)        Pointer to start of region
 351  * @param size (IN)        Size of region
 352  * @param flags (IN)       Flags indicating what operation will be performed. Valid
 353  *                         values are MCA_BTL_DES_FLAGS_PUT, MCA_BTL_DES_FLAGS_GET,
 354  *                         and MCA_BTL_DES_FLAGS_ATOMIC
 355  *
 356  * @returns a memory registration handle valid for both local and remote operations
 357  * @returns NULL if the region could not be registered
 358  *
 359  * This function registers the specified region with the hardware for use with
 360  * the btl_put, btl_get, btl_atomic_cas, btl_atomic_op, and btl_atomic_fop
 361  * functions. Care should be taken to not hold an excessive number of registrations
 362  * as they may use limited system/NIC resources.
 363  */
 364 struct mca_btl_base_registration_handle_t *mca_btl_template_register_mem (
 365     struct mca_btl_base_module_t* btl, struct mca_btl_base_endpoint_t *endpoint, void *base,
 366     size_t size, uint32_t flags)
 367 {
 368     /* mca_btl_template_module_t* template_btl = (mca_btl_template_module_t*) btl; */
 369     /* TODO */
 370     return NULL;
 371 }
 372 
 373 /**
 374  * @brief Deregister a memory region
 375  *
 376  * @param btl (IN)         BTL module region was registered with
 377  * @param handle (IN)      BTL registration handle to deregister
 378  *
 379  * This function deregisters the memory region associated with the specified handle. Care
 380  * should be taken to not perform any RDMA or atomic operation on this memory region
 381  * after it is deregistered. It is erroneous to specify a memory handle associated with
 382  * a remote node.
 383  */
 384 int mca_btl_template_deregister_mem (struct mca_btl_base_module_t* btl,
 385                                      struct mca_btl_base_registration_handle_t *handle)
 386 {
 387     /* mca_btl_template_module_t* template_btl = (mca_btl_template_module_t*) btl; */
 388     /* TODO */
 389     return OPAL_ERR_NOT_IMPLEMENTED;
 390 }
 391 
 392 
 393 /*
 394  * Cleanup/release module resources.
 395  */
 396 
 397 int mca_btl_template_finalize(struct mca_btl_base_module_t* btl)
 398 {
 399     mca_btl_template_module_t* template_btl = (mca_btl_template_module_t*) btl;
 400     OBJ_DESTRUCT(&template_btl->template_lock);
 401     OBJ_DESTRUCT(&template_btl->template_frag_eager);
 402     OBJ_DESTRUCT(&template_btl->template_frag_max);
 403     OBJ_DESTRUCT(&template_btl->template_frag_user);
 404     free(template_btl);
 405     return OPAL_SUCCESS;
 406 }
 407 
 408 int mca_btl_template_ft_event(int state) {
 409     if(OPAL_CRS_CHECKPOINT == state) {
 410         ;
 411     }
 412     else if(OPAL_CRS_CONTINUE == state) {
 413         ;
 414     }
 415     else if(OPAL_CRS_RESTART == state) {
 416         ;
 417     }
 418     else if(OPAL_CRS_TERM == state ) {
 419         ;
 420     }
 421     else {
 422         ;
 423     }
 424 
 425     return OPAL_SUCCESS;
 426 }