root/opal/mca/btl/vader/btl_vader.h

INCLUDED FROM

DEFINITIONS

1. vader_memmove

   1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
   2 /*
   3  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
   4  *                         University Research and Technology
   5  *                         Corporation.  All rights reserved.
   6  * Copyright (c) 2004-2009 The University of Tennessee and The University
   7  *                         of Tennessee Research Foundation.  All rights
   8  *                         reserved.
   9  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
  10  *                         University of Stuttgart.  All rights reserved.
  11  * Copyright (c) 2004-2005 The Regents of the University of California.
  12  *                         All rights reserved.
  13  * Copyright (c) 2006-2007 Voltaire. All rights reserved.
  14  * Copyright (c) 2009-2010 Cisco Systems, Inc.  All rights reserved.
  15  * Copyright (c) 2010-2018 Los Alamos National Security, LLC. All rights
  16  *                         reserved.
  17  * Copyright (c) 2015      Mellanox Technologies. All rights reserved.
  18  * Copyright (c) 2018      Triad National Security, LLC. All rights
  19  *                         reserved.
  20  *
  21  * $COPYRIGHT$
  22  *
  23  * Additional copyrights may follow
  24  *
  25  * $HEADER$
  26  */
  27 /**
  28  * @file
  29  */
  30 #ifndef MCA_BTL_VADER_H
  31 #define MCA_BTL_VADER_H
  32 
  33 #include "opal_config.h"
  34 
  35 #include <stddef.h>
  36 #include <stdlib.h>
  37 #include <string.h>
  38 
  39 # include <stdint.h>
  40 #ifdef HAVE_SCHED_H
  41 # include <sched.h>
  42 #endif  /* HAVE_SCHED_H */
  43 #ifdef HAVE_UNISTD_H
  44 # include <unistd.h>
  45 #endif /* HAVE_UNISTD_H */
  46 
  47 #include "opal/mca/shmem/base/base.h"
  48 
  49 #include "opal/class/opal_free_list.h"
  50 #include "opal/sys/atomic.h"
  51 #include "opal/mca/btl/btl.h"
  52 #include "opal/mca/rcache/rcache.h"
  53 #include "opal/mca/rcache/base/rcache_base_vma.h"
  54 #include "opal/mca/btl/base/base.h"
  55 #include "opal/mca/rcache/rcache.h"
  56 #include "opal/mca/rcache/base/base.h"
  57 #include "opal/mca/btl/base/btl_base_error.h"
  58 #include "opal/mca/mpool/base/base.h"
  59 #include "opal/util/proc.h"
  60 #include "btl_vader_endpoint.h"
  61 
  62 #include "opal/mca/pmix/pmix.h"
  63 
  64 #include "btl_vader_xpmem.h"
  65 #include "btl_vader_knem.h"
  66 
  67 BEGIN_C_DECLS
  68 
  69 #define min(a,b) ((a) < (b) ? (a) : (b))
  70 
  71 /*
  72  * Shared Memory resource managment
  73  */
  74 
  75 struct vader_fifo_t;
  76 
  77 /*
  78  * Modex data
  79  */
  80 union vader_modex_t {
  81 #if OPAL_BTL_VADER_HAVE_XPMEM
  82     struct vader_modex_xpmem_t {
  83         xpmem_segid_t seg_id;
  84         void *segment_base;
  85     } xpmem;
  86 #endif
  87     opal_shmem_ds_t seg_ds;
  88 };
  89 
  90 /**
  91  * Single copy mechanisms
  92  */
  93 enum {
  94     MCA_BTL_VADER_XPMEM = 0,
  95     MCA_BTL_VADER_CMA   = 1,
  96     MCA_BTL_VADER_KNEM  = 2,
  97     MCA_BTL_VADER_NONE  = 3,
  98     MCA_BTL_VADER_EMUL  = 4,
  99 };
 100 
 101 /**
 102  * Shared Memory (VADER) BTL module.
 103  */
 104 struct mca_btl_vader_component_t {
 105     mca_btl_base_component_3_0_0_t super;   /**< base BTL component */
 106     int vader_free_list_num;                /**< initial size of free lists */
 107     int vader_free_list_max;                /**< maximum size of free lists */
 108     int vader_free_list_inc;                /**< number of elements to alloc when growing free lists */
 109 #if OPAL_BTL_VADER_HAVE_XPMEM
 110     xpmem_segid_t my_seg_id;                /**< this rank's xpmem segment id */
 111     mca_rcache_base_vma_module_t *vma_module; /**< registration cache for xpmem segments */
 112 #endif
 113     opal_shmem_ds_t seg_ds;                 /**< this rank's shared memory segment (when not using xpmem) */
 114 
 115     opal_mutex_t lock;                      /**< lock to protect concurrent updates to this structure's members */
 116     char *my_segment;                       /**< this rank's base pointer */
 117     size_t segment_size;                    /**< size of my_segment */
 118     int32_t num_smp_procs;                  /**< current number of smp procs on this host */
 119     opal_free_list_t vader_frags_eager;     /**< free list of vader send frags */
 120     opal_free_list_t vader_frags_max_send;  /**< free list of vader max send frags (large fragments) */
 121     opal_free_list_t vader_frags_user;      /**< free list of small inline frags */
 122     opal_free_list_t vader_fboxes;          /**< free list of available fast-boxes */
 123 
 124     unsigned int fbox_threshold;            /**< number of sends required before we setup a send fast box for a peer */
 125     unsigned int fbox_max;                  /**< maximum number of send fast boxes to allocate */
 126     unsigned int fbox_size;                 /**< size of each peer fast box allocation */
 127 
 128     int single_copy_mechanism;              /**< single copy mechanism to use */
 129 
 130     int memcpy_limit;                       /**< Limit where we switch from memmove to memcpy */
 131     int log_attach_align;                   /**< Log of the alignment for xpmem segments */
 132     unsigned int max_inline_send;           /**< Limit for copy-in-copy-out fragments */
 133 
 134     mca_btl_base_endpoint_t *endpoints;     /**< array of local endpoints (one for each local peer including myself) */
 135     mca_btl_base_endpoint_t **fbox_in_endpoints; /**< array of fast box in endpoints */
 136     unsigned int num_fbox_in_endpoints;     /**< number of fast boxes to poll */
 137     struct vader_fifo_t *my_fifo;           /**< pointer to the local fifo */
 138 
 139     opal_list_t pending_endpoints;          /**< list of endpoints with pending fragments */
 140     opal_list_t pending_fragments;          /**< fragments pending remote completion */
 141 
 142     char *backing_directory;                /**< directory to place shared memory backing files */
 143 
 144     /* knem stuff */
 145 #if OPAL_BTL_VADER_HAVE_KNEM
 146     unsigned int knem_dma_min;              /**< minimum size to enable DMA for knem transfers (0 disables) */
 147 #endif
 148     mca_mpool_base_module_t *mpool;
 149 };
 150 typedef struct mca_btl_vader_component_t mca_btl_vader_component_t;
 151 OPAL_MODULE_DECLSPEC extern mca_btl_vader_component_t mca_btl_vader_component;
 152 
 153 /**
 154  * VADER BTL Interface
 155  */
 156 struct mca_btl_vader_t {
 157     mca_btl_base_module_t  super;       /**< base BTL interface */
 158     bool btl_inited;  /**< flag indicating if btl has been inited */
 159     mca_btl_base_module_error_cb_fn_t error_cb;
 160 #if OPAL_BTL_VADER_HAVE_KNEM
 161     int knem_fd;
 162 
 163     /* registration cache */
 164     mca_rcache_base_module_t *knem_rcache;
 165 #endif
 166 };
 167 typedef struct mca_btl_vader_t mca_btl_vader_t;
 168 OPAL_MODULE_DECLSPEC extern mca_btl_vader_t mca_btl_vader;
 169 
 170 /* number of peers on the node (not including self) */
 171 #define MCA_BTL_VADER_NUM_LOCAL_PEERS opal_process_info.num_local_peers
 172 
 173 /* local rank in the group */
 174 #define MCA_BTL_VADER_LOCAL_RANK opal_process_info.my_local_rank
 175 
 176 /* memcpy is faster at larger sizes but is undefined if the
 177    pointers are aliased (TODO -- readd alias check) */
 178 static inline void vader_memmove (void *dst, void *src, size_t size)
 179 {
 180     if (size >= (size_t) mca_btl_vader_component.memcpy_limit) {
 181         memcpy (dst, src, size);
 182     } else {
 183         memmove (dst, src, size);
 184     }
 185 }
 186 
 187 /**
 188  * Initiate a send to the peer.
 189  *
 190  * @param btl (IN)      BTL module
 191  * @param peer (IN)     BTL peer addressing
 192  */
 193 int mca_btl_vader_send(struct mca_btl_base_module_t *btl,
 194                        struct mca_btl_base_endpoint_t *endpoint,
 195                        struct mca_btl_base_descriptor_t *descriptor,
 196                        mca_btl_base_tag_t tag);
 197 
 198 /**
 199  * Initiate an inline send to the peer.
 200  *
 201  * @param btl (IN)      BTL module
 202  * @param peer (IN)     BTL peer addressing
 203  */
 204 int mca_btl_vader_sendi (struct mca_btl_base_module_t *btl,
 205                          struct mca_btl_base_endpoint_t *endpoint,
 206                          struct opal_convertor_t *convertor,
 207                          void *header, size_t header_size,
 208                          size_t payload_size, uint8_t order,
 209                          uint32_t flags, mca_btl_base_tag_t tag,
 210                          mca_btl_base_descriptor_t **descriptor);
 211 
 212 /**
 213  * Initiate an synchronous put.
 214  *
 215  * @param btl (IN)         BTL module
 216  * @param endpoint (IN)    BTL addressing information
 217  * @param descriptor (IN)  Description of the data to be transferred
 218  */
 219 #if OPAL_BTL_VADER_HAVE_XPMEM
 220 int mca_btl_vader_put_xpmem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
 221                              uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
 222                              mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
 223                              int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
 224 #endif
 225 
 226 #if OPAL_BTL_VADER_HAVE_CMA
 227 int mca_btl_vader_put_cma (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
 228                            uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
 229                            mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
 230                            int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
 231 #endif
 232 
 233 #if OPAL_BTL_VADER_HAVE_KNEM
 234 int mca_btl_vader_put_knem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
 235                             uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
 236                             mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
 237                             int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
 238 #endif
 239 
 240 int mca_btl_vader_put_sc_emu (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
 241                                uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
 242                                mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
 243                                int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
 244 
 245 /**
 246  * Initiate an synchronous get.
 247  *
 248  * @param btl (IN)         BTL module
 249  * @param endpoint (IN)    BTL addressing information
 250  * @param descriptor (IN)  Description of the data to be transferred
 251  */
 252 #if OPAL_BTL_VADER_HAVE_XPMEM
 253 int mca_btl_vader_get_xpmem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
 254                              uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
 255                              mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
 256                              int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
 257 #endif
 258 
 259 #if OPAL_BTL_VADER_HAVE_CMA
 260 int mca_btl_vader_get_cma (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
 261                            uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
 262                            mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
 263                            int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
 264 #endif
 265 
 266 #if OPAL_BTL_VADER_HAVE_KNEM
 267 int mca_btl_vader_get_knem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
 268                             uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
 269                             mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
 270                             int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
 271 #endif
 272 
 273 int mca_btl_vader_get_sc_emu (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
 274                                uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
 275                                mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
 276                                int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
 277 
 278 int mca_btl_vader_emu_aop (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
 279                            uint64_t remote_address, mca_btl_base_registration_handle_t *remote_handle,
 280                            mca_btl_base_atomic_op_t op, uint64_t operand, int flags, int order,
 281                            mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
 282 
 283 int mca_btl_vader_emu_afop (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
 284                             void *local_address, uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
 285                             mca_btl_base_registration_handle_t *remote_handle, mca_btl_base_atomic_op_t op,
 286                             uint64_t operand, int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc,
 287                             void *cbcontext, void *cbdata);
 288 
 289 int mca_btl_vader_emu_acswap (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
 290                               void *local_address, uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
 291                               mca_btl_base_registration_handle_t *remote_handle, uint64_t compare, uint64_t value, int flags,
 292                               int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
 293 
 294 void mca_btl_vader_sc_emu_init (void);
 295 
 296 /**
 297  * Allocate a segment.
 298  *
 299  * @param btl (IN)      BTL module
 300  * @param size (IN)     Request segment size.
 301  */
 302 mca_btl_base_descriptor_t* mca_btl_vader_alloc (struct mca_btl_base_module_t* btl,
 303                                                 struct mca_btl_base_endpoint_t* endpoint,
 304                                                 uint8_t order, size_t size, uint32_t flags);
 305 
 306 /**
 307  * Return a segment allocated by this BTL.
 308  *
 309  * @param btl (IN)      BTL module
 310  * @param segment (IN)  Allocated segment.
 311  */
 312 int mca_btl_vader_free (struct mca_btl_base_module_t *btl, mca_btl_base_descriptor_t *des);
 313 
 314 
 315 END_C_DECLS
 316 
 317 #endif