opal/mca/btl/uct/btl

/* [<][>][^][v][top][bottom][index][help] */
This source file includes following definitions.
mca_btl_uct_tl_supports_rdma
mca_btl_uct_tl_support_am
mca_btl_uct_tl_supports_conn
mca_btl_uct_tl_requires_connection_tl
   1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
   2 /*
   3  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
   4  *                         University Research and Technology
   5  *                         Corporation.  All rights reserved.
   6  * Copyright (c) 2004-2009 The University of Tennessee and The University
   7  *                         of Tennessee Research Foundation.  All rights
   8  *                         reserved.
   9  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
  10  *                         University of Stuttgart.  All rights reserved.
  11  * Copyright (c) 2004-2005 The Regents of the University of California.
  12  *                         All rights reserved.
  13  * Copyright (c) 2015-2018 Los Alamos National Security, LLC. All rights
  14  *                         reserved.
  15  * $COPYRIGHT$
  16  *
  17  * Additional copyrights may follow
  18  *
  19  * $HEADER$
  20  */
  21 /**
  22  * @file
  23  */
  24 #ifndef MCA_BTL_UCT_H
  25 #define MCA_BTL_UCT_H
  26 
  27 #include "opal_config.h"
  28 #include <sys/types.h>
  29 #include <string.h>
  30 
  31 /* Open MPI includes */
  32 #include "opal/mca/event/event.h"
  33 #include "opal/mca/btl/base/base.h"
  34 #include "opal/mca/mpool/mpool.h"
  35 #include "opal/mca/btl/base/btl_base_error.h"
  36 #include "opal/mca/rcache/base/base.h"
  37 #include "opal/class/opal_fifo.h"
  38 #include "opal/class/opal_hash_table.h"
  39 #include "opal/mca/pmix/pmix.h"
  40 #include "opal/threads/tsd.h"
  41 #include <uct/api/uct.h>
  42 
  43 #include "btl_uct_types.h"
  44 
  45 BEGIN_C_DECLS
  46 
  47 /* detection for old vs new atomic flags */
  48 #if defined(UCT_IFACE_FLAG_ATOMIC_ADD32)
  49 #define OPAL_HAVE_UCT_EP_ATOMIC64_POST 0
  50 #else
  51 #define OPAL_HAVE_UCT_EP_ATOMIC64_POST 1
  52 #endif
  53 
  54 /**
  55  * @brief UCT BTL module
  56  */
  57 struct mca_btl_uct_module_t {
  58     /** base BTL interface */
  59     mca_btl_base_module_t super;
  60 
  61     /** whether the module has been fully initialized or not */
  62     bool initialized;
  63 
  64     /** lock for the hash table */
  65     opal_mutex_t endpoint_lock;
  66 
  67     /** endpoint hash table */
  68     opal_hash_table_t id_to_endpoint;
  69 
  70     /** mutex to protect the module */
  71     opal_recursive_mutex_t lock;
  72 
  73     /** async context */
  74     ucs_async_context_t *ucs_async;
  75 
  76     /** transport for active messaging */
  77     mca_btl_uct_tl_t *am_tl;
  78 
  79     /** transport for RDMA/AMOs */
  80     mca_btl_uct_tl_t *rdma_tl;
  81 
  82     /** transport for forming connections (if needed) */
  83     mca_btl_uct_tl_t *conn_tl;
  84 
  85     /** array containing the am_tl and rdma_tl */
  86     mca_btl_uct_tl_t *comm_tls[2];
  87 
  88     /** registration cache */
  89     mca_rcache_base_module_t *rcache;
  90 
  91     /** name of the memory domain backing this module */
  92     char *md_name;
  93 
  94     /** am and rdma share endpoints */
  95     bool shared_endpoints;
  96 
  97     /** memory domain */
  98     mca_btl_uct_md_t *md;
  99 
 100     /** un-registered frags that will be used with uct_ep_am_short() */
 101     opal_free_list_t short_frags;
 102 
 103     /** registered frags that will be used with uct_ep_am_zcopy() */
 104     opal_free_list_t eager_frags;
 105 
 106     /** large registered frags for packing non-contiguous data */
 107     opal_free_list_t max_frags;
 108 
 109     /** frags that were waiting on connections that are now ready to send */
 110     opal_list_t pending_frags;
 111 
 112     /** pending connection requests */
 113     opal_fifo_t pending_connection_reqs;
 114 };
 115 typedef struct mca_btl_uct_module_t mca_btl_uct_module_t;
 116 
 117 extern mca_btl_uct_module_t mca_btl_uct_module_template;
 118 
 119 /**
 120  * @brief UCT BTL component
 121  */
 122 struct mca_btl_uct_component_t {
 123     /** base BTL component */
 124     mca_btl_base_component_3_0_0_t super;
 125 
 126     /** number of TL modules */
 127     int module_count;
 128 
 129     /** All BTL UCT modules (1 per memory domain) */
 130     mca_btl_uct_module_t *modules[MCA_BTL_UCT_MAX_MODULES];
 131 
 132     /** allowed UCT memory domains */
 133     char *memory_domains;
 134 
 135     /** allowed transports */
 136     char *allowed_transports;
 137 
 138     /** number of worker contexts to create */
 139     int num_contexts_per_module;
 140 
 141 #if OPAL_C_HAVE__THREAD_LOCAL
 142     /** bind threads to contexts */
 143     bool bind_threads_to_contexts;
 144 #endif
 145 
 146     /** disable UCX memory hooks */
 147     bool disable_ucx_memory_hooks;
 148 };
 149 typedef struct mca_btl_uct_component_t mca_btl_uct_component_t;
 150 
 151 OPAL_MODULE_DECLSPEC extern mca_btl_uct_component_t mca_btl_uct_component;
 152 
 153 struct mca_btl_base_registration_handle_t {
 154     /** The packed memory handle. The size of this field is defined by UCT. */
 155     uint8_t packed_handle[1];
 156 };
 157 
 158 struct mca_btl_uct_reg_t {
 159     mca_rcache_base_registration_t base;
 160 
 161     /** UCT memory handle */
 162     uct_mem_h uct_memh;
 163 
 164     /** remote handle */
 165     mca_btl_base_registration_handle_t handle;
 166 };
 167 typedef struct mca_btl_uct_reg_t mca_btl_uct_reg_t;
 168 
 169 OBJ_CLASS_DECLARATION(mca_btl_uct_reg_t);
 170 
 171 #define MCA_BTL_UCT_REG_REMOTE_TO_LOCAL(reg) ((mca_btl_uct_reg_t *)((intptr_t) (reg) - offsetof (mca_btl_uct_reg_t, handle)))
 172 
 173 /**
 174  * Initiate an asynchronous put.
 175  * Completion Semantics: if this function returns a 1 then the operation
 176  *                       is complete. a return of OPAL_SUCCESS indicates
 177  *                       the put operation has been queued with the
 178  *                       network. the local_handle can not be deregistered
 179  *                       until all outstanding operations on that handle
 180  *                       have been completed.
 181  *
 182  * @param btl (IN)            BTL module
 183  * @param endpoint (IN)       BTL addressing information
 184  * @param local_address (IN)  Local address to put from (registered)
 185  * @param remote_address (IN) Remote address to put to (registered remotely)
 186  * @param local_handle (IN)   Registration handle for region containing
 187  *                            (local_address, local_address + size)
 188  * @param remote_handle (IN)  Remote registration handle for region containing
 189  *                            (remote_address, remote_address + size)
 190  * @param size (IN)           Number of bytes to put
 191  * @param flags (IN)          Flags for this put operation
 192  * @param order (IN)          Ordering
 193  * @param cbfunc (IN)         Function to call on completion (if queued)
 194  * @param cbcontext (IN)      Context for the callback
 195  * @param cbdata (IN)         Data for callback
 196  *
 197  * @retval OPAL_SUCCESS    The descriptor was successfully queued for a put
 198  * @retval OPAL_ERROR      The descriptor was NOT successfully queued for a put
 199  * @retval OPAL_ERR_OUT_OF_RESOURCE  Insufficient resources to queue the put
 200  *                         operation. Try again later
 201  * @retval OPAL_ERR_NOT_AVAILABLE  Put can not be performed due to size or
 202  *                         alignment restrictions.
 203  */
 204 int mca_btl_uct_put (struct mca_btl_base_module_t *btl,
 205     struct mca_btl_base_endpoint_t *endpoint, void *local_address,
 206     uint64_t remote_address, struct mca_btl_base_registration_handle_t *local_handle,
 207     struct mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
 208     int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
 209 
 210 /**
 211  * Initiate an asynchronous get.
 212  * Completion Semantics: if this function returns a 1 then the operation
 213  *                       is complete. a return of OPAL_SUCCESS indicates
 214  *                       the get operation has been queued with the
 215  *                       network. the local_handle can not be deregistered
 216  *                       until all outstanding operations on that handle
 217  *                       have been completed.
 218  *
 219  * @param btl (IN)            BTL module
 220  * @param endpoint (IN)       BTL addressing information
 221  * @param local_address (IN)  Local address to put from (registered)
 222  * @param remote_address (IN) Remote address to put to (registered remotely)
 223  * @param local_handle (IN)   Registration handle for region containing
 224  *                            (local_address, local_address + size)
 225  * @param remote_handle (IN)  Remote registration handle for region containing
 226  *                            (remote_address, remote_address + size)
 227  * @param size (IN)           Number of bytes to put
 228  * @param flags (IN)          Flags for this put operation
 229  * @param order (IN)          Ordering
 230  * @param cbfunc (IN)         Function to call on completion (if queued)
 231  * @param cbcontext (IN)      Context for the callback
 232  * @param cbdata (IN)         Data for callback
 233  *
 234  * @retval OPAL_SUCCESS    The descriptor was successfully queued for a put
 235  * @retval OPAL_ERROR      The descriptor was NOT successfully queued for a put
 236  * @retval OPAL_ERR_OUT_OF_RESOURCE  Insufficient resources to queue the put
 237  *                         operation. Try again later
 238  * @retval OPAL_ERR_NOT_AVAILABLE  Put can not be performed due to size or
 239  *                         alignment restrictions.
 240  */
 241 int mca_btl_uct_get (struct mca_btl_base_module_t *btl,
 242     struct mca_btl_base_endpoint_t *endpoint, void *local_address,
 243     uint64_t remote_address, struct mca_btl_base_registration_handle_t *local_handle,
 244     struct mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
 245     int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
 246 
 247  /**
 248   * Fault Tolerance Event Notification Function
 249   * @param state Checkpoint Stae
 250   * @return OPAL_SUCCESS or failure status
 251   */
 252 int mca_btl_uct_ft_event(int state);
 253 
 254 int mca_btl_uct_aop (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
 255                      uint64_t remote_address, mca_btl_base_registration_handle_t *remote_handle,
 256                      mca_btl_base_atomic_op_t op, uint64_t operand, int flags, int order,
 257                      mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
 258 
 259 int mca_btl_uct_afop (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
 260                       void *local_address, uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
 261                       mca_btl_base_registration_handle_t *remote_handle, mca_btl_base_atomic_op_t op,
 262                       uint64_t operand, int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc,
 263                       void *cbcontext, void *cbdata);
 264 
 265 int mca_btl_uct_acswap (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
 266                         void *local_address, uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
 267                         mca_btl_base_registration_handle_t *remote_handle, uint64_t compare, uint64_t value, int flags,
 268                         int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
 269 
 270 
 271 int mca_btl_uct_flush (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint);
 272 int mca_btl_uct_flush_thread (mca_btl_base_module_t *btl);
 273 
 274 int mca_btl_uct_finalize (mca_btl_base_module_t *btl);
 275 
 276 int mca_btl_uct_reg_mem (void *reg_data, void *base, size_t size, mca_rcache_base_registration_t *reg);
 277 int mca_btl_uct_dereg_mem (void *reg_data, mca_rcache_base_registration_t *reg);
 278 
 279 ucs_status_t mca_btl_uct_am_handler (void *arg, void *data, size_t length, unsigned flags);
 280 
 281 struct mca_btl_base_endpoint_t *mca_btl_uct_get_ep (struct mca_btl_base_module_t *module, opal_proc_t *proc);
 282 
 283 int mca_btl_uct_query_tls (mca_btl_uct_module_t *module, mca_btl_uct_md_t *md, uct_tl_resource_desc_t *tl_descs, unsigned tl_count);
 284 int mca_btl_uct_process_connection_request (mca_btl_uct_module_t *module, mca_btl_uct_conn_req_t *req);
 285 
 286 /**
 287  * @brief Checks if a tl is suitable for using for RDMA
 288  *
 289  * @param[in] tl  btl/uct tl pointer
 290  */
 291 static inline bool mca_btl_uct_tl_supports_rdma (mca_btl_uct_tl_t *tl)
 292 {
 293     return (MCA_BTL_UCT_TL_ATTR(tl, 0).cap.flags & (UCT_IFACE_FLAG_PUT_ZCOPY | UCT_IFACE_FLAG_GET_ZCOPY)) ==
 294         (UCT_IFACE_FLAG_PUT_ZCOPY | UCT_IFACE_FLAG_GET_ZCOPY);
 295 }
 296 
 297 /**
 298  * @brief Checks if a tl is suitable for using for active messaging
 299  */
 300 static inline bool mca_btl_uct_tl_support_am (mca_btl_uct_tl_t *tl)
 301 {
 302     return (MCA_BTL_UCT_TL_ATTR(tl, 0).cap.flags & (UCT_IFACE_FLAG_AM_SHORT | UCT_IFACE_FLAG_AM_BCOPY | UCT_IFACE_FLAG_AM_ZCOPY));
 303 }
 304 
 305 /**
 306  * @brief Checks if a tl can be used for passing data to connect endpoints
 307  *
 308  * @param[in] tl  btl/uct tl pointer
 309  */
 310 static inline bool mca_btl_uct_tl_supports_conn (mca_btl_uct_tl_t *tl)
 311 {
 312     return (MCA_BTL_UCT_TL_ATTR(tl, 0).cap.flags & (UCT_IFACE_FLAG_AM_SHORT | UCT_IFACE_FLAG_CONNECT_TO_IFACE)) ==
 313         (UCT_IFACE_FLAG_AM_SHORT | UCT_IFACE_FLAG_CONNECT_TO_IFACE);
 314 }
 315 
 316 /**
 317  * @brief Check if tl endpoints need to be connected via a connection tl
 318  *
 319  * @param[in] tl  btl/uct tl pointer
 320  */
 321 static inline bool mca_btl_uct_tl_requires_connection_tl (mca_btl_uct_tl_t *tl)
 322 {
 323     return !(MCA_BTL_UCT_TL_ATTR(tl, 0).cap.flags & UCT_IFACE_FLAG_CONNECT_TO_IFACE);
 324 }
 325 
 326 END_C_DECLS
 327 #endif
/* [<][>][^][v][top][bottom][index][help] */
root/opal/mca/btl/uct/btl_uct.h

INCLUDED FROM

DEFINITIONS
