1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ 2 /* 3 * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana 4 * University Research and Technology 5 * Corporation. All rights reserved. 6 * Copyright (c) 2004-2009 The University of Tennessee and The University 7 * of Tennessee Research Foundation. All rights 8 * reserved. 9 * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 10 * University of Stuttgart. All rights reserved. 11 * Copyright (c) 2004-2005 The Regents of the University of California. 12 * All rights reserved. 13 * Copyright (c) 2015 Los Alamos National Security, LLC. All rights 14 * reserved. 15 * $COPYRIGHT$ 16 * 17 * Additional copyrights may follow 18 * 19 * $HEADER$ 20 */ 21 /** 22 * @file 23 */ 24 #ifndef MCA_BTL_TEMPLATE_H 25 #define MCA_BTL_TEMPLATE_H 26 27 #include "opal_config.h" 28 #include <sys/types.h> 29 #include <string.h> 30 31 /* Open MPI includes */ 32 #include "opal/mca/event/event.h" 33 #include "opal/mca/btl/btl.h" 34 #include "opal/mca/btl/base/base.h" 35 #include "opal/mca/mpool/mpool.h" 36 37 BEGIN_C_DECLS 38 39 #define MCA_BTL_HAS_MPOOL 1 40 41 /** 42 * Infiniband (TEMPLATE) BTL component. 43 */ 44 45 struct mca_btl_template_component_t { 46 mca_btl_base_component_3_0_0_t super; /**< base BTL component */ 47 48 uint32_t template_num_btls; 49 /**< number of hcas available to the TEMPLATE component */ 50 51 struct mca_btl_template_module_t *template_btls; 52 /**< array of available BTL modules */ 53 54 int template_free_list_num; 55 /**< initial size of free lists */ 56 57 int template_free_list_max; 58 /**< maximum size of free lists */ 59 60 int template_free_list_inc; 61 /**< number of elements to alloc when growing free lists */ 62 63 opal_list_t template_procs; 64 /**< list of template proc structures */ 65 66 opal_mutex_t template_lock; 67 /**< lock for accessing module state */ 68 69 char* template_mpool_name; 70 /**< name of memory pool */ 71 72 bool leave_pinned; 73 /**< pin memory on first use and leave pinned */ 74 }; 75 typedef struct mca_btl_template_component_t mca_btl_template_component_t; 76 77 OPAL_MODULE_DECLSPEC extern mca_btl_template_component_t mca_btl_template_component; 78 79 /** 80 * BTL Module Interface 81 */ 82 struct mca_btl_template_module_t { 83 mca_btl_base_module_t super; /**< base BTL interface */ 84 85 /* free list of fragment descriptors */ 86 opal_free_list_t template_frag_eager; 87 opal_free_list_t template_frag_max; 88 opal_free_list_t template_frag_user; 89 90 /* lock for accessing module state */ 91 opal_mutex_t template_lock; 92 93 #if MCA_BTL_HAS_MPOOL 94 struct mca_mpool_base_module_t* template_mpool; 95 #endif 96 }; 97 typedef struct mca_btl_template_module_t mca_btl_template_module_t; 98 extern mca_btl_template_module_t mca_btl_template_module; 99 100 101 /** 102 * TEMPLATE component initialization. 103 * 104 * @param num_btl_modules (OUT) Number of BTLs returned in BTL array. 105 * @param allow_multi_user_threads (OUT) Flag indicating wether BTL supports user threads (TRUE) 106 * @param have_hidden_threads (OUT) Flag indicating wether BTL uses threads (TRUE) 107 */ 108 extern mca_btl_base_module_t** mca_btl_template_component_init( 109 int *num_btl_modules, 110 bool allow_multi_user_threads, 111 bool have_hidden_threads 112 ); 113 114 115 /** 116 * TEMPLATE component progress. 117 */ 118 extern int mca_btl_template_component_progress(void); 119 120 121 122 /** 123 * Cleanup any resources held by the BTL. 124 * 125 * @param btl BTL instance. 126 * @return OPAL_SUCCESS or error status on failure. 127 */ 128 129 extern int mca_btl_template_finalize( 130 struct mca_btl_base_module_t* btl 131 ); 132 133 134 /** 135 * PML->BTL notification of change in the process list. 136 * 137 * @param btl (IN) 138 * @param nprocs (IN) Number of processes 139 * @param procs (IN) Set of processes 140 * @param peers (OUT) Set of (optional) peer addressing info. 141 * @param peers (IN/OUT) Set of processes that are reachable via this BTL. 142 * @return OPAL_SUCCESS or error status on failure. 143 * 144 */ 145 146 extern int mca_btl_template_add_procs( 147 struct mca_btl_base_module_t* btl, 148 size_t nprocs, 149 struct opal_proc_t **procs, 150 struct mca_btl_base_endpoint_t** peers, 151 opal_bitmap_t* reachable 152 ); 153 154 /** 155 * PML->BTL notification of change in the process list. 156 * 157 * @param btl (IN) BTL instance 158 * @param nproc (IN) Number of processes. 159 * @param procs (IN) Set of processes. 160 * @param peers (IN) Set of peer data structures. 161 * @return Status indicating if cleanup was successful 162 * 163 */ 164 165 extern int mca_btl_template_del_procs( 166 struct mca_btl_base_module_t* btl, 167 size_t nprocs, 168 struct opal_proc_t **procs, 169 struct mca_btl_base_endpoint_t** peers 170 ); 171 172 173 /** 174 * Initiate an asynchronous send. 175 * 176 * @param btl (IN) BTL module 177 * @param endpoint (IN) BTL addressing information 178 * @param descriptor (IN) Description of the data to be transfered 179 * @param tag (IN) The tag value used to notify the peer. 180 */ 181 182 extern int mca_btl_template_send( 183 struct mca_btl_base_module_t* btl, 184 struct mca_btl_base_endpoint_t* btl_peer, 185 struct mca_btl_base_descriptor_t* descriptor, 186 mca_btl_base_tag_t tag 187 ); 188 189 190 /** 191 * Initiate an asynchronous put. 192 * Completion Semantics: if this function returns a 1 then the operation 193 * is complete. a return of OPAL_SUCCESS indicates 194 * the put operation has been queued with the 195 * network. the local_handle can not be deregistered 196 * until all outstanding operations on that handle 197 * have been completed. 198 * 199 * @param btl (IN) BTL module 200 * @param endpoint (IN) BTL addressing information 201 * @param local_address (IN) Local address to put from (registered) 202 * @param remote_address (IN) Remote address to put to (registered remotely) 203 * @param local_handle (IN) Registration handle for region containing 204 * (local_address, local_address + size) 205 * @param remote_handle (IN) Remote registration handle for region containing 206 * (remote_address, remote_address + size) 207 * @param size (IN) Number of bytes to put 208 * @param flags (IN) Flags for this put operation 209 * @param order (IN) Ordering 210 * @param cbfunc (IN) Function to call on completion (if queued) 211 * @param cbcontext (IN) Context for the callback 212 * @param cbdata (IN) Data for callback 213 * 214 * @retval OPAL_SUCCESS The descriptor was successfully queued for a put 215 * @retval OPAL_ERROR The descriptor was NOT successfully queued for a put 216 * @retval OPAL_ERR_OUT_OF_RESOURCE Insufficient resources to queue the put 217 * operation. Try again later 218 * @retval OPAL_ERR_NOT_AVAILABLE Put can not be performed due to size or 219 * alignment restrictions. 220 */ 221 int mca_btl_template_put (struct mca_btl_base_module_t *btl, 222 struct mca_btl_base_endpoint_t *endpoint, void *local_address, 223 uint64_t remote_address, struct mca_btl_base_registration_handle_t *local_handle, 224 struct mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags, 225 int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata); 226 227 /** 228 * Initiate an asynchronous get. 229 * Completion Semantics: if this function returns a 1 then the operation 230 * is complete. a return of OPAL_SUCCESS indicates 231 * the get operation has been queued with the 232 * network. the local_handle can not be deregistered 233 * until all outstanding operations on that handle 234 * have been completed. 235 * 236 * @param btl (IN) BTL module 237 * @param endpoint (IN) BTL addressing information 238 * @param local_address (IN) Local address to put from (registered) 239 * @param remote_address (IN) Remote address to put to (registered remotely) 240 * @param local_handle (IN) Registration handle for region containing 241 * (local_address, local_address + size) 242 * @param remote_handle (IN) Remote registration handle for region containing 243 * (remote_address, remote_address + size) 244 * @param size (IN) Number of bytes to put 245 * @param flags (IN) Flags for this put operation 246 * @param order (IN) Ordering 247 * @param cbfunc (IN) Function to call on completion (if queued) 248 * @param cbcontext (IN) Context for the callback 249 * @param cbdata (IN) Data for callback 250 * 251 * @retval OPAL_SUCCESS The descriptor was successfully queued for a put 252 * @retval OPAL_ERROR The descriptor was NOT successfully queued for a put 253 * @retval OPAL_ERR_OUT_OF_RESOURCE Insufficient resources to queue the put 254 * operation. Try again later 255 * @retval OPAL_ERR_NOT_AVAILABLE Put can not be performed due to size or 256 * alignment restrictions. 257 */ 258 int mca_btl_template_get (struct mca_btl_base_module_t *btl, 259 struct mca_btl_base_endpoint_t *endpoint, void *local_address, 260 uint64_t remote_address, struct mca_btl_base_registration_handle_t *local_handle, 261 struct mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags, 262 int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata); 263 264 /** 265 * @brief Register a memory region for put/get/atomic operations. 266 * 267 * @param btl (IN) BTL module 268 * @param endpoint(IN) BTL addressing information (or NULL for all endpoints) 269 * @param base (IN) Pointer to start of region 270 * @param size (IN) Size of region 271 * @param flags (IN) Flags indicating what operation will be performed. Valid 272 * values are MCA_BTL_DES_FLAGS_PUT, MCA_BTL_DES_FLAGS_GET, 273 * and MCA_BTL_DES_FLAGS_ATOMIC 274 * 275 * @returns a memory registration handle valid for both local and remote operations 276 * @returns NULL if the region could not be registered 277 * 278 * This function registers the specified region with the hardware for use with 279 * the btl_put, btl_get, btl_atomic_cas, btl_atomic_op, and btl_atomic_fop 280 * functions. Care should be taken to not hold an excessive number of registrations 281 * as they may use limited system/NIC resources. 282 */ 283 struct mca_btl_base_registration_handle_t *mca_btl_template_register_mem ( 284 struct mca_btl_base_module_t* btl, struct mca_btl_base_endpoint_t *endpoint, void *base, 285 size_t size, uint32_t flags); 286 287 /** 288 * @brief Deregister a memory region 289 * 290 * @param btl (IN) BTL module region was registered with 291 * @param handle (IN) BTL registration handle to deregister 292 * 293 * This function deregisters the memory region associated with the specified handle. Care 294 * should be taken to not perform any RDMA or atomic operation on this memory region 295 * after it is deregistered. It is erroneous to specify a memory handle associated with 296 * a remote node. 297 */ 298 int mca_btl_template_deregister_mem (struct mca_btl_base_module_t* btl, 299 struct mca_btl_base_registration_handle_t *handle); 300 301 /** 302 * Register a callback function that is called on receipt 303 * of a fragment. 304 * 305 * @param btl (IN) BTL module 306 * @return Status indicating if registration was successful 307 * 308 */ 309 310 extern int mca_btl_template_register( 311 struct mca_btl_base_module_t* btl, 312 mca_btl_base_tag_t tag, 313 mca_btl_base_module_recv_cb_fn_t cbfunc, 314 void* cbdata); 315 316 /** 317 * Allocate a descriptor with a segment of the requested size. 318 * Note that the BTL layer may choose to return a smaller size 319 * if it cannot support the request. 320 * 321 * @param btl (IN) BTL module 322 * @param size (IN) Request segment size. 323 */ 324 325 extern mca_btl_base_descriptor_t* mca_btl_template_alloc( 326 struct mca_btl_base_module_t* btl, 327 struct mca_btl_base_endpoint_t* endpoint, 328 uint8_t order, 329 size_t size, 330 uint32_t flags); 331 332 333 /** 334 * Return a segment allocated by this BTL. 335 * 336 * @param btl (IN) BTL module 337 * @param descriptor (IN) Allocated descriptor. 338 */ 339 340 extern int mca_btl_template_free( 341 struct mca_btl_base_module_t* btl, 342 mca_btl_base_descriptor_t* des); 343 344 345 /** 346 * Prepare a descriptor for send/rdma using the supplied 347 * convertor. If the convertor references data that is contigous, 348 * the descriptor may simply point to the user buffer. Otherwise, 349 * this routine is responsible for allocating buffer space and 350 * packing if required. 351 * 352 * @param btl (IN) BTL module 353 * @param endpoint (IN) BTL peer addressing 354 * @param convertor (IN) Data type convertor 355 * @param reserve (IN) Additional bytes requested by upper layer to precede user data 356 * @param size (IN/OUT) Number of bytes to prepare (IN), number of bytes actually prepared (OUT) 357 */ 358 359 mca_btl_base_descriptor_t* mca_btl_template_prepare_src( 360 struct mca_btl_base_module_t* btl, 361 struct mca_btl_base_endpoint_t* peer, 362 struct opal_convertor_t* convertor, 363 uint8_t order, 364 size_t reserve, 365 size_t* size, 366 uint32_t flags 367 ); 368 369 /** 370 * Fault Tolerance Event Notification Function 371 * @param state Checkpoint Stae 372 * @return OPAL_SUCCESS or failure status 373 */ 374 int mca_btl_template_ft_event(int state); 375 376 END_C_DECLS 377 #endif