1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
2 /*
3 * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
4 * University Research and Technology
5 * Corporation. All rights reserved.
6 * Copyright (c) 2004-2009 The University of Tennessee and The University
7 * of Tennessee Research Foundation. All rights
8 * reserved.
9 * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
10 * University of Stuttgart. All rights reserved.
11 * Copyright (c) 2004-2005 The Regents of the University of California.
12 * All rights reserved.
13 * Copyright (c) 2015 Los Alamos National Security, LLC. All rights
14 * reserved.
15 * $COPYRIGHT$
16 *
17 * Additional copyrights may follow
18 *
19 * $HEADER$
20 */
21 /**
22 * @file
23 */
24 #ifndef MCA_BTL_TEMPLATE_H
25 #define MCA_BTL_TEMPLATE_H
26
27 #include "opal_config.h"
28 #include <sys/types.h>
29 #include <string.h>
30
31 /* Open MPI includes */
32 #include "opal/mca/event/event.h"
33 #include "opal/mca/btl/btl.h"
34 #include "opal/mca/btl/base/base.h"
35 #include "opal/mca/mpool/mpool.h"
36
37 BEGIN_C_DECLS
38
39 #define MCA_BTL_HAS_MPOOL 1
40
41 /**
42 * Infiniband (TEMPLATE) BTL component.
43 */
44
45 struct mca_btl_template_component_t {
46 mca_btl_base_component_3_0_0_t super; /**< base BTL component */
47
48 uint32_t template_num_btls;
49 /**< number of hcas available to the TEMPLATE component */
50
51 struct mca_btl_template_module_t *template_btls;
52 /**< array of available BTL modules */
53
54 int template_free_list_num;
55 /**< initial size of free lists */
56
57 int template_free_list_max;
58 /**< maximum size of free lists */
59
60 int template_free_list_inc;
61 /**< number of elements to alloc when growing free lists */
62
63 opal_list_t template_procs;
64 /**< list of template proc structures */
65
66 opal_mutex_t template_lock;
67 /**< lock for accessing module state */
68
69 char* template_mpool_name;
70 /**< name of memory pool */
71
72 bool leave_pinned;
73 /**< pin memory on first use and leave pinned */
74 };
75 typedef struct mca_btl_template_component_t mca_btl_template_component_t;
76
77 OPAL_MODULE_DECLSPEC extern mca_btl_template_component_t mca_btl_template_component;
78
79 /**
80 * BTL Module Interface
81 */
82 struct mca_btl_template_module_t {
83 mca_btl_base_module_t super; /**< base BTL interface */
84
85 /* free list of fragment descriptors */
86 opal_free_list_t template_frag_eager;
87 opal_free_list_t template_frag_max;
88 opal_free_list_t template_frag_user;
89
90 /* lock for accessing module state */
91 opal_mutex_t template_lock;
92
93 #if MCA_BTL_HAS_MPOOL
94 struct mca_mpool_base_module_t* template_mpool;
95 #endif
96 };
97 typedef struct mca_btl_template_module_t mca_btl_template_module_t;
98 extern mca_btl_template_module_t mca_btl_template_module;
99
100
101 /**
102 * TEMPLATE component initialization.
103 *
104 * @param num_btl_modules (OUT) Number of BTLs returned in BTL array.
105 * @param allow_multi_user_threads (OUT) Flag indicating wether BTL supports user threads (TRUE)
106 * @param have_hidden_threads (OUT) Flag indicating wether BTL uses threads (TRUE)
107 */
108 extern mca_btl_base_module_t** mca_btl_template_component_init(
109 int *num_btl_modules,
110 bool allow_multi_user_threads,
111 bool have_hidden_threads
112 );
113
114
115 /**
116 * TEMPLATE component progress.
117 */
118 extern int mca_btl_template_component_progress(void);
119
120
121
122 /**
123 * Cleanup any resources held by the BTL.
124 *
125 * @param btl BTL instance.
126 * @return OPAL_SUCCESS or error status on failure.
127 */
128
129 extern int mca_btl_template_finalize(
130 struct mca_btl_base_module_t* btl
131 );
132
133
134 /**
135 * PML->BTL notification of change in the process list.
136 *
137 * @param btl (IN)
138 * @param nprocs (IN) Number of processes
139 * @param procs (IN) Set of processes
140 * @param peers (OUT) Set of (optional) peer addressing info.
141 * @param peers (IN/OUT) Set of processes that are reachable via this BTL.
142 * @return OPAL_SUCCESS or error status on failure.
143 *
144 */
145
146 extern int mca_btl_template_add_procs(
147 struct mca_btl_base_module_t* btl,
148 size_t nprocs,
149 struct opal_proc_t **procs,
150 struct mca_btl_base_endpoint_t** peers,
151 opal_bitmap_t* reachable
152 );
153
154 /**
155 * PML->BTL notification of change in the process list.
156 *
157 * @param btl (IN) BTL instance
158 * @param nproc (IN) Number of processes.
159 * @param procs (IN) Set of processes.
160 * @param peers (IN) Set of peer data structures.
161 * @return Status indicating if cleanup was successful
162 *
163 */
164
165 extern int mca_btl_template_del_procs(
166 struct mca_btl_base_module_t* btl,
167 size_t nprocs,
168 struct opal_proc_t **procs,
169 struct mca_btl_base_endpoint_t** peers
170 );
171
172
173 /**
174 * Initiate an asynchronous send.
175 *
176 * @param btl (IN) BTL module
177 * @param endpoint (IN) BTL addressing information
178 * @param descriptor (IN) Description of the data to be transfered
179 * @param tag (IN) The tag value used to notify the peer.
180 */
181
182 extern int mca_btl_template_send(
183 struct mca_btl_base_module_t* btl,
184 struct mca_btl_base_endpoint_t* btl_peer,
185 struct mca_btl_base_descriptor_t* descriptor,
186 mca_btl_base_tag_t tag
187 );
188
189
190 /**
191 * Initiate an asynchronous put.
192 * Completion Semantics: if this function returns a 1 then the operation
193 * is complete. a return of OPAL_SUCCESS indicates
194 * the put operation has been queued with the
195 * network. the local_handle can not be deregistered
196 * until all outstanding operations on that handle
197 * have been completed.
198 *
199 * @param btl (IN) BTL module
200 * @param endpoint (IN) BTL addressing information
201 * @param local_address (IN) Local address to put from (registered)
202 * @param remote_address (IN) Remote address to put to (registered remotely)
203 * @param local_handle (IN) Registration handle for region containing
204 * (local_address, local_address + size)
205 * @param remote_handle (IN) Remote registration handle for region containing
206 * (remote_address, remote_address + size)
207 * @param size (IN) Number of bytes to put
208 * @param flags (IN) Flags for this put operation
209 * @param order (IN) Ordering
210 * @param cbfunc (IN) Function to call on completion (if queued)
211 * @param cbcontext (IN) Context for the callback
212 * @param cbdata (IN) Data for callback
213 *
214 * @retval OPAL_SUCCESS The descriptor was successfully queued for a put
215 * @retval OPAL_ERROR The descriptor was NOT successfully queued for a put
216 * @retval OPAL_ERR_OUT_OF_RESOURCE Insufficient resources to queue the put
217 * operation. Try again later
218 * @retval OPAL_ERR_NOT_AVAILABLE Put can not be performed due to size or
219 * alignment restrictions.
220 */
221 int mca_btl_template_put (struct mca_btl_base_module_t *btl,
222 struct mca_btl_base_endpoint_t *endpoint, void *local_address,
223 uint64_t remote_address, struct mca_btl_base_registration_handle_t *local_handle,
224 struct mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
225 int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
226
227 /**
228 * Initiate an asynchronous get.
229 * Completion Semantics: if this function returns a 1 then the operation
230 * is complete. a return of OPAL_SUCCESS indicates
231 * the get operation has been queued with the
232 * network. the local_handle can not be deregistered
233 * until all outstanding operations on that handle
234 * have been completed.
235 *
236 * @param btl (IN) BTL module
237 * @param endpoint (IN) BTL addressing information
238 * @param local_address (IN) Local address to put from (registered)
239 * @param remote_address (IN) Remote address to put to (registered remotely)
240 * @param local_handle (IN) Registration handle for region containing
241 * (local_address, local_address + size)
242 * @param remote_handle (IN) Remote registration handle for region containing
243 * (remote_address, remote_address + size)
244 * @param size (IN) Number of bytes to put
245 * @param flags (IN) Flags for this put operation
246 * @param order (IN) Ordering
247 * @param cbfunc (IN) Function to call on completion (if queued)
248 * @param cbcontext (IN) Context for the callback
249 * @param cbdata (IN) Data for callback
250 *
251 * @retval OPAL_SUCCESS The descriptor was successfully queued for a put
252 * @retval OPAL_ERROR The descriptor was NOT successfully queued for a put
253 * @retval OPAL_ERR_OUT_OF_RESOURCE Insufficient resources to queue the put
254 * operation. Try again later
255 * @retval OPAL_ERR_NOT_AVAILABLE Put can not be performed due to size or
256 * alignment restrictions.
257 */
258 int mca_btl_template_get (struct mca_btl_base_module_t *btl,
259 struct mca_btl_base_endpoint_t *endpoint, void *local_address,
260 uint64_t remote_address, struct mca_btl_base_registration_handle_t *local_handle,
261 struct mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
262 int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
263
264 /**
265 * @brief Register a memory region for put/get/atomic operations.
266 *
267 * @param btl (IN) BTL module
268 * @param endpoint(IN) BTL addressing information (or NULL for all endpoints)
269 * @param base (IN) Pointer to start of region
270 * @param size (IN) Size of region
271 * @param flags (IN) Flags indicating what operation will be performed. Valid
272 * values are MCA_BTL_DES_FLAGS_PUT, MCA_BTL_DES_FLAGS_GET,
273 * and MCA_BTL_DES_FLAGS_ATOMIC
274 *
275 * @returns a memory registration handle valid for both local and remote operations
276 * @returns NULL if the region could not be registered
277 *
278 * This function registers the specified region with the hardware for use with
279 * the btl_put, btl_get, btl_atomic_cas, btl_atomic_op, and btl_atomic_fop
280 * functions. Care should be taken to not hold an excessive number of registrations
281 * as they may use limited system/NIC resources.
282 */
283 struct mca_btl_base_registration_handle_t *mca_btl_template_register_mem (
284 struct mca_btl_base_module_t* btl, struct mca_btl_base_endpoint_t *endpoint, void *base,
285 size_t size, uint32_t flags);
286
287 /**
288 * @brief Deregister a memory region
289 *
290 * @param btl (IN) BTL module region was registered with
291 * @param handle (IN) BTL registration handle to deregister
292 *
293 * This function deregisters the memory region associated with the specified handle. Care
294 * should be taken to not perform any RDMA or atomic operation on this memory region
295 * after it is deregistered. It is erroneous to specify a memory handle associated with
296 * a remote node.
297 */
298 int mca_btl_template_deregister_mem (struct mca_btl_base_module_t* btl,
299 struct mca_btl_base_registration_handle_t *handle);
300
301 /**
302 * Register a callback function that is called on receipt
303 * of a fragment.
304 *
305 * @param btl (IN) BTL module
306 * @return Status indicating if registration was successful
307 *
308 */
309
310 extern int mca_btl_template_register(
311 struct mca_btl_base_module_t* btl,
312 mca_btl_base_tag_t tag,
313 mca_btl_base_module_recv_cb_fn_t cbfunc,
314 void* cbdata);
315
316 /**
317 * Allocate a descriptor with a segment of the requested size.
318 * Note that the BTL layer may choose to return a smaller size
319 * if it cannot support the request.
320 *
321 * @param btl (IN) BTL module
322 * @param size (IN) Request segment size.
323 */
324
325 extern mca_btl_base_descriptor_t* mca_btl_template_alloc(
326 struct mca_btl_base_module_t* btl,
327 struct mca_btl_base_endpoint_t* endpoint,
328 uint8_t order,
329 size_t size,
330 uint32_t flags);
331
332
333 /**
334 * Return a segment allocated by this BTL.
335 *
336 * @param btl (IN) BTL module
337 * @param descriptor (IN) Allocated descriptor.
338 */
339
340 extern int mca_btl_template_free(
341 struct mca_btl_base_module_t* btl,
342 mca_btl_base_descriptor_t* des);
343
344
345 /**
346 * Prepare a descriptor for send/rdma using the supplied
347 * convertor. If the convertor references data that is contigous,
348 * the descriptor may simply point to the user buffer. Otherwise,
349 * this routine is responsible for allocating buffer space and
350 * packing if required.
351 *
352 * @param btl (IN) BTL module
353 * @param endpoint (IN) BTL peer addressing
354 * @param convertor (IN) Data type convertor
355 * @param reserve (IN) Additional bytes requested by upper layer to precede user data
356 * @param size (IN/OUT) Number of bytes to prepare (IN), number of bytes actually prepared (OUT)
357 */
358
359 mca_btl_base_descriptor_t* mca_btl_template_prepare_src(
360 struct mca_btl_base_module_t* btl,
361 struct mca_btl_base_endpoint_t* peer,
362 struct opal_convertor_t* convertor,
363 uint8_t order,
364 size_t reserve,
365 size_t* size,
366 uint32_t flags
367 );
368
369 /**
370 * Fault Tolerance Event Notification Function
371 * @param state Checkpoint Stae
372 * @return OPAL_SUCCESS or failure status
373 */
374 int mca_btl_template_ft_event(int state);
375
376 END_C_DECLS
377 #endif