This source file includes following definitions.
- vader_memmove
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30 #ifndef MCA_BTL_VADER_H
31 #define MCA_BTL_VADER_H
32
33 #include "opal_config.h"
34
35 #include <stddef.h>
36 #include <stdlib.h>
37 #include <string.h>
38
39 # include <stdint.h>
40 #ifdef HAVE_SCHED_H
41 # include <sched.h>
42 #endif
43 #ifdef HAVE_UNISTD_H
44 # include <unistd.h>
45 #endif
46
47 #include "opal/mca/shmem/base/base.h"
48
49 #include "opal/class/opal_free_list.h"
50 #include "opal/sys/atomic.h"
51 #include "opal/mca/btl/btl.h"
52 #include "opal/mca/rcache/rcache.h"
53 #include "opal/mca/rcache/base/rcache_base_vma.h"
54 #include "opal/mca/btl/base/base.h"
55 #include "opal/mca/rcache/rcache.h"
56 #include "opal/mca/rcache/base/base.h"
57 #include "opal/mca/btl/base/btl_base_error.h"
58 #include "opal/mca/mpool/base/base.h"
59 #include "opal/util/proc.h"
60 #include "btl_vader_endpoint.h"
61
62 #include "opal/mca/pmix/pmix.h"
63
64 #include "btl_vader_xpmem.h"
65 #include "btl_vader_knem.h"
66
67 BEGIN_C_DECLS
68
69 #define min(a,b) ((a) < (b) ? (a) : (b))
70
71
72
73
74
75 struct vader_fifo_t;
76
77
78
79
80 union vader_modex_t {
81 #if OPAL_BTL_VADER_HAVE_XPMEM
82 struct vader_modex_xpmem_t {
83 xpmem_segid_t seg_id;
84 void *segment_base;
85 } xpmem;
86 #endif
87 opal_shmem_ds_t seg_ds;
88 };
89
90
91
92
93 enum {
94 MCA_BTL_VADER_XPMEM = 0,
95 MCA_BTL_VADER_CMA = 1,
96 MCA_BTL_VADER_KNEM = 2,
97 MCA_BTL_VADER_NONE = 3,
98 MCA_BTL_VADER_EMUL = 4,
99 };
100
101
102
103
104 struct mca_btl_vader_component_t {
105 mca_btl_base_component_3_0_0_t super;
106 int vader_free_list_num;
107 int vader_free_list_max;
108 int vader_free_list_inc;
109 #if OPAL_BTL_VADER_HAVE_XPMEM
110 xpmem_segid_t my_seg_id;
111 mca_rcache_base_vma_module_t *vma_module;
112 #endif
113 opal_shmem_ds_t seg_ds;
114
115 opal_mutex_t lock;
116 char *my_segment;
117 size_t segment_size;
118 int32_t num_smp_procs;
119 opal_free_list_t vader_frags_eager;
120 opal_free_list_t vader_frags_max_send;
121 opal_free_list_t vader_frags_user;
122 opal_free_list_t vader_fboxes;
123
124 unsigned int fbox_threshold;
125 unsigned int fbox_max;
126 unsigned int fbox_size;
127
128 int single_copy_mechanism;
129
130 int memcpy_limit;
131 int log_attach_align;
132 unsigned int max_inline_send;
133
134 mca_btl_base_endpoint_t *endpoints;
135 mca_btl_base_endpoint_t **fbox_in_endpoints;
136 unsigned int num_fbox_in_endpoints;
137 struct vader_fifo_t *my_fifo;
138
139 opal_list_t pending_endpoints;
140 opal_list_t pending_fragments;
141
142 char *backing_directory;
143
144
145 #if OPAL_BTL_VADER_HAVE_KNEM
146 unsigned int knem_dma_min;
147 #endif
148 mca_mpool_base_module_t *mpool;
149 };
150 typedef struct mca_btl_vader_component_t mca_btl_vader_component_t;
151 OPAL_MODULE_DECLSPEC extern mca_btl_vader_component_t mca_btl_vader_component;
152
153
154
155
156 struct mca_btl_vader_t {
157 mca_btl_base_module_t super;
158 bool btl_inited;
159 mca_btl_base_module_error_cb_fn_t error_cb;
160 #if OPAL_BTL_VADER_HAVE_KNEM
161 int knem_fd;
162
163
164 mca_rcache_base_module_t *knem_rcache;
165 #endif
166 };
167 typedef struct mca_btl_vader_t mca_btl_vader_t;
168 OPAL_MODULE_DECLSPEC extern mca_btl_vader_t mca_btl_vader;
169
170
171 #define MCA_BTL_VADER_NUM_LOCAL_PEERS opal_process_info.num_local_peers
172
173
174 #define MCA_BTL_VADER_LOCAL_RANK opal_process_info.my_local_rank
175
176
177
178 static inline void vader_memmove (void *dst, void *src, size_t size)
179 {
180 if (size >= (size_t) mca_btl_vader_component.memcpy_limit) {
181 memcpy (dst, src, size);
182 } else {
183 memmove (dst, src, size);
184 }
185 }
186
187
188
189
190
191
192
193 int mca_btl_vader_send(struct mca_btl_base_module_t *btl,
194 struct mca_btl_base_endpoint_t *endpoint,
195 struct mca_btl_base_descriptor_t *descriptor,
196 mca_btl_base_tag_t tag);
197
198
199
200
201
202
203
204 int mca_btl_vader_sendi (struct mca_btl_base_module_t *btl,
205 struct mca_btl_base_endpoint_t *endpoint,
206 struct opal_convertor_t *convertor,
207 void *header, size_t header_size,
208 size_t payload_size, uint8_t order,
209 uint32_t flags, mca_btl_base_tag_t tag,
210 mca_btl_base_descriptor_t **descriptor);
211
212
213
214
215
216
217
218
219 #if OPAL_BTL_VADER_HAVE_XPMEM
220 int mca_btl_vader_put_xpmem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
221 uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
222 mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
223 int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
224 #endif
225
226 #if OPAL_BTL_VADER_HAVE_CMA
227 int mca_btl_vader_put_cma (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
228 uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
229 mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
230 int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
231 #endif
232
233 #if OPAL_BTL_VADER_HAVE_KNEM
234 int mca_btl_vader_put_knem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
235 uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
236 mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
237 int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
238 #endif
239
240 int mca_btl_vader_put_sc_emu (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
241 uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
242 mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
243 int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
244
245
246
247
248
249
250
251
252 #if OPAL_BTL_VADER_HAVE_XPMEM
253 int mca_btl_vader_get_xpmem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
254 uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
255 mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
256 int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
257 #endif
258
259 #if OPAL_BTL_VADER_HAVE_CMA
260 int mca_btl_vader_get_cma (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
261 uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
262 mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
263 int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
264 #endif
265
266 #if OPAL_BTL_VADER_HAVE_KNEM
267 int mca_btl_vader_get_knem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
268 uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
269 mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
270 int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
271 #endif
272
273 int mca_btl_vader_get_sc_emu (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
274 uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
275 mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
276 int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
277
278 int mca_btl_vader_emu_aop (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
279 uint64_t remote_address, mca_btl_base_registration_handle_t *remote_handle,
280 mca_btl_base_atomic_op_t op, uint64_t operand, int flags, int order,
281 mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
282
283 int mca_btl_vader_emu_afop (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
284 void *local_address, uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
285 mca_btl_base_registration_handle_t *remote_handle, mca_btl_base_atomic_op_t op,
286 uint64_t operand, int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc,
287 void *cbcontext, void *cbdata);
288
289 int mca_btl_vader_emu_acswap (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
290 void *local_address, uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
291 mca_btl_base_registration_handle_t *remote_handle, uint64_t compare, uint64_t value, int flags,
292 int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
293
294 void mca_btl_vader_sc_emu_init (void);
295
296
297
298
299
300
301
302 mca_btl_base_descriptor_t* mca_btl_vader_alloc (struct mca_btl_base_module_t* btl,
303 struct mca_btl_base_endpoint_t* endpoint,
304 uint8_t order, size_t size, uint32_t flags);
305
306
307
308
309
310
311
312 int mca_btl_vader_free (struct mca_btl_base_module_t *btl, mca_btl_base_descriptor_t *des);
313
314
315 END_C_DECLS
316
317 #endif