This source file includes following definitions.
- mca_coll_hcoll_init_query
- mca_coll_hcoll_module_clear
- mca_coll_hcoll_module_construct
- mca_coll_hcoll_mem_release_cb
- mca_coll_hcoll_module_destruct
- mca_coll_hcoll_save_coll_handlers
- hcoll_comm_attr_del_fn
- mca_coll_hcoll_module_enable
- mca_coll_hcoll_comm_query
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15 #include "ompi_config.h"
16 #include "coll_hcoll.h"
17 #include "coll_hcoll_dtypes.h"
18
19 int hcoll_comm_attr_keyval;
20 int hcoll_type_attr_keyval;
21 mca_coll_hcoll_dtype_t zero_dte_mapping;
22
23
24
25
26 int mca_coll_hcoll_init_query(bool enable_progress_threads, bool enable_mpi_threads)
27 {
28 #if HCOLL_API < HCOLL_VERSION(3,2)
29 if (enable_mpi_threads) {
30 HCOL_VERBOSE(1, "MPI_THREAD_MULTIPLE not suppported; skipping hcoll component");
31 return OMPI_ERROR;
32 }
33 #endif
34 return OMPI_SUCCESS;
35 }
36
37 static void mca_coll_hcoll_module_clear(mca_coll_hcoll_module_t *hcoll_module)
38 {
39 hcoll_module->hcoll_context = NULL;
40 hcoll_module->previous_barrier = NULL;
41 hcoll_module->previous_bcast = NULL;
42 hcoll_module->previous_reduce = NULL;
43 hcoll_module->previous_allreduce = NULL;
44 hcoll_module->previous_allgather = NULL;
45 hcoll_module->previous_allgatherv = NULL;
46 hcoll_module->previous_gather = NULL;
47 hcoll_module->previous_gatherv = NULL;
48 hcoll_module->previous_scatterv = NULL;
49 hcoll_module->previous_alltoall = NULL;
50 hcoll_module->previous_alltoallv = NULL;
51 hcoll_module->previous_alltoallw = NULL;
52 hcoll_module->previous_reduce = NULL;
53 hcoll_module->previous_reduce_scatter = NULL;
54 hcoll_module->previous_ibarrier = NULL;
55 hcoll_module->previous_ibcast = NULL;
56 hcoll_module->previous_iallreduce = NULL;
57 hcoll_module->previous_iallgather = NULL;
58 hcoll_module->previous_iallgatherv = NULL;
59 hcoll_module->previous_igatherv = NULL;
60 hcoll_module->previous_ireduce = NULL;
61 hcoll_module->previous_ialltoall = NULL;
62 hcoll_module->previous_ialltoallv = NULL;
63
64 hcoll_module->previous_barrier_module = NULL;
65 hcoll_module->previous_bcast_module = NULL;
66 hcoll_module->previous_allreduce_module = NULL;
67 hcoll_module->previous_reduce_module = NULL;
68 hcoll_module->previous_allgather_module = NULL;
69 hcoll_module->previous_allgatherv_module = NULL;
70 hcoll_module->previous_gather_module = NULL;
71 hcoll_module->previous_gatherv_module = NULL;
72 hcoll_module->previous_scatterv_module = NULL;
73 hcoll_module->previous_alltoall_module = NULL;
74 hcoll_module->previous_alltoallv_module = NULL;
75 hcoll_module->previous_alltoallw_module = NULL;
76 hcoll_module->previous_reduce_scatter_module = NULL;
77 hcoll_module->previous_ibarrier_module = NULL;
78 hcoll_module->previous_ibcast_module = NULL;
79 hcoll_module->previous_iallreduce_module = NULL;
80 hcoll_module->previous_ireduce_module = NULL;
81 hcoll_module->previous_iallgather_module = NULL;
82 hcoll_module->previous_iallgatherv_module = NULL;
83 hcoll_module->previous_igatherv_module = NULL;
84 hcoll_module->previous_ialltoall_module = NULL;
85 hcoll_module->previous_ialltoallv_module = NULL;
86
87
88 }
89
90 static void mca_coll_hcoll_module_construct(mca_coll_hcoll_module_t *hcoll_module)
91 {
92 mca_coll_hcoll_module_clear(hcoll_module);
93 }
94
95 void mca_coll_hcoll_mem_release_cb(void *buf, size_t length,
96 void *cbdata, bool from_alloc)
97 {
98 hcoll_mem_unmap(buf, length, cbdata, from_alloc);
99 }
100
101 #define OBJ_RELEASE_IF_NOT_NULL( obj ) if( NULL != (obj) ) OBJ_RELEASE( obj );
102
103 static void mca_coll_hcoll_module_destruct(mca_coll_hcoll_module_t *hcoll_module)
104 {
105 int context_destroyed;
106
107 if (hcoll_module->comm == &ompi_mpi_comm_world.comm){
108 if (OMPI_SUCCESS != ompi_attr_free_keyval(COMM_ATTR, &hcoll_comm_attr_keyval, 0)) {
109 HCOL_VERBOSE(1,"hcoll ompi_attr_free_keyval failed");
110 }
111 }
112
113
114
115
116
117
118 if (hcoll_module->hcoll_context != NULL){
119 OBJ_RELEASE_IF_NOT_NULL(hcoll_module->previous_barrier_module);
120 OBJ_RELEASE_IF_NOT_NULL(hcoll_module->previous_bcast_module);
121 OBJ_RELEASE_IF_NOT_NULL(hcoll_module->previous_allreduce_module);
122 OBJ_RELEASE_IF_NOT_NULL(hcoll_module->previous_allgather_module);
123 OBJ_RELEASE_IF_NOT_NULL(hcoll_module->previous_allgatherv_module);
124 OBJ_RELEASE_IF_NOT_NULL(hcoll_module->previous_gatherv_module);
125 OBJ_RELEASE_IF_NOT_NULL(hcoll_module->previous_scatterv_module);
126 OBJ_RELEASE_IF_NOT_NULL(hcoll_module->previous_alltoall_module);
127 OBJ_RELEASE_IF_NOT_NULL(hcoll_module->previous_alltoallv_module);
128 OBJ_RELEASE_IF_NOT_NULL(hcoll_module->previous_reduce_module);
129
130 OBJ_RELEASE_IF_NOT_NULL(hcoll_module->previous_ibarrier_module);
131 OBJ_RELEASE_IF_NOT_NULL(hcoll_module->previous_ibcast_module);
132 OBJ_RELEASE_IF_NOT_NULL(hcoll_module->previous_iallreduce_module);
133 OBJ_RELEASE_IF_NOT_NULL(hcoll_module->previous_iallgather_module);
134 OBJ_RELEASE_IF_NOT_NULL(hcoll_module->previous_iallgatherv_module);
135 OBJ_RELEASE_IF_NOT_NULL(hcoll_module->previous_igatherv_module);
136 OBJ_RELEASE_IF_NOT_NULL(hcoll_module->previous_ialltoall_module);
137 OBJ_RELEASE_IF_NOT_NULL(hcoll_module->previous_ialltoallv_module);
138 OBJ_RELEASE_IF_NOT_NULL(hcoll_module->previous_ireduce_module);
139
140
141
142
143
144
145
146
147
148 #if !defined(HAVE_HCOLL_CONTEXT_FREE)
149 context_destroyed = 0;
150 hcoll_destroy_context(hcoll_module->hcoll_context,
151 (rte_grp_handle_t)hcoll_module->comm,
152 &context_destroyed);
153 #endif
154 }
155 mca_coll_hcoll_module_clear(hcoll_module);
156 }
157
158 #define HCOL_SAVE_PREV_COLL_API(__api) do {\
159 hcoll_module->previous_ ## __api = comm->c_coll->coll_ ## __api;\
160 hcoll_module->previous_ ## __api ## _module = comm->c_coll->coll_ ## __api ## _module;\
161 if (!comm->c_coll->coll_ ## __api || !comm->c_coll->coll_ ## __api ## _module) {\
162 return OMPI_ERROR;\
163 }\
164 OBJ_RETAIN(hcoll_module->previous_ ## __api ## _module);\
165 } while(0)
166
167
168 static int mca_coll_hcoll_save_coll_handlers(mca_coll_hcoll_module_t *hcoll_module)
169 {
170 ompi_communicator_t *comm;
171 comm = hcoll_module->comm;
172
173 HCOL_SAVE_PREV_COLL_API(barrier);
174 HCOL_SAVE_PREV_COLL_API(bcast);
175 HCOL_SAVE_PREV_COLL_API(allreduce);
176 HCOL_SAVE_PREV_COLL_API(reduce);
177 HCOL_SAVE_PREV_COLL_API(allgather);
178 HCOL_SAVE_PREV_COLL_API(allgatherv);
179 HCOL_SAVE_PREV_COLL_API(gatherv);
180 HCOL_SAVE_PREV_COLL_API(scatterv);
181 HCOL_SAVE_PREV_COLL_API(alltoall);
182 HCOL_SAVE_PREV_COLL_API(alltoallv);
183
184 HCOL_SAVE_PREV_COLL_API(ibarrier);
185 HCOL_SAVE_PREV_COLL_API(ibcast);
186 HCOL_SAVE_PREV_COLL_API(iallreduce);
187 HCOL_SAVE_PREV_COLL_API(ireduce);
188 HCOL_SAVE_PREV_COLL_API(iallgather);
189 HCOL_SAVE_PREV_COLL_API(iallgatherv);
190 HCOL_SAVE_PREV_COLL_API(igatherv);
191 HCOL_SAVE_PREV_COLL_API(ialltoall);
192 HCOL_SAVE_PREV_COLL_API(ialltoallv);
193
194
195
196
197
198
199
200
201
202
203 return OMPI_SUCCESS;
204 }
205
206
207
208
209
210
211 static int hcoll_comm_attr_del_fn(MPI_Comm comm, int keyval, void *attr_val, void *extra)
212 {
213
214 mca_coll_hcoll_module_t *hcoll_module;
215 hcoll_module = (mca_coll_hcoll_module_t*) attr_val;
216
217 #ifdef HAVE_HCOLL_CONTEXT_FREE
218 hcoll_context_free(hcoll_module->hcoll_context, (rte_grp_handle_t)comm);
219 #else
220 hcoll_group_destroy_notify(hcoll_module->hcoll_context);
221 #endif
222 return OMPI_SUCCESS;
223
224 }
225
226
227
228 static int mca_coll_hcoll_module_enable(mca_coll_base_module_t *module,
229 struct ompi_communicator_t *comm)
230 {
231 int ret;
232
233 if (OMPI_SUCCESS != mca_coll_hcoll_save_coll_handlers((mca_coll_hcoll_module_t *)module)){
234 HCOL_ERROR("coll_hcol: mca_coll_hcoll_save_coll_handlers failed");
235 return OMPI_ERROR;
236 }
237
238 ret = ompi_attr_set_c(COMM_ATTR, comm, &comm->c_keyhash, hcoll_comm_attr_keyval, (void *)module, false);
239 if (OMPI_SUCCESS != ret) {
240 HCOL_VERBOSE(1,"hcoll ompi_attr_set_c failed");
241 return OMPI_ERROR;
242 }
243
244 return OMPI_SUCCESS;
245 }
246
247
248 OBJ_CLASS_INSTANCE(mca_coll_hcoll_dtype_t,
249 opal_free_list_item_t,
250 NULL,NULL);
251
252
253
254
255
256
257 mca_coll_base_module_t *
258 mca_coll_hcoll_comm_query(struct ompi_communicator_t *comm, int *priority)
259 {
260 mca_coll_base_module_t *module;
261 mca_coll_hcoll_module_t *hcoll_module;
262 ompi_attribute_fn_ptr_union_t del_fn;
263 ompi_attribute_fn_ptr_union_t copy_fn;
264 mca_coll_hcoll_component_t *cm;
265 int err;
266 int rc;
267 cm = &mca_coll_hcoll_component;
268 *priority = 0;
269 module = NULL;
270
271 if (!cm->hcoll_enable){
272 return NULL;
273 }
274
275 if (OMPI_COMM_IS_INTER(comm) || ompi_comm_size(comm) < cm->hcoll_np
276 || ompi_comm_size(comm) < 2){
277 return NULL;
278 }
279
280
281 if (!cm->libhcoll_initialized)
282 {
283
284
285
286
287 opal_progress_register(hcoll_progress_fn);
288
289 HCOL_VERBOSE(10,"Calling hcoll_init();");
290 #if HCOLL_API >= HCOLL_VERSION(3,2)
291 hcoll_read_init_opts(&cm->init_opts);
292 cm->init_opts->base_tag = MCA_COLL_BASE_TAG_HCOLL_BASE;
293 cm->init_opts->max_tag = mca_pml.pml_max_tag;
294 cm->init_opts->enable_thread_support = ompi_mpi_thread_multiple;
295
296 rc = hcoll_init_with_opts(&cm->init_opts);
297 #else
298 hcoll_set_runtime_tag_offset(MCA_COLL_BASE_TAG_HCOLL_BASE, mca_pml.pml_max_tag);
299 rc = hcoll_init();
300 #endif
301
302 if (HCOLL_SUCCESS != rc){
303 cm->hcoll_enable = 0;
304 opal_progress_unregister(hcoll_progress_fn);
305 HCOL_ERROR("Hcol library init failed");
306 return NULL;
307 }
308 #if HCOLL_API >= HCOLL_VERSION(3,2)
309 if (cm->init_opts->mem_hook_needed) {
310 #else
311 if (hcoll_check_mem_release_cb_needed()) {
312 #endif
313 rc = mca_base_framework_open(&opal_memory_base_framework, 0);
314 if (OPAL_SUCCESS != rc) {
315 HCOL_VERBOSE(1, "failed to initialize memory base framework: %d, "
316 "memory hooks will not be used", rc);
317 } else {
318 if ((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) ==
319 ((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) &
320 opal_mem_hooks_support_level())) {
321 HCOL_VERBOSE(1, "using OPAL memory hooks as external events");
322 cm->using_mem_hooks = 1;
323 opal_mem_hooks_register_release(mca_coll_hcoll_mem_release_cb, NULL);
324 setenv("MXM_HCOLL_MEM_ON_DEMAND_MAP", "y", 0);
325 }
326 }
327 } else {
328 cm->using_mem_hooks = 0;
329 }
330 copy_fn.attr_communicator_copy_fn = (MPI_Comm_internal_copy_attr_function*) MPI_COMM_NULL_COPY_FN;
331 del_fn.attr_communicator_delete_fn = hcoll_comm_attr_del_fn;
332 err = ompi_attr_create_keyval(COMM_ATTR, copy_fn, del_fn, &hcoll_comm_attr_keyval, NULL ,0, NULL);
333 if (OMPI_SUCCESS != err) {
334 cm->hcoll_enable = 0;
335 hcoll_finalize();
336 opal_progress_unregister(hcoll_progress_fn);
337 HCOL_ERROR("Hcol comm keyval create failed");
338 return NULL;
339 }
340
341 if (mca_coll_hcoll_component.derived_types_support_enabled) {
342 zero_dte_mapping.type = DTE_ZERO;
343 copy_fn.attr_datatype_copy_fn = (MPI_Type_internal_copy_attr_function *) MPI_TYPE_NULL_COPY_FN;
344 del_fn.attr_datatype_delete_fn = hcoll_type_attr_del_fn;
345 err = ompi_attr_create_keyval(TYPE_ATTR, copy_fn, del_fn, &hcoll_type_attr_keyval, NULL ,0, NULL);
346 if (OMPI_SUCCESS != err) {
347 cm->hcoll_enable = 0;
348 hcoll_finalize();
349 opal_progress_unregister(hcoll_progress_fn);
350 HCOL_ERROR("Hcol type keyval create failed");
351 return NULL;
352 }
353 }
354 OBJ_CONSTRUCT(&cm->dtypes, opal_free_list_t);
355 opal_free_list_init(&cm->dtypes, sizeof(mca_coll_hcoll_dtype_t),
356 8, OBJ_CLASS(mca_coll_hcoll_dtype_t), 0, 0,
357 32, -1, 32, NULL, 0, NULL, NULL, NULL);
358
359 }
360
361 hcoll_module = OBJ_NEW(mca_coll_hcoll_module_t);
362 if (!hcoll_module){
363 if (!cm->libhcoll_initialized) {
364 cm->hcoll_enable = 0;
365 hcoll_finalize();
366 opal_progress_unregister(hcoll_progress_fn);
367 }
368 return NULL;
369 }
370
371 hcoll_module->comm = comm;
372
373 HCOL_VERBOSE(10,"Creating hcoll_context for comm %p, comm_id %d, comm_size %d",
374 (void*)comm,comm->c_contextid,ompi_comm_size(comm));
375
376 hcoll_module->hcoll_context =
377 hcoll_create_context((rte_grp_handle_t)comm);
378
379 if (NULL == hcoll_module->hcoll_context){
380 HCOL_VERBOSE(1,"hcoll_create_context returned NULL");
381 OBJ_RELEASE(hcoll_module);
382 if (!cm->libhcoll_initialized) {
383 cm->hcoll_enable = 0;
384 hcoll_finalize();
385 opal_progress_unregister(hcoll_progress_fn);
386 }
387 return NULL;
388 }
389
390 hcoll_module->super.coll_module_enable = mca_coll_hcoll_module_enable;
391 hcoll_module->super.coll_barrier = hcoll_collectives.coll_barrier ? mca_coll_hcoll_barrier : NULL;
392 hcoll_module->super.coll_bcast = hcoll_collectives.coll_bcast ? mca_coll_hcoll_bcast : NULL;
393 hcoll_module->super.coll_allgather = hcoll_collectives.coll_allgather ? mca_coll_hcoll_allgather : NULL;
394 hcoll_module->super.coll_allgatherv = hcoll_collectives.coll_allgatherv ? mca_coll_hcoll_allgatherv : NULL;
395 hcoll_module->super.coll_allreduce = hcoll_collectives.coll_allreduce ? mca_coll_hcoll_allreduce : NULL;
396 hcoll_module->super.coll_alltoall = hcoll_collectives.coll_alltoall ? mca_coll_hcoll_alltoall : NULL;
397 hcoll_module->super.coll_alltoallv = hcoll_collectives.coll_alltoallv ? mca_coll_hcoll_alltoallv : NULL;
398 hcoll_module->super.coll_gatherv = hcoll_collectives.coll_gatherv ? mca_coll_hcoll_gatherv : NULL;
399 hcoll_module->super.coll_scatterv = hcoll_collectives.coll_scatterv ? mca_coll_hcoll_scatterv : NULL;
400 hcoll_module->super.coll_reduce = hcoll_collectives.coll_reduce ? mca_coll_hcoll_reduce : NULL;
401 hcoll_module->super.coll_ibarrier = hcoll_collectives.coll_ibarrier ? mca_coll_hcoll_ibarrier : NULL;
402 hcoll_module->super.coll_ibcast = hcoll_collectives.coll_ibcast ? mca_coll_hcoll_ibcast : NULL;
403 hcoll_module->super.coll_iallgather = hcoll_collectives.coll_iallgather ? mca_coll_hcoll_iallgather : NULL;
404 #if HCOLL_API >= HCOLL_VERSION(3,5)
405 hcoll_module->super.coll_iallgatherv = hcoll_collectives.coll_iallgatherv ? mca_coll_hcoll_iallgatherv : NULL;
406 #else
407 hcoll_module->super.coll_iallgatherv = NULL;
408 #endif
409 hcoll_module->super.coll_iallreduce = hcoll_collectives.coll_iallreduce ? mca_coll_hcoll_iallreduce : NULL;
410 #if HCOLL_API >= HCOLL_VERSION(3,5)
411 hcoll_module->super.coll_ireduce = hcoll_collectives.coll_ireduce ? mca_coll_hcoll_ireduce : NULL;
412 #else
413 hcoll_module->super.coll_ireduce = NULL;
414 #endif
415 hcoll_module->super.coll_gather = NULL;
416 hcoll_module->super.coll_igatherv = hcoll_collectives.coll_igatherv ? mca_coll_hcoll_igatherv : NULL;
417 hcoll_module->super.coll_ialltoall = NULL;
418 #if HCOLL_API >= HCOLL_VERSION(3,7)
419 hcoll_module->super.coll_ialltoallv = hcoll_collectives.coll_ialltoallv ? mca_coll_hcoll_ialltoallv : NULL;
420 #else
421 hcoll_module->super.coll_ialltoallv = NULL;
422 #endif
423 *priority = cm->hcoll_priority;
424 module = &hcoll_module->super;
425
426 if (!cm->libhcoll_initialized) {
427 cm->libhcoll_initialized = true;
428 }
429
430 return module;
431 }
432
433
434 OBJ_CLASS_INSTANCE(mca_coll_hcoll_module_t,
435 mca_coll_base_module_t,
436 mca_coll_hcoll_module_construct,
437 mca_coll_hcoll_module_destruct);
438
439
440