This source file includes following definitions.
- ompi_comm_init
- ompi_comm_allocate
- ompi_comm_finalize
- ompi_comm_construct
- ompi_comm_destruct
- OMPI_COMM_SET_INFO_FN
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32 #include "ompi_config.h"
33
34 #include <stdio.h>
35
36 #include "opal/util/bit_ops.h"
37 #include "opal/util/info_subscriber.h"
38 #include "opal/util/string_copy.h"
39 #include "opal/mca/pmix/pmix.h"
40 #include "ompi/constants.h"
41 #include "ompi/mca/pml/pml.h"
42 #include "ompi/mca/coll/base/base.h"
43 #include "ompi/mca/topo/base/base.h"
44 #include "ompi/runtime/params.h"
45 #include "ompi/communicator/communicator.h"
46 #include "ompi/attribute/attribute.h"
47 #include "ompi/dpm/dpm.h"
48 #include "ompi/memchecker.h"
49
50
51
52
53
54
55
56 opal_pointer_array_t ompi_mpi_communicators = {{0}};
57 opal_pointer_array_t ompi_comm_f_to_c_table = {{0}};
58
59 ompi_predefined_communicator_t ompi_mpi_comm_world = {{{{0}}}};
60 ompi_predefined_communicator_t ompi_mpi_comm_self = {{{{0}}}};
61 ompi_predefined_communicator_t ompi_mpi_comm_null = {{{{0}}}};
62 ompi_communicator_t *ompi_mpi_comm_parent = NULL;
63
64 ompi_predefined_communicator_t *ompi_mpi_comm_world_addr =
65 &ompi_mpi_comm_world;
66 ompi_predefined_communicator_t *ompi_mpi_comm_self_addr =
67 &ompi_mpi_comm_self;
68 ompi_predefined_communicator_t *ompi_mpi_comm_null_addr =
69 &ompi_mpi_comm_null;
70
71 static void ompi_comm_construct(ompi_communicator_t* comm);
72 static void ompi_comm_destruct(ompi_communicator_t* comm);
73
74 OBJ_CLASS_INSTANCE(ompi_communicator_t, opal_infosubscriber_t,
75 ompi_comm_construct,
76 ompi_comm_destruct);
77
78
79
80
81 int ompi_comm_num_dyncomm=0;
82
83
84
85
86 int ompi_comm_init(void)
87 {
88 ompi_group_t *group;
89 size_t size;
90
91
92 OBJ_CONSTRUCT(&ompi_mpi_communicators, opal_pointer_array_t);
93 if( OPAL_SUCCESS != opal_pointer_array_init(&ompi_mpi_communicators, 16,
94 OMPI_FORTRAN_HANDLE_MAX, 64) ) {
95 return OMPI_ERROR;
96 }
97
98
99 OBJ_CONSTRUCT(&ompi_comm_f_to_c_table, opal_pointer_array_t);
100 if( OPAL_SUCCESS != opal_pointer_array_init(&ompi_comm_f_to_c_table, 8,
101 OMPI_FORTRAN_HANDLE_MAX, 32) ) {
102 return OMPI_ERROR;
103 }
104
105
106 OBJ_CONSTRUCT(&ompi_mpi_comm_world, ompi_communicator_t);
107 assert(ompi_mpi_comm_world.comm.c_f_to_c_index == 0);
108 group = OBJ_NEW(ompi_group_t);
109
110 size = ompi_process_info.num_procs;
111 group->grp_proc_pointers = (ompi_proc_t **) calloc (size, sizeof (ompi_proc_t *));
112 group->grp_proc_count = size;
113
114 for (size_t i = 0 ; i < size ; ++i) {
115 opal_process_name_t name = {.vpid = i, .jobid = OMPI_PROC_MY_NAME->jobid};
116
117 group->grp_proc_pointers[i] = (ompi_proc_t *) ompi_proc_lookup (name);
118 if (NULL == group->grp_proc_pointers[i]) {
119
120 group->grp_proc_pointers[i] = (ompi_proc_t *) ompi_proc_name_to_sentinel (name);
121 } else {
122 OBJ_RETAIN (group->grp_proc_pointers[i]);
123 }
124 }
125
126 OMPI_GROUP_SET_INTRINSIC (group);
127 OMPI_GROUP_SET_DENSE (group);
128 ompi_set_group_rank(group, ompi_proc_local());
129
130 ompi_mpi_comm_world.comm.c_contextid = 0;
131 ompi_mpi_comm_world.comm.c_id_start_index = 4;
132 ompi_mpi_comm_world.comm.c_id_available = 4;
133 ompi_mpi_comm_world.comm.c_my_rank = group->grp_my_rank;
134 ompi_mpi_comm_world.comm.c_local_group = group;
135 ompi_mpi_comm_world.comm.c_remote_group = group;
136 OBJ_RETAIN(ompi_mpi_comm_world.comm.c_remote_group);
137 ompi_mpi_comm_world.comm.c_cube_dim = opal_cube_dim((int)size);
138 ompi_mpi_comm_world.comm.error_handler = &ompi_mpi_errors_are_fatal.eh;
139 OBJ_RETAIN( &ompi_mpi_errors_are_fatal.eh );
140 OMPI_COMM_SET_PML_ADDED(&ompi_mpi_comm_world.comm);
141 opal_pointer_array_set_item (&ompi_mpi_communicators, 0, &ompi_mpi_comm_world);
142
143 opal_string_copy(ompi_mpi_comm_world.comm.c_name, "MPI_COMM_WORLD",
144 sizeof(ompi_mpi_comm_world.comm.c_name));
145 ompi_mpi_comm_world.comm.c_flags |= OMPI_COMM_NAMEISSET;
146 ompi_mpi_comm_world.comm.c_flags |= OMPI_COMM_INTRINSIC;
147
148
149
150
151
152 ompi_attr_hash_init(&ompi_mpi_comm_world.comm.c_keyhash);
153
154
155
156
157
158
159
160
161 opal_process_name_t wildcard = {OMPI_PROC_MY_NAME->jobid, OPAL_VPID_WILDCARD};
162 char *str=NULL;
163 int rc;
164
165 OPAL_MODEX_RECV_VALUE_OPTIONAL(rc, OPAL_PMIX_MAPBY, &wildcard, &str, OPAL_STRING);
166 if ( 0 == rc && NULL != str) {
167 if ( strstr ( str, "BYNODE") ) {
168 OMPI_COMM_SET_MAPBY_NODE(&ompi_mpi_comm_world.comm);
169 }
170 if (NULL != str) {
171 free(str);
172 }
173 }
174
175 OBJ_CONSTRUCT(&ompi_mpi_comm_self, ompi_communicator_t);
176 assert(ompi_mpi_comm_self.comm.c_f_to_c_index == 1);
177 group = OBJ_NEW(ompi_group_t);
178 group->grp_proc_pointers = ompi_proc_self(&size);
179 group->grp_my_rank = 0;
180 group->grp_proc_count = (int)size;
181 OMPI_GROUP_SET_INTRINSIC (group);
182 OMPI_GROUP_SET_DENSE (group);
183
184 ompi_mpi_comm_self.comm.c_contextid = 1;
185 ompi_mpi_comm_self.comm.c_id_start_index = 20;
186 ompi_mpi_comm_self.comm.c_id_available = 20;
187 ompi_mpi_comm_self.comm.c_my_rank = group->grp_my_rank;
188 ompi_mpi_comm_self.comm.c_local_group = group;
189 ompi_mpi_comm_self.comm.c_remote_group = group;
190 OBJ_RETAIN(ompi_mpi_comm_self.comm.c_remote_group);
191 ompi_mpi_comm_self.comm.error_handler = &ompi_mpi_errors_are_fatal.eh;
192 OBJ_RETAIN( &ompi_mpi_errors_are_fatal.eh );
193 OMPI_COMM_SET_PML_ADDED(&ompi_mpi_comm_self.comm);
194 opal_pointer_array_set_item (&ompi_mpi_communicators, 1, &ompi_mpi_comm_self);
195
196 opal_string_copy(ompi_mpi_comm_self.comm.c_name, "MPI_COMM_SELF",
197 sizeof(ompi_mpi_comm_self.comm.c_name));
198 ompi_mpi_comm_self.comm.c_flags |= OMPI_COMM_NAMEISSET;
199 ompi_mpi_comm_self.comm.c_flags |= OMPI_COMM_INTRINSIC;
200
201
202
203
204 ompi_mpi_comm_self.comm.c_keyhash = NULL;
205
206
207 OBJ_CONSTRUCT(&ompi_mpi_comm_null, ompi_communicator_t);
208 assert(ompi_mpi_comm_null.comm.c_f_to_c_index == 2);
209 ompi_mpi_comm_null.comm.c_local_group = &ompi_mpi_group_null.group;
210 ompi_mpi_comm_null.comm.c_remote_group = &ompi_mpi_group_null.group;
211 OBJ_RETAIN(&ompi_mpi_group_null.group);
212 OBJ_RETAIN(&ompi_mpi_group_null.group);
213
214 ompi_mpi_comm_null.comm.c_contextid = 2;
215 ompi_mpi_comm_null.comm.c_my_rank = MPI_PROC_NULL;
216
217 ompi_mpi_comm_null.comm.error_handler = &ompi_mpi_errors_are_fatal.eh;
218 OBJ_RETAIN( &ompi_mpi_errors_are_fatal.eh );
219 opal_pointer_array_set_item (&ompi_mpi_communicators, 2, &ompi_mpi_comm_null);
220
221 opal_string_copy(ompi_mpi_comm_null.comm.c_name, "MPI_COMM_NULL",
222 sizeof(ompi_mpi_comm_null.comm.c_name));
223 ompi_mpi_comm_null.comm.c_flags |= OMPI_COMM_NAMEISSET;
224 ompi_mpi_comm_null.comm.c_flags |= OMPI_COMM_INTRINSIC;
225
226
227 ompi_mpi_comm_parent = &ompi_mpi_comm_null.comm;
228 OBJ_RETAIN(&ompi_mpi_comm_null);
229 OBJ_RETAIN(&ompi_mpi_group_null.group);
230 OBJ_RETAIN(&ompi_mpi_errors_are_fatal.eh);
231
232
233 ompi_comm_request_init ();
234
235 return OMPI_SUCCESS;
236 }
237
238
239 ompi_communicator_t *ompi_comm_allocate ( int local_size, int remote_size )
240 {
241 ompi_communicator_t *new_comm;
242
243
244 new_comm = OBJ_NEW(ompi_communicator_t);
245 new_comm->super.s_info = NULL;
246 new_comm->c_local_group = ompi_group_allocate ( local_size );
247 if ( 0 < remote_size ) {
248 new_comm->c_remote_group = ompi_group_allocate (remote_size);
249 new_comm->c_flags |= OMPI_COMM_INTER;
250 } else {
251
252
253
254
255 new_comm->c_remote_group = new_comm->c_local_group;
256 OBJ_RETAIN(new_comm->c_remote_group);
257 }
258
259
260 new_comm->c_cube_dim = opal_cube_dim(local_size);
261
262 return new_comm;
263 }
264
265 int ompi_comm_finalize(void)
266 {
267 int max, i;
268 ompi_communicator_t *comm;
269
270
271 OBJ_DESTRUCT( &ompi_mpi_comm_self );
272
273
274 ompi_dpm_dyn_finalize();
275
276
277
278
279 if (NULL != ompi_mpi_comm_world.comm.c_keyhash) {
280
281 (void) ompi_attr_delete_all(COMM_ATTR, &ompi_mpi_comm_world.comm, ompi_mpi_comm_world.comm.c_keyhash);
282 OBJ_RELEASE(ompi_mpi_comm_world.comm.c_keyhash);
283 }
284
285
286 OBJ_DESTRUCT( &ompi_mpi_comm_world );
287
288
289 if( ompi_mpi_comm_parent != &ompi_mpi_comm_null.comm ) {
290
291
292
293
294
295
296 OBJ_DESTRUCT (ompi_mpi_comm_parent);
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315 }
316
317
318 OBJ_DESTRUCT( &ompi_mpi_comm_null );
319
320
321 max = opal_pointer_array_get_size(&ompi_mpi_communicators);
322 for ( i=3; i<max; i++ ) {
323 comm = (ompi_communicator_t *)opal_pointer_array_get_item(&ompi_mpi_communicators, i);
324 if ( NULL != comm ) {
325
326 OBJ_RELEASE(comm);
327 comm=(ompi_communicator_t *)opal_pointer_array_get_item(&ompi_mpi_communicators, i);
328 if ( NULL != comm ) {
329
330 if ( !OMPI_COMM_IS_EXTRA_RETAIN(comm)) {
331
332
333
334
335
336
337
338
339
340
341 if ( ompi_debug_show_handle_leaks && !(OMPI_COMM_IS_FREED(comm)) ){
342 opal_output(0,"WARNING: MPI_Comm still allocated in MPI_Finalize\n");
343 ompi_comm_dump ( comm);
344 OBJ_RELEASE(comm);
345 }
346 }
347 }
348 }
349 }
350
351 OBJ_DESTRUCT (&ompi_mpi_communicators);
352 OBJ_DESTRUCT (&ompi_comm_f_to_c_table);
353
354
355 ompi_comm_request_fini ();
356
357 return OMPI_SUCCESS;
358 }
359
360
361
362
363
364
365 static void ompi_comm_construct(ompi_communicator_t* comm)
366 {
367 comm->c_f_to_c_index = opal_pointer_array_add(&ompi_comm_f_to_c_table, comm);
368 comm->c_name[0] = '\0';
369 comm->c_contextid = MPI_UNDEFINED;
370 comm->c_id_available = MPI_UNDEFINED;
371 comm->c_id_start_index = MPI_UNDEFINED;
372 comm->c_flags = 0;
373 comm->c_my_rank = 0;
374 comm->c_cube_dim = 0;
375 comm->c_local_group = NULL;
376 comm->c_remote_group = NULL;
377 comm->error_handler = NULL;
378 comm->c_pml_comm = NULL;
379 comm->c_topo = NULL;
380 comm->c_coll = NULL;
381
382
383
384 comm->c_keyhash = NULL;
385
386 comm->errhandler_type = OMPI_ERRHANDLER_TYPE_COMM;
387 #ifdef OMPI_WANT_PERUSE
388 comm->c_peruse_handles = NULL;
389 #endif
390 OBJ_CONSTRUCT(&comm->c_lock, opal_mutex_t);
391 }
392
393 static void ompi_comm_destruct(ompi_communicator_t* comm)
394 {
395
396
397
398
399
400
401
402 if ( NULL != comm->c_coll ) {
403 mca_coll_base_comm_unselect(comm);
404 }
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421 if ( MPI_COMM_NULL != comm && OMPI_COMM_IS_PML_ADDED(comm) ) {
422 MCA_PML_CALL(del_comm (comm));
423 }
424
425
426 if (NULL != comm->c_topo) {
427 OBJ_RELEASE(comm->c_topo);
428 comm->c_topo = NULL;
429 }
430
431 if (NULL != comm->c_local_group) {
432 OBJ_RELEASE ( comm->c_local_group );
433 comm->c_local_group = NULL;
434 if ( OMPI_COMM_IS_INTRA(comm) ) {
435
436
437
438 OBJ_RELEASE ( comm->c_remote_group );
439 comm->c_remote_group = NULL;
440 }
441 }
442
443 if (NULL != comm->c_remote_group) {
444 OBJ_RELEASE ( comm->c_remote_group );
445 comm->c_remote_group = NULL;
446 }
447
448 if (NULL != comm->error_handler) {
449 OBJ_RELEASE ( comm->error_handler );
450 comm->error_handler = NULL;
451 }
452
453
454 if ( MPI_UNDEFINED != (int)comm->c_contextid &&
455 NULL != opal_pointer_array_get_item(&ompi_mpi_communicators,
456 comm->c_contextid)) {
457 opal_pointer_array_set_item ( &ompi_mpi_communicators,
458 comm->c_contextid, NULL);
459 }
460
461
462 if ( MPI_UNDEFINED != comm->c_f_to_c_index &&
463 NULL != opal_pointer_array_get_item(&ompi_comm_f_to_c_table,
464 comm->c_f_to_c_index)) {
465 opal_pointer_array_set_item ( &ompi_comm_f_to_c_table,
466 comm->c_f_to_c_index, NULL);
467 }
468
469 OBJ_DESTRUCT(&comm->c_lock);
470 }
471
472 #define OMPI_COMM_SET_INFO_FN(name, flag) \
473 static char *ompi_comm_set_ ## name (opal_infosubscriber_t *obj, char *key, char *value) \
474 { \
475 ompi_communicator_t *comm = (ompi_communicator_t *) obj; \
476 \
477 if (opal_str_to_bool(value)) { \
478 comm->c_assertions |= flag; \
479 } else { \
480 comm->c_assertions &= ~flag; \
481 } \
482 \
483 return OMPI_COMM_CHECK_ASSERT(comm, flag) ? "true" : "false"; \
484 }
485
486 OMPI_COMM_SET_INFO_FN(no_any_source, OMPI_COMM_ASSERT_NO_ANY_SOURCE)
487 OMPI_COMM_SET_INFO_FN(no_any_tag, OMPI_COMM_ASSERT_NO_ANY_TAG)
488 OMPI_COMM_SET_INFO_FN(allow_overtake, OMPI_COMM_ASSERT_ALLOW_OVERTAKE)
489 OMPI_COMM_SET_INFO_FN(exact_length, OMPI_COMM_ASSERT_EXACT_LENGTH)
490
491 void ompi_comm_assert_subscribe (ompi_communicator_t *comm, int32_t assert_flag)
492 {
493 switch (assert_flag) {
494 case OMPI_COMM_ASSERT_NO_ANY_SOURCE:
495 opal_infosubscribe_subscribe (&comm->super, "mpi_assert_no_any_source", "false", ompi_comm_set_no_any_source);
496 break;
497 case OMPI_COMM_ASSERT_NO_ANY_TAG:
498 opal_infosubscribe_subscribe (&comm->super, "mpi_assert_no_any_tag", "false", ompi_comm_set_no_any_tag);
499 break;
500 case OMPI_COMM_ASSERT_ALLOW_OVERTAKE:
501 opal_infosubscribe_subscribe (&comm->super, "mpi_assert_allow_overtaking", "false", ompi_comm_set_allow_overtake);
502 break;
503 case OMPI_COMM_ASSERT_EXACT_LENGTH:
504 opal_infosubscribe_subscribe (&comm->super, "mpi_assert_exact_length", "false", ompi_comm_set_exact_length);
505 break;
506 }
507 }