1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
2 /*
3 * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
4 * University Research and Technology
5 * Corporation. All rights reserved.
6 * Copyright (c) 2004-2017 The University of Tennessee and The University
7 * of Tennessee Research Foundation. All rights
8 * reserved.
9 * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
10 * University of Stuttgart. All rights reserved.
11 * Copyright (c) 2004-2005 The Regents of the University of California.
12 * All rights reserved.
13 * Copyright (c) 2006-2017 Cisco Systems, Inc. All rights reserved
14 * Copyright (c) 2006-2017 University of Houston. All rights reserved.
15 * Copyright (c) 2009 Sun Microsystems, Inc. All rights reserved.
16 * Copyright (c) 2011-2013 Inria. All rights reserved.
17 * Copyright (c) 2011-2013 Universite Bordeaux 1
18 * Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights
19 * reserved.
20 * Copyright (c) 2014-2015 Intel, Inc. All rights reserved.
21 * Copyright (c) 2015 Research Organization for Information Science
22 * and Technology (RIST). All rights reserved.
23 * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
24 * $COPYRIGHT$
25 *
26 * Additional copyrights may follow
27 *
28 * $HEADER$
29 */
30
31 #ifndef OMPI_COMMUNICATOR_H
32 #define OMPI_COMMUNICATOR_H
33
34 #include "ompi_config.h"
35 #include "opal/class/opal_object.h"
36 #include "opal/class/opal_hash_table.h"
37 #include "opal/util/info_subscriber.h"
38 #include "ompi/errhandler/errhandler.h"
39 #include "opal/threads/mutex.h"
40 #include "ompi/communicator/comm_request.h"
41
42 #include "mpi.h"
43 #include "ompi/group/group.h"
44 #include "ompi/mca/coll/coll.h"
45 #include "ompi/info/info.h"
46 #include "ompi/proc/proc.h"
47
48 BEGIN_C_DECLS
49
50 OMPI_DECLSPEC OBJ_CLASS_DECLARATION(ompi_communicator_t);
51
52 #define OMPI_COMM_INTER 0x00000001
53 #define OMPI_COMM_NAMEISSET 0x00000002
54 #define OMPI_COMM_INTRINSIC 0x00000004
55 #define OMPI_COMM_DYNAMIC 0x00000008
56 #define OMPI_COMM_ISFREED 0x00000010
57 #define OMPI_COMM_INVALID 0x00000020
58 #define OMPI_COMM_CART 0x00000100
59 #define OMPI_COMM_GRAPH 0x00000200
60 #define OMPI_COMM_DIST_GRAPH 0x00000400
61 #define OMPI_COMM_PML_ADDED 0x00001000
62 #define OMPI_COMM_EXTRA_RETAIN 0x00004000
63 #define OMPI_COMM_MAPBY_NODE 0x00008000
64
65 /* some utility #defines */
66 #define OMPI_COMM_IS_INTER(comm) ((comm)->c_flags & OMPI_COMM_INTER)
67 #define OMPI_COMM_IS_INTRA(comm) (!((comm)->c_flags & OMPI_COMM_INTER))
68 #define OMPI_COMM_IS_CART(comm) ((comm)->c_flags & OMPI_COMM_CART)
69 #define OMPI_COMM_IS_GRAPH(comm) ((comm)->c_flags & OMPI_COMM_GRAPH)
70 #define OMPI_COMM_IS_DIST_GRAPH(comm) ((comm)->c_flags & OMPI_COMM_DIST_GRAPH)
71 #define OMPI_COMM_IS_INTRINSIC(comm) ((comm)->c_flags & OMPI_COMM_INTRINSIC)
72 #define OMPI_COMM_IS_FREED(comm) ((comm)->c_flags & OMPI_COMM_ISFREED)
73 #define OMPI_COMM_IS_DYNAMIC(comm) ((comm)->c_flags & OMPI_COMM_DYNAMIC)
74 #define OMPI_COMM_IS_INVALID(comm) ((comm)->c_flags & OMPI_COMM_INVALID)
75 #define OMPI_COMM_IS_PML_ADDED(comm) ((comm)->c_flags & OMPI_COMM_PML_ADDED)
76 #define OMPI_COMM_IS_EXTRA_RETAIN(comm) ((comm)->c_flags & OMPI_COMM_EXTRA_RETAIN)
77 #define OMPI_COMM_IS_TOPO(comm) (OMPI_COMM_IS_CART((comm)) || \
78 OMPI_COMM_IS_GRAPH((comm)) || \
79 OMPI_COMM_IS_DIST_GRAPH((comm)))
80 #define OMPI_COMM_IS_MAPBY_NODE(comm) ((comm)->c_flags & OMPI_COMM_MAPBY_NODE)
81
82 #define OMPI_COMM_SET_DYNAMIC(comm) ((comm)->c_flags |= OMPI_COMM_DYNAMIC)
83 #define OMPI_COMM_SET_INVALID(comm) ((comm)->c_flags |= OMPI_COMM_INVALID)
84
85 #define OMPI_COMM_SET_PML_ADDED(comm) ((comm)->c_flags |= OMPI_COMM_PML_ADDED)
86 #define OMPI_COMM_SET_EXTRA_RETAIN(comm) ((comm)->c_flags |= OMPI_COMM_EXTRA_RETAIN)
87 #define OMPI_COMM_SET_MAPBY_NODE(comm) ((comm)->c_flags |= OMPI_COMM_MAPBY_NODE)
88
89 /* a set of special tags: */
90
91 /* to recognize an MPI_Comm_join in the comm_connect_accept routine. */
92 #define OMPI_COMM_ALLGATHER_TAG -31078
93 #define OMPI_COMM_BARRIER_TAG -31079
94 #define OMPI_COMM_ALLREDUCE_TAG -31080
95
96 #define OMPI_COMM_ASSERT_NO_ANY_TAG 0x00000001
97 #define OMPI_COMM_ASSERT_NO_ANY_SOURCE 0x00000002
98 #define OMPI_COMM_ASSERT_EXACT_LENGTH 0x00000004
99 #define OMPI_COMM_ASSERT_ALLOW_OVERTAKE 0x00000008
100
101 #define OMPI_COMM_CHECK_ASSERT(comm, flag) !!((comm)->c_assertions & flag)
102 #define OMPI_COMM_CHECK_ASSERT_NO_ANY_TAG(comm) OMPI_COMM_CHECK_ASSERT(comm, OMPI_COMM_ASSERT_NO_ANY_TAG)
103 #define OMPI_COMM_CHECK_ASSERT_NO_ANY_SOURCE(comm) OMPI_COMM_CHECK_ASSERT(comm, OMPI_COMM_ASSERT_NO_ANY_SOURCE)
104 #define OMPI_COMM_CHECK_ASSERT_EXACT_LENGTH(comm) OMPI_COMM_CHECK_ASSERT(comm, OMPI_COMM_ASSERT_EXACT_LENGTH)
105 #define OMPI_COMM_CHECK_ASSERT_ALLOW_OVERTAKE(comm) OMPI_COMM_CHECK_ASSERT(comm, OMPI_COMM_ASSERT_ALLOW_OVERTAKE)
106
107 /**
108 * Modes required for acquiring the new comm-id.
109 * The first (INTER/INTRA) indicates whether the
110 * input comm was an inter/intra-comm, the second
111 * whether the new communicator will be an inter/intra
112 * comm
113 */
114 #define OMPI_COMM_CID_INTRA 0x00000020
115 #define OMPI_COMM_CID_INTER 0x00000040
116 #define OMPI_COMM_CID_INTRA_BRIDGE 0x00000080
117 #define OMPI_COMM_CID_INTRA_PMIX 0x00000100
118 #define OMPI_COMM_CID_GROUP 0x00000200
119
120 /**
121 * The block of CIDs allocated for MPI_COMM_WORLD
122 * and other communicators
123 */
124 #define OMPI_COMM_BLOCK_WORLD 16
125 #define OMPI_COMM_BLOCK_OTHERS 8
126
127 /* A macro comparing two CIDs */
128 #define OMPI_COMM_CID_IS_LOWER(comm1,comm2) ( ((comm1)->c_contextid < (comm2)->c_contextid)? 1:0)
129
130
131 OMPI_DECLSPEC extern opal_pointer_array_t ompi_mpi_communicators;
132 OMPI_DECLSPEC extern opal_pointer_array_t ompi_comm_f_to_c_table;
133
134 struct ompi_communicator_t {
135 opal_infosubscriber_t super;
136 opal_mutex_t c_lock; /* mutex for name and potentially
137 attributes */
138 char c_name[MPI_MAX_OBJECT_NAME];
139 uint32_t c_contextid;
140 int c_my_rank;
141 uint32_t c_flags; /* flags, e.g. intercomm,
142 topology, etc. */
143 uint32_t c_assertions; /* info assertions */
144
145 int c_id_available; /* the currently available Cid for allocation
146 to a child*/
147 int c_id_start_index; /* the starting index of the block of cids
148 allocated to this communicator*/
149
150 ompi_group_t *c_local_group;
151 ompi_group_t *c_remote_group;
152
153 struct ompi_communicator_t *c_local_comm; /* a duplicate of the
154 local communicator in
155 case the comm is an
156 inter-comm*/
157
158 /* Attributes */
159 struct opal_hash_table_t *c_keyhash;
160
161 /**< inscribing cube dimension */
162 int c_cube_dim;
163
164 /* Standard information about the selected topology module (or NULL
165 if this is not a cart, graph or dist graph communicator) */
166 struct mca_topo_base_module_t* c_topo;
167
168 /* index in Fortran <-> C translation array */
169 int c_f_to_c_index;
170
171 #ifdef OMPI_WANT_PERUSE
172 /*
173 * Place holder for the PERUSE events.
174 */
175 struct ompi_peruse_handle_t** c_peruse_handles;
176 #endif
177
178 /* Error handling. This field does not have the "c_" prefix so
179 that the OMPI_ERRHDL_* macros can find it, regardless of whether
180 it's a comm, window, or file. */
181
182 ompi_errhandler_t *error_handler;
183 ompi_errhandler_type_t errhandler_type;
184
185 /* Hooks for PML to hang things */
186 struct mca_pml_comm_t *c_pml_comm;
187
188 /* Collectives module interface and data */
189 mca_coll_base_comm_coll_t *c_coll;
190 };
191 typedef struct ompi_communicator_t ompi_communicator_t;
192
193 /**
194 * Padded struct to maintain back compatibiltiy.
195 *
196 * The following ompi_predefined_xxx_t structure is used to maintain
197 * backwards binary compatibility for MPI applications compiled
198 * against one version of OMPI library but dynamically linked at
199 * runtime with another. The issue is between versions the actual
200 * structure may change in size (even between debug and optimized
201 * compilation -- the structure contents change, and therefore the
202 * overall size changes).
203 *
204 * This is problematic with predefined handles because the storage of
205 * the structure ends up being located to an application's BSS. This
206 * causes problems because if one version has the predefined as size X
207 * and then the application is dynamically linked with a version that
208 * has a size of Y (where X != Y) then the application will
209 * unintentionally overrun the memory initially allocated for the
210 * structure.
211 *
212 * The solution we are using below creates a parent structure
213 * (ompi_predefined_xxx_t) that contains the base structure
214 * (ompi_xxx_t) followed by a character padding that is the size of
215 * the total size we choose to preallocate for the structure minus the
216 * amount used by the base structure. In this way, we've normalized
217 * the size of each predefined handle across multiple versions and
218 * configurations of Open MPI (e.g., MPI_COMM_WORLD will refer to a
219 * back-end struct that is X bytes long, even if we change the
220 * back-end ompi_communicator_t between version A.B and version C.D in
221 * Open MPI). When we come close to filling up the the padding we can
222 * add a pointer at the back end of the base structure to point to an
223 * extension of the type. Or we can just increase the padding and
224 * break backwards binary compatibility.
225 *
226 * The above method was decided after several failed attempts
227 * described below.
228 *
229 * - Original implementation - suffered that the base structure seemed
230 * to always change in size between Open MPI versions and/or
231 * configurations (e.g., optimized vs. debugging build).
232 *
233 * - Convert all predefined handles to run-time-assigned pointers
234 * (i.e., global variables) - This worked except in cases where an MPI
235 * application wanted to assign the predefined handle value to a
236 * global variable -- we could not guarantee to have the global
237 * variable filled until MPI_INIT was called (recall that MPI
238 * predefined handles must be assignable before MPI_INIT; e.g.,
239 * "MPI_Comm foo = MPI_COMM_WORLD").
240 *
241 * - union of struct and padding - Similar to current implementation
242 * except using a union for the parent. This worked except in cases
243 * where the compilers did not support C99 union static initalizers.
244 * It would have been a pain to convert a bunch of the code to use
245 * non-static initializers (e.g., MPI datatypes).
246 */
247
248 /* Define for the preallocated size of the predefined handle.
249 * Note that we are using a pointer type as the base memory chunk
250 * size so when the bitness changes the size of the handle changes.
251 * This is done so we don't end up needing a structure that is
252 * incredibly larger than necessary because of the bitness.
253 *
254 * This padding mechanism works as a (likely) compile time check for when the
255 * size of the ompi_communicator_t exceeds the predetermined size of the
256 * ompi_predefined_communicator_t. It also allows us to change the size of
257 * the ompi_communicator_t without impacting the size of the
258 * ompi_predefined_communicator_t structure for some number of additions.
259 *
260 * Note: we used to define the PAD as a multiple of sizeof(void*).
261 * However, this makes a different size PAD, depending on
262 * sizeof(void*). In some cases
263 * (https://github.com/open-mpi/ompi/issues/3610), 32 bit builds can
264 * run out of space when 64 bit builds are still ok. So we changed to
265 * use just a naked byte size. As a rule of thumb, however, the size
266 * should probably still be a multiple of 8 so that it has the
267 * possibility of being nicely aligned.
268 *
269 * As an example:
270 * If the size of ompi_communicator_t is less than the size of the _PAD then
271 * the _PAD ensures that the size of the ompi_predefined_communicator_t is
272 * whatever size is defined below in the _PAD macro.
273 * However, if the size of the ompi_communicator_t grows larger than the _PAD
274 * (say by adding a few more function pointers to the structure) then the
275 * 'padding' variable will be initialized to a large number often triggering
276 * a 'array is too large' compile time error. This signals two things:
277 * 1) That the _PAD should be increased.
278 * 2) That users need to be made aware of the size change for the
279 * ompi_predefined_communicator_t structure.
280 *
281 * Q: So you just made a change to communicator structure, do you need to adjust
282 * the PREDEFINED_COMMUNICATOR_PAD macro?
283 * A: Most likely not, but it would be good to check.
284 */
285 #define PREDEFINED_COMMUNICATOR_PAD 512
286
287 struct ompi_predefined_communicator_t {
288 struct ompi_communicator_t comm;
289 char padding[PREDEFINED_COMMUNICATOR_PAD - sizeof(ompi_communicator_t)];
290 };
291 typedef struct ompi_predefined_communicator_t ompi_predefined_communicator_t;
292
293 OMPI_DECLSPEC extern ompi_communicator_t *ompi_mpi_comm_parent;
294 OMPI_DECLSPEC extern ompi_predefined_communicator_t ompi_mpi_comm_world;
295 OMPI_DECLSPEC extern ompi_predefined_communicator_t ompi_mpi_comm_self;
296 OMPI_DECLSPEC extern ompi_predefined_communicator_t ompi_mpi_comm_null;
297
298 /*
299 * These variables are for the MPI F03 bindings (F03 must bind Fortran
300 * varaiables to symbols; it cannot bind Fortran variables to the
301 * address of a C variable).
302 */
303 OMPI_DECLSPEC extern ompi_predefined_communicator_t *ompi_mpi_comm_world_addr;
304 OMPI_DECLSPEC extern ompi_predefined_communicator_t *ompi_mpi_comm_self_addr;
305 OMPI_DECLSPEC extern ompi_predefined_communicator_t *ompi_mpi_comm_null_addr;
306
307
308 /**
309 * Is this a valid communicator? This is a complicated question.
310 * :-)
311 *
312 * According to MPI-1:5.2.4 (p137):
313 *
314 * "The predefined constant MPI_COMM_NULL is the value used for
315 * invalid communicator handles."
316 *
317 * Hence, MPI_COMM_NULL is not valid. However, MPI-2:4.12.4 (p50)
318 * clearly states that the MPI_*_C2F and MPI_*_F2C functions
319 * should treat MPI_COMM_NULL as a valid communicator -- it
320 * distinctly differentiates between "invalid" handles and
321 * "MPI_*_NULL" handles. Some feel that the MPI-1 definition
322 * still holds for all other MPI functions; others feel that the
323 * MPi-2 definitions trump the MPI-1 definition. Regardless of
324 * who is right, there is ambiguity here. So we have left
325 * ompi_comm_invalid() as originally coded -- per the MPI-1
326 * definition, where MPI_COMM_NULL is an invalid communicator.
327 * The MPI_Comm_c2f() function, therefore, calls
328 * ompi_comm_invalid() but also explictily checks to see if the
329 * handle is MPI_COMM_NULL.
330 */
331 static inline int ompi_comm_invalid(ompi_communicator_t* comm)
332 {
333 if ((NULL == comm) || (MPI_COMM_NULL == comm) ||
334 (OMPI_COMM_IS_FREED(comm)) || (OMPI_COMM_IS_INVALID(comm)) )
335 return true;
336 else
337 return false;
338 }
339
340 /**
341 * rank w/in the communicator
342 */
343 static inline int ompi_comm_rank(ompi_communicator_t* comm)
344 {
345 return comm->c_my_rank;
346 }
347
348 /**
349 * size of the communicator
350 */
351 static inline int ompi_comm_size(ompi_communicator_t* comm)
352 {
353 return comm->c_local_group->grp_proc_count;
354 }
355
356 /**
357 * size of the remote group for inter-communicators.
358 * returns zero for an intra-communicator
359 */
360 static inline int ompi_comm_remote_size(ompi_communicator_t* comm)
361 {
362 return (comm->c_flags & OMPI_COMM_INTER ? comm->c_remote_group->grp_proc_count : 0);
363 }
364
365 /**
366 * Context ID for the communicator, suitable for passing to
367 * ompi_comm_lookup for getting the communicator back
368 */
369 static inline uint32_t ompi_comm_get_cid(ompi_communicator_t* comm)
370 {
371 return comm->c_contextid;
372 }
373
374 /* return pointer to communicator associated with context id cid,
375 * No error checking is done*/
376 static inline ompi_communicator_t *ompi_comm_lookup(uint32_t cid)
377 {
378 /* array of pointers to communicators, indexed by context ID */
379 return (ompi_communicator_t*)opal_pointer_array_get_item(&ompi_mpi_communicators, cid);
380 }
381
382 static inline struct ompi_proc_t* ompi_comm_peer_lookup(ompi_communicator_t* comm, int peer_id)
383 {
384 #if OPAL_ENABLE_DEBUG
385 if(peer_id >= comm->c_remote_group->grp_proc_count) {
386 opal_output(0, "ompi_comm_peer_lookup: invalid peer index (%d)", peer_id);
387 return (struct ompi_proc_t *) NULL;
388 }
389 #endif
390 /*return comm->c_remote_group->grp_proc_pointers[peer_id];*/
391 return ompi_group_peer_lookup(comm->c_remote_group,peer_id);
392 }
393
394 static inline bool ompi_comm_peer_invalid(ompi_communicator_t* comm, int peer_id)
395 {
396 if(peer_id < 0 || peer_id >= comm->c_remote_group->grp_proc_count) {
397 return true;
398 }
399 return false;
400 }
401
402
403 /**
404 * Initialise MPI_COMM_WORLD and MPI_COMM_SELF
405 */
406 int ompi_comm_init(void);
407
408 /**
409 * extract the local group from a communicator
410 */
411 OMPI_DECLSPEC int ompi_comm_group (ompi_communicator_t *comm, ompi_group_t **group);
412
413 /**
414 * create a communicator based on a group
415 */
416 int ompi_comm_create (ompi_communicator_t* comm, ompi_group_t *group,
417 ompi_communicator_t** newcomm);
418
419
420 /**
421 * Non-collective create communicator based on a group
422 */
423 int ompi_comm_create_group (ompi_communicator_t *comm, ompi_group_t *group, int tag,
424 ompi_communicator_t **newcomm);
425
426 /**
427 * Take an almost complete communicator and reserve the CID as well
428 * as activate it (initialize the collective and the topologies).
429 */
430 int ompi_comm_enable(ompi_communicator_t *old_comm,
431 ompi_communicator_t *new_comm,
432 int new_rank,
433 int num_procs,
434 ompi_proc_t** topo_procs);
435
436 /**
437 * Back end of MPI_DIST_GRAPH_CREATE_ADJACENT
438 */
439 int ompi_topo_dist_graph_create_adjacent(ompi_communicator_t *old_comm,
440 int indegree, int sources[],
441 int sourceweights[], int outdegree,
442 int destinations[], int destweights[],
443 MPI_Info info, int reorder,
444 MPI_Comm *comm_dist_graph);
445
446 /**
447 * split a communicator based on color and key. Parameters
448 * are identical to the MPI-counterpart of the function.
449 *
450 * @param comm: input communicator
451 * @param color
452 * @param key
453 *
454 * @
455 */
456 OMPI_DECLSPEC int ompi_comm_split (ompi_communicator_t *comm, int color, int key,
457 ompi_communicator_t** newcomm, bool pass_on_topo);
458
459 /**
460 * split a communicator based on type and key. Parameters
461 * are identical to the MPI-counterpart of the function.
462 *
463 * @param comm: input communicator
464 * @param color
465 * @param key
466 *
467 * @
468 */
469 OMPI_DECLSPEC int ompi_comm_split_type(ompi_communicator_t *comm,
470 int split_type, int key,
471 struct opal_info_t *info,
472 ompi_communicator_t** newcomm);
473
474 /**
475 * dup a communicator. Parameter are identical to the MPI-counterpart
476 * of the function. It has been extracted, since we need to be able
477 * to dup a communicator internally as well.
478 *
479 * @param comm: input communicator
480 * @param newcomm: the new communicator or MPI_COMM_NULL if any error is detected.
481 */
482 OMPI_DECLSPEC int ompi_comm_dup (ompi_communicator_t *comm, ompi_communicator_t **newcomm);
483
484 /**
485 * dup a communicator (non-blocking). Parameter are identical to the MPI-counterpart
486 * of the function. It has been extracted, since we need to be able
487 * to dup a communicator internally as well.
488 *
489 * @param comm: input communicator
490 * @param newcomm: the new communicator or MPI_COMM_NULL if any error is detected.
491 */
492 OMPI_DECLSPEC int ompi_comm_idup (ompi_communicator_t *comm, ompi_communicator_t **newcomm, ompi_request_t **request);
493
494 /**
495 * dup a communicator with info. Parameter are identical to the MPI-counterpart
496 * of the function. It has been extracted, since we need to be able
497 * to dup a communicator internally as well.
498 *
499 * @param comm: input communicator
500 * @param newcomm: the new communicator or MPI_COMM_NULL if any error is detected.
501 */
502 OMPI_DECLSPEC int ompi_comm_dup_with_info (ompi_communicator_t *comm, opal_info_t *info, ompi_communicator_t **newcomm);
503
504 /**
505 * dup a communicator (non-blocking) with info.
506 * of the function. It has been extracted, since we need to be able
507 * to dup a communicator internally as well.
508 *
509 * @param comm: input communicator
510 * @param newcomm: the new communicator or MPI_COMM_NULL if any error is detected.
511 */
512 OMPI_DECLSPEC int ompi_comm_idup_with_info (ompi_communicator_t *comm, opal_info_t *info, ompi_communicator_t **newcomm, ompi_request_t **req);
513
514 /**
515 * compare two communicators.
516 *
517 * @param comm1,comm2: input communicators
518 *
519 */
520 int ompi_comm_compare(ompi_communicator_t *comm1, ompi_communicator_t *comm2, int *result);
521
522 /**
523 * free a communicator
524 */
525 OMPI_DECLSPEC int ompi_comm_free (ompi_communicator_t **comm);
526
527 /**
528 * allocate a new communicator structure
529 * @param local_group_size
530 * @param remote_group_size
531 *
532 * This routine allocates the structure, the according local and
533 * remote groups, the proc-arrays in the local and remote group.
534 * It furthermore sets the fortran index correctly,
535 * and sets all other elements to zero.
536 */
537 ompi_communicator_t* ompi_comm_allocate (int local_group_size,
538 int remote_group_size);
539
540 /**
541 * allocate new communicator ID
542 * @param newcomm: pointer to the new communicator
543 * @param oldcomm: original comm
544 * @param bridgecomm: bridge comm for intercomm_create
545 * @param mode: combination of input
546 * OMPI_COMM_CID_INTRA: intra-comm
547 * OMPI_COMM_CID_INTER: inter-comm
548 * OMPI_COMM_CID_GROUP: only decide CID within the ompi_group_t
549 * associated with the communicator. arg0
550 * must point to an int which will be used
551 * as the pml tag for communication.
552 * OMPI_COMM_CID_INTRA_BRIDGE: 2 intracomms connected by
553 * a bridge comm. arg0 and arg1 must point
554 * to integers representing the local and
555 * remote leader ranks. the remote leader rank
556 * is a rank in the bridgecomm.
557 * OMPI_COMM_CID_INTRA_PMIX: 2 intracomms, leaders talk
558 * through PMIx. arg0 must point to an integer
559 * representing the local leader rank. arg1
560 * must point to a string representing the
561 * port of the remote leader.
562 * @param send_first: to avoid a potential deadlock for
563 * the OOB version.
564 * This routine has to be thread safe in the final version.
565 */
566 OMPI_DECLSPEC int ompi_comm_nextcid (ompi_communicator_t *newcomm, ompi_communicator_t *comm,
567 ompi_communicator_t *bridgecomm, const void *arg0, const void *arg1,
568 bool send_first, int mode);
569
570 /**
571 * allocate new communicator ID (non-blocking)
572 * @param newcomm: pointer to the new communicator
573 * @param oldcomm: original comm
574 * @param bridgecomm: bridge comm for intercomm_create
575 * @param mode: combination of input
576 * OMPI_COMM_CID_INTRA: intra-comm
577 * OMPI_COMM_CID_INTER: inter-comm
578 * This routine has to be thread safe in the final version.
579 */
580 OMPI_DECLSPEC int ompi_comm_nextcid_nb (ompi_communicator_t *newcomm, ompi_communicator_t *comm,
581 ompi_communicator_t *bridgecomm, const void *arg0, const void *arg1,
582 bool send_first, int mode, ompi_request_t **req);
583
584 /**
585 * shut down the communicator infrastructure.
586 */
587 int ompi_comm_finalize (void);
588
589 /**
590 * This is THE routine, where all the communicator stuff
591 * is really set.
592 *
593 * @param[out] newcomm new ompi communicator object
594 * @param[in] oldcomm old communicator
595 * @param[in] local_size size of local_ranks array
596 * @param[in] local_ranks local ranks (not used if local_group != NULL)
597 * @param[in] remote_size size of remote_ranks array
598 * @param[in] remote_ranks remote ranks (intercomm) (not used if remote_group != NULL)
599 * @param[in] attr attributes (can be NULL)
600 * @param[in] errh error handler
601 * @param[in] copy_topocomponent whether to copy the topology
602 * @param[in] local_group local process group (may be NULL if local_ranks array supplied)
603 * @param[in] remote_group remote process group (may be NULL)
604 */
605 OMPI_DECLSPEC int ompi_comm_set ( ompi_communicator_t** newcomm,
606 ompi_communicator_t* oldcomm,
607 int local_size,
608 int *local_ranks,
609 int remote_size,
610 int *remote_ranks,
611 opal_hash_table_t *attr,
612 ompi_errhandler_t *errh,
613 bool copy_topocomponent,
614 ompi_group_t *local_group,
615 ompi_group_t *remote_group );
616
617 /**
618 * This is THE routine, where all the communicator stuff
619 * is really set. Non-blocking version.
620 *
621 * @param[out] newcomm new ompi communicator object
622 * @param[in] oldcomm old communicator
623 * @param[in] local_size size of local_ranks array
624 * @param[in] local_ranks local ranks (not used if local_group != NULL)
625 * @param[in] remote_size size of remote_ranks array
626 * @param[in] remote_ranks remote ranks (intercomm) (not used if remote_group != NULL)
627 * @param[in] attr attributes (can be NULL)
628 * @param[in] errh error handler
629 * @param[in] copy_topocomponent whether to copy the topology
630 * @param[in] local_group local process group (may be NULL if local_ranks array supplied)
631 * @param[in] remote_group remote process group (may be NULL)
632 * @param[out] req ompi_request_t object for tracking completion
633 */
634 OMPI_DECLSPEC int ompi_comm_set_nb ( ompi_communicator_t **ncomm,
635 ompi_communicator_t *oldcomm,
636 int local_size,
637 int *local_ranks,
638 int remote_size,
639 int *remote_ranks,
640 opal_hash_table_t *attr,
641 ompi_errhandler_t *errh,
642 bool copy_topocomponent,
643 ompi_group_t *local_group,
644 ompi_group_t *remote_group,
645 ompi_request_t **req );
646
647 /**
648 * This is a short-hand routine used in intercomm_create.
649 * The routine makes sure, that all processes have afterwards
650 * a list of ompi_proc_t pointers for the remote group.
651 */
652 int ompi_comm_get_rprocs ( ompi_communicator_t *local_comm,
653 ompi_communicator_t *bridge_comm,
654 int local_leader,
655 int remote_leader,
656 int tag,
657 int rsize,
658 struct ompi_proc_t ***prprocs );
659
660 /**
661 * This routine verifies, whether local_group and remote group are overlapping
662 * in intercomm_create
663 */
664 int ompi_comm_overlapping_groups (int size, struct ompi_proc_t ** lprocs,
665 int rsize, struct ompi_proc_t ** rprocs);
666
667 /**
668 * This is a routine determining whether the local or the
669 * remote group will be first in the new intra-comm.
670 * Just used from within MPI_Intercomm_merge.
671 */
672 int ompi_comm_determine_first ( ompi_communicator_t *intercomm,
673 int high );
674
675
676 OMPI_DECLSPEC int ompi_comm_activate (ompi_communicator_t **newcomm, ompi_communicator_t *comm,
677 ompi_communicator_t *bridgecomm, const void *arg0,
678 const void *arg1, bool send_first, int mode);
679
680 /**
681 * Non-blocking variant of comm_activate.
682 *
683 * @param[inout] newcomm New communicator
684 * @param[in] comm Parent communicator
685 * @param[in] bridgecomm Bridge communicator (used for PMIX and bridge modes)
686 * @param[in] arg0 Mode argument 0
687 * @param[in] arg1 Mode argument 1
688 * @param[in] send_first Send first from this process (PMIX mode only)
689 * @param[in] mode Collective mode
690 * @param[out] req New request object to track this operation
691 */
692 OMPI_DECLSPEC int ompi_comm_activate_nb (ompi_communicator_t **newcomm, ompi_communicator_t *comm,
693 ompi_communicator_t *bridgecomm, const void *arg0,
694 const void *arg1, bool send_first, int mode, ompi_request_t **req);
695
696 /**
697 * a simple function to dump the structure
698 */
699 int ompi_comm_dump ( ompi_communicator_t *comm );
700
701 /* setting name */
702 int ompi_comm_set_name (ompi_communicator_t *comm, const char *name );
703
704 /* global variable to save the number od dynamic communicators */
705 extern int ompi_comm_num_dyncomm;
706
707
708 /* check whether any of the processes has requested support for
709 MPI_THREAD_MULTIPLE. Note, that this produces global
710 information across MPI_COMM_WORLD, in contrary to the local
711 flag ompi_mpi_thread_provided
712 */
713 OMPI_DECLSPEC int ompi_comm_cid_init ( void );
714
715
716 void ompi_comm_assert_subscribe (ompi_communicator_t *comm, int32_t assert_flag);
717
718 END_C_DECLS
719
720 #endif /* OMPI_COMMUNICATOR_H */
721