1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
2 /*
3 * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
4 * University Research and Technology
5 * Corporation. All rights reserved.
6 * Copyright (c) 2004-2013 The University of Tennessee and The University
7 * of Tennessee Research Foundation. All rights
8 * reserved.
9 * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
10 * University of Stuttgart. All rights reserved.
11 * Copyright (c) 2004-2005 The Regents of the University of California.
12 * All rights reserved.
13 * Copyright (c) 2008 Cisco Systems, Inc. All rights reserved.
14 * Copyright (c) 2012-2013 Inria. All rights reserved.
15 * Copyright (c) 2014 Los Alamos National Security, LLC. All rights
16 * reserved.
17 * Copyright (c) 2014-2015 Research Organization for Information Science
18 * and Technology (RIST). All rights reserved.
19 * $COPYRIGHT$
20 *
21 * Additional copyrights may follow
22 *
23 * $HEADER$
24 */
25
26 #include "ompi_config.h"
27 #include "ompi/mca/topo/base/base.h"
28 #include "ompi/communicator/communicator.h"
29
30 /*
31 * function - partitions a communicator into subgroups which
32 * form lower-dimensional cartesian subgrids
33 *
34 * @param comm communicator with cartesian structure (handle)
35 * @param remain_dims the 'i'th entry of 'remain_dims' specifies whether
36 * the 'i'th dimension is kept in the subgrid (true)
37 * or is dropped (false) (logical vector)
38 * @param new_comm communicator containing the subgrid that includes the
39 * calling process (handle)
40 *
41 * @retval MPI_SUCCESS
42 * @retval MPI_ERR_TOPOLOGY
43 * @retval MPI_ERR_COMM
44 */
45 int mca_topo_base_cart_sub (ompi_communicator_t* comm,
46 const int *remain_dims,
47 ompi_communicator_t** new_comm)
48 {
49 struct ompi_communicator_t *temp_comm;
50 mca_topo_base_comm_cart_2_2_0_t *old_cart;
51 int errcode, colour, key, colfactor, keyfactor;
52 int ndim, dim, i;
53 int *d, *dorig = NULL, *dold, *c, *p, *porig = NULL, *pold;
54 mca_topo_base_module_t* topo;
55 mca_topo_base_comm_cart_2_2_0_t* cart;
56
57 *new_comm = MPI_COMM_NULL;
58 old_cart = comm->c_topo->mtc.cart;
59
60 /*
61 * Compute colour and key used in splitting the communicator.
62 */
63 colour = key = 0;
64 colfactor = keyfactor = 1;
65 ndim = 0;
66
67 i = old_cart->ndims - 1;
68 d = old_cart->dims + i;
69 c = comm->c_topo->mtc.cart->coords + i;
70
71 for (; i >= 0; --i, --d, --c) {
72 dim = *d;
73 if (remain_dims[i] == 0) {
74 colour += colfactor * (*c);
75 colfactor *= dim;
76 } else {
77 ++ndim;
78 key += keyfactor * (*c);
79 keyfactor *= dim;
80 }
81 }
82 /* Special case: if all of remain_dims were false, we need to make
83 a 0-dimension cartesian communicator with just ourselves in it
84 (you can't have a communicator unless you're in it). */
85 if (0 == ndim) {
86 colour = ompi_comm_rank (comm);
87 }
88 /* Split the communicator. */
89 errcode = ompi_comm_split(comm, colour, key, &temp_comm, false);
90 if (errcode != OMPI_SUCCESS) {
91 return errcode;
92 }
93
94 /* Fill the communicator with topology information. */
95 if (temp_comm != MPI_COMM_NULL) {
96
97 assert( NULL == temp_comm->c_topo );
98 if (OMPI_SUCCESS != (errcode = mca_topo_base_comm_select(temp_comm,
99 comm->c_topo,
100 &topo,
101 OMPI_COMM_CART))) {
102 ompi_comm_free(&temp_comm);
103 return OMPI_ERR_OUT_OF_RESOURCE;
104 }
105 if (ndim >= 1) {
106 /* Copy the dimensions */
107 dorig = d = (int*)malloc(ndim * sizeof(int));
108 dold = old_cart->dims;
109 /* Copy the periods */
110 porig = p = (int*)malloc(ndim * sizeof(int));
111 pold = old_cart->periods;
112 for (i = 0; i < old_cart->ndims; ++i, ++dold, ++pold) {
113 if (remain_dims[i]) {
114 *d++ = *dold;
115 *p++ = *pold;
116 }
117 }
118 }
119 cart = OBJ_NEW(mca_topo_base_comm_cart_2_2_0_t);
120 if( NULL == cart ) {
121 ompi_comm_free(&temp_comm);
122 if (NULL != dorig) {
123 free(dorig);
124 }
125 if (NULL != porig) {
126 free(porig);
127 }
128 return OMPI_ERR_OUT_OF_RESOURCE;
129 }
130 cart->ndims = ndim;
131 cart->dims = dorig;
132 cart->periods = porig;
133
134 /* NTH: protect against a 0-byte alloc in the ndims = 0 case */
135 if (ndim > 0) {
136 cart->coords = (int*)malloc(sizeof(int) * ndim);
137 if (NULL == cart->coords) {
138 free(cart->periods);
139 if(NULL != cart->dims) free(cart->dims);
140 OBJ_RELEASE(cart);
141 return OMPI_ERR_OUT_OF_RESOURCE;
142 }
143 { /* setup the cartesian topology */
144 int nprocs = temp_comm->c_local_group->grp_proc_count,
145 rank = temp_comm->c_local_group->grp_my_rank;
146
147 for (i = 0; i < ndim; ++i) {
148 nprocs /= cart->dims[i];
149 cart->coords[i] = rank / nprocs;
150 rank %= nprocs;
151 }
152 }
153 }
154
155 temp_comm->c_topo = topo;
156 temp_comm->c_topo->mtc.cart = cart;
157 temp_comm->c_topo->reorder = false;
158 temp_comm->c_flags |= OMPI_COMM_CART;
159 }
160
161 *new_comm = temp_comm;
162
163 return MPI_SUCCESS;
164 }