This source file includes following definitions.
- nbc_reduce_scatter_init
- ompi_coll_libnbc_ireduce_scatter
- nbc_reduce_scatter_inter_init
- ompi_coll_libnbc_ireduce_scatter_inter
- ompi_coll_libnbc_reduce_scatter_init
- ompi_coll_libnbc_reduce_scatter_inter_init
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24 #include "opal/align.h"
25
26 #include "nbc_internal.h"
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45 static int nbc_reduce_scatter_init(const void* sendbuf, void* recvbuf, const int *recvcounts, MPI_Datatype datatype,
46 MPI_Op op, struct ompi_communicator_t *comm, ompi_request_t ** request,
47 struct mca_coll_base_module_2_3_0_t *module, bool persistent) {
48 int peer, rank, maxr, p, res, count;
49 MPI_Aint ext;
50 ptrdiff_t gap, span, span_align;
51 char *sbuf, inplace;
52 NBC_Schedule *schedule;
53 void *tmpbuf;
54 ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
55 char *rbuf, *lbuf, *buf;
56
57 NBC_IN_PLACE(sendbuf, recvbuf, inplace);
58
59 rank = ompi_comm_rank (comm);
60 p = ompi_comm_size (comm);
61
62 res = ompi_datatype_type_extent (datatype, &ext);
63 if (MPI_SUCCESS != res) {
64 NBC_Error("MPI Error in ompi_datatype_type_extent() (%i)", res);
65 return res;
66 }
67
68 count = 0;
69 for (int r = 0 ; r < p ; ++r) {
70 count += recvcounts[r];
71 }
72
73 if ((1 == p && (!persistent || inplace)) || 0 == count) {
74 if (!inplace) {
75
76 res = NBC_Copy(sendbuf, recvcounts[0], datatype, recvbuf, recvcounts[0], datatype, comm);
77 if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
78 return res;
79 }
80 }
81
82 return nbc_get_noop_request(persistent, request);
83 }
84
85 maxr = (int) ceil ((log((double) p) / LOG2));
86
87 span = opal_datatype_span(&datatype->super, count, &gap);
88 span_align = OPAL_ALIGN(span, datatype->super.align, ptrdiff_t);
89 tmpbuf = malloc (span_align + span);
90 if (OPAL_UNLIKELY(NULL == tmpbuf)) {
91 return OMPI_ERR_OUT_OF_RESOURCE;
92 }
93
94 rbuf = (char *)(-gap);
95 lbuf = (char *)(span_align - gap);
96
97 schedule = OBJ_NEW(NBC_Schedule);
98 if (OPAL_UNLIKELY(NULL == schedule)) {
99 free(tmpbuf);
100 return OMPI_ERR_OUT_OF_RESOURCE;
101 }
102
103 for (int r = 1, firstred = 1 ; r <= maxr ; ++r) {
104 if ((rank % (1 << r)) == 0) {
105
106 peer = rank + (1 << (r - 1));
107 if (peer < p) {
108
109 res = NBC_Sched_recv(rbuf, true, count, datatype, peer, schedule, true);
110 if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
111 OBJ_RELEASE(schedule);
112 free(tmpbuf);
113 return res;
114 }
115
116
117 if (firstred) {
118
119 res = NBC_Sched_op (sendbuf, false, rbuf, true, count, datatype, op, schedule, true);
120 firstred = 0;
121 } else {
122
123 res = NBC_Sched_op (lbuf, true, rbuf, true, count, datatype, op, schedule, true);
124 }
125
126 if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
127 OBJ_RELEASE(schedule);
128 free(tmpbuf);
129 return res;
130 }
131
132 buf = rbuf; rbuf = lbuf ; lbuf = buf;
133 }
134 } else {
135
136 peer = rank - (1 << (r - 1));
137 if (firstred) {
138
139 res = NBC_Sched_send (sendbuf, false, count, datatype, peer, schedule, false);
140 } else {
141
142 res = NBC_Sched_send (lbuf, true, count, datatype, peer, schedule, false);
143 }
144 if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
145 OBJ_RELEASE(schedule);
146 free(tmpbuf);
147 return res;
148 }
149
150
151 break;
152 }
153 }
154
155 res = NBC_Sched_barrier(schedule);
156 if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
157 OBJ_RELEASE(schedule);
158 free(tmpbuf);
159 return res;
160 }
161
162
163 if (rank == 0) {
164 for (long int r = 1, offset = 0 ; r < p ; ++r) {
165 offset += recvcounts[r-1];
166 sbuf = lbuf + (offset*ext);
167
168 res = NBC_Sched_send (sbuf, true, recvcounts[r], datatype, r, schedule,
169 false);
170 if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
171 OBJ_RELEASE(schedule);
172 free(tmpbuf);
173 return res;
174 }
175 }
176
177 if (p == 1) {
178
179 res = NBC_Sched_copy ((void *)sendbuf, false, recvcounts[0], datatype,
180 recvbuf, false, recvcounts[0], datatype, schedule, false);
181 } else {
182 res = NBC_Sched_copy (lbuf, true, recvcounts[0], datatype, recvbuf, false,
183 recvcounts[0], datatype, schedule, false);
184 }
185 } else {
186 res = NBC_Sched_recv (recvbuf, false, recvcounts[rank], datatype, 0, schedule, false);
187 }
188
189 if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
190 OBJ_RELEASE(schedule);
191 free(tmpbuf);
192 return res;
193 }
194
195 res = NBC_Sched_commit (schedule);
196 if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
197 OBJ_RELEASE(schedule);
198 free(tmpbuf);
199 return res;
200 }
201
202 res = NBC_Schedule_request(schedule, comm, libnbc_module, persistent, request, tmpbuf);
203 if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
204 OBJ_RELEASE(schedule);
205 free(tmpbuf);
206 return res;
207 }
208
209 return OMPI_SUCCESS;
210 }
211
212 int ompi_coll_libnbc_ireduce_scatter (const void* sendbuf, void* recvbuf, const int *recvcounts, MPI_Datatype datatype,
213 MPI_Op op, struct ompi_communicator_t *comm, ompi_request_t ** request,
214 struct mca_coll_base_module_2_3_0_t *module) {
215 int res = nbc_reduce_scatter_init(sendbuf, recvbuf, recvcounts, datatype, op,
216 comm, request, module, false);
217 if (OPAL_LIKELY(OMPI_SUCCESS != res)) {
218 return res;
219 }
220 res = NBC_Start(*(ompi_coll_libnbc_request_t **)request);
221 if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
222 NBC_Return_handle (*(ompi_coll_libnbc_request_t **)request);
223 *request = &ompi_request_null.request;
224 return res;
225 }
226
227 return OMPI_SUCCESS;
228 }
229 static int nbc_reduce_scatter_inter_init (const void* sendbuf, void* recvbuf, const int *recvcounts, MPI_Datatype datatype,
230 MPI_Op op, struct ompi_communicator_t *comm, ompi_request_t ** request,
231 struct mca_coll_base_module_2_3_0_t *module, bool persistent) {
232 int rank, res, count, lsize, rsize;
233 MPI_Aint ext;
234 ptrdiff_t gap, span, span_align;
235 NBC_Schedule *schedule;
236 void *tmpbuf = NULL;
237 ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
238
239 rank = ompi_comm_rank (comm);
240 lsize = ompi_comm_size(comm);
241 rsize = ompi_comm_remote_size (comm);
242
243 res = ompi_datatype_type_extent (datatype, &ext);
244 if (MPI_SUCCESS != res) {
245 NBC_Error("MPI Error in ompi_datatype_type_extent() (%i)", res);
246 return res;
247 }
248
249 count = 0;
250 for (int r = 0 ; r < lsize ; ++r) {
251 count += recvcounts[r];
252 }
253
254 span = opal_datatype_span(&datatype->super, count, &gap);
255 span_align = OPAL_ALIGN(span, datatype->super.align, ptrdiff_t);
256
257 if (count > 0) {
258 tmpbuf = malloc (span_align + span);
259 if (OPAL_UNLIKELY(NULL == tmpbuf)) {
260 return OMPI_ERR_OUT_OF_RESOURCE;
261 }
262 }
263
264 schedule = OBJ_NEW(NBC_Schedule);
265 if (OPAL_UNLIKELY(NULL == schedule)) {
266 free(tmpbuf);
267 return OMPI_ERR_OUT_OF_RESOURCE;
268 }
269
270
271 res = NBC_Sched_send(sendbuf, false, count, datatype, 0, schedule, false);
272 if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
273 OBJ_RELEASE(schedule);
274 free(tmpbuf);
275 return res;
276 }
277
278 if (0 == rank) {
279 char *lbuf, *rbuf;
280 lbuf = (char *)(-gap);
281 rbuf = (char *)(span_align-gap);
282 res = NBC_Sched_recv (lbuf, true, count, datatype, 0, schedule, true);
283 if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
284 OBJ_RELEASE(schedule);
285 free(tmpbuf);
286 return res;
287 }
288
289 for (int peer = 1 ; peer < rsize ; ++peer) {
290 char *tbuf;
291 res = NBC_Sched_recv (rbuf, true, count, datatype, peer, schedule, true);
292 if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
293 OBJ_RELEASE(schedule);
294 free(tmpbuf);
295 return res;
296 }
297
298 res = NBC_Sched_op (lbuf, true, rbuf, true, count, datatype,
299 op, schedule, true);
300 if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
301 OBJ_RELEASE(schedule);
302 free(tmpbuf);
303 return res;
304 }
305 tbuf = lbuf; lbuf = rbuf; rbuf = tbuf;
306 }
307
308
309 res = NBC_Sched_copy (lbuf, true, recvcounts[0], datatype, recvbuf, false,
310 recvcounts[0], datatype, schedule, false);
311 if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
312 OBJ_RELEASE(schedule);
313 free(tmpbuf);
314 return res;
315 }
316 for (int peer = 1, offset = recvcounts[0] * ext; peer < lsize ; ++peer) {
317 res = NBC_Sched_local_send (lbuf + offset, true, recvcounts[peer], datatype, peer, schedule,
318 false);
319 if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
320 OBJ_RELEASE(schedule);
321 free(tmpbuf);
322 return res;
323 }
324
325 offset += recvcounts[peer] * ext;
326 }
327 } else {
328
329 res = NBC_Sched_local_recv (recvbuf, false, recvcounts[rank], datatype, 0, schedule, false);
330 if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
331 OBJ_RELEASE(schedule);
332 free(tmpbuf);
333 return res;
334 }
335 }
336
337 res = NBC_Sched_commit (schedule);
338 if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
339 OBJ_RELEASE(schedule);
340 free(tmpbuf);
341 return res;
342 }
343
344 res = NBC_Schedule_request(schedule, comm, libnbc_module, persistent, request, tmpbuf);
345 if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
346 OBJ_RELEASE(schedule);
347 free(tmpbuf);
348 return res;
349 }
350
351 return OMPI_SUCCESS;
352 }
353
354 int ompi_coll_libnbc_ireduce_scatter_inter (const void* sendbuf, void* recvbuf, const int *recvcounts, MPI_Datatype datatype,
355 MPI_Op op, struct ompi_communicator_t *comm, ompi_request_t ** request,
356 struct mca_coll_base_module_2_3_0_t *module) {
357 int res = nbc_reduce_scatter_inter_init(sendbuf, recvbuf, recvcounts, datatype, op,
358 comm, request, module, false);
359 if (OPAL_LIKELY(OMPI_SUCCESS != res)) {
360 return res;
361 }
362 res = NBC_Start(*(ompi_coll_libnbc_request_t **)request);
363 if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
364 NBC_Return_handle (*(ompi_coll_libnbc_request_t **)request);
365 *request = &ompi_request_null.request;
366 return res;
367 }
368
369 return OMPI_SUCCESS;
370 }
371
372 int ompi_coll_libnbc_reduce_scatter_init(const void* sendbuf, void* recvbuf, const int *recvcounts, MPI_Datatype datatype,
373 MPI_Op op, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
374 struct mca_coll_base_module_2_3_0_t *module) {
375 int res = nbc_reduce_scatter_init(sendbuf, recvbuf, recvcounts, datatype, op,
376 comm, request, module, true);
377 if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
378 return res;
379 }
380
381 return OMPI_SUCCESS;
382 }
383
384 int ompi_coll_libnbc_reduce_scatter_inter_init(const void* sendbuf, void* recvbuf, const int *recvcounts, MPI_Datatype datatype,
385 MPI_Op op, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
386 struct mca_coll_base_module_2_3_0_t *module) {
387 int res = nbc_reduce_scatter_inter_init(sendbuf, recvbuf, recvcounts, datatype, op,
388 comm, request, module, true);
389 if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
390 return res;
391 }
392
393 return OMPI_SUCCESS;
394 }