This source file includes following definitions.
- mca_pml_cm_irecv_init
- mca_pml_cm_irecv
- mca_pml_cm_recv_fast_completion
- mca_pml_cm_recv
- mca_pml_cm_isend_init
- mca_pml_cm_isend
- mca_pml_cm_send
- mca_pml_cm_iprobe
- mca_pml_cm_probe
- mca_pml_cm_improbe
- mca_pml_cm_mprobe
- mca_pml_cm_imrecv
- mca_pml_cm_mrecv
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 #ifndef PML_CM_H
18 #define PML_CM_H
19
20 #ifdef HAVE_ALLOCA_H
21 #include <alloca.h>
22 #endif
23
24 #include "ompi_config.h"
25 #include "ompi/request/request.h"
26 #include "ompi/mca/pml/pml.h"
27 #include "ompi/mca/pml/base/base.h"
28 #include "ompi/datatype/ompi_datatype.h"
29 #include "ompi/communicator/communicator.h"
30 #include "ompi/request/request.h"
31 #include "ompi/mca/mtl/mtl.h"
32
33
34 #include "pml_cm_request.h"
35 #include "ompi/mca/pml/base/pml_base_recvreq.h"
36 #include "ompi/mca/mtl/mtl.h"
37 #include "pml_cm_recvreq.h"
38 #include "pml_cm_sendreq.h"
39 #include "ompi/message/message.h"
40
41
42 BEGIN_C_DECLS
43
44 struct mca_mtl_request_t;
45
46
47
48
49
50 extern void (*send_completion_callbacks[])
51 (struct mca_mtl_request_t *mtl_request);
52
53 struct ompi_pml_cm_t {
54 mca_pml_base_module_t super;
55 int free_list_num;
56 int free_list_max;
57 int free_list_inc;
58 };
59 typedef struct ompi_pml_cm_t ompi_pml_cm_t;
60 extern ompi_pml_cm_t ompi_pml_cm;
61
62
63 OMPI_DECLSPEC extern int mca_pml_cm_add_procs(struct ompi_proc_t **procs, size_t nprocs);
64 OMPI_DECLSPEC extern int mca_pml_cm_del_procs(struct ompi_proc_t **procs, size_t nprocs);
65
66 OMPI_DECLSPEC extern int mca_pml_cm_enable(bool enable);
67 OMPI_DECLSPEC extern int mca_pml_cm_progress(void);
68
69 OMPI_DECLSPEC extern int mca_pml_cm_add_comm(struct ompi_communicator_t* comm);
70 OMPI_DECLSPEC extern int mca_pml_cm_del_comm(struct ompi_communicator_t* comm);
71
72
73 __opal_attribute_always_inline__ static inline int
74 mca_pml_cm_irecv_init(void *addr,
75 size_t count,
76 ompi_datatype_t * datatype,
77 int src,
78 int tag,
79 struct ompi_communicator_t *comm,
80 struct ompi_request_t **request)
81 {
82 mca_pml_cm_hvy_recv_request_t *recvreq;
83 uint32_t flags = 0;
84 #if OPAL_ENABLE_HETEROGENEOUS_SUPPORT
85 ompi_proc_t* ompi_proc;
86 #endif
87
88 MCA_PML_CM_HVY_RECV_REQUEST_ALLOC(recvreq);
89 if( OPAL_UNLIKELY(NULL == recvreq) ) return OMPI_ERR_OUT_OF_RESOURCE;
90
91 MCA_PML_CM_HVY_RECV_REQUEST_INIT(recvreq, ompi_proc, comm, tag, src,
92 datatype, addr, count, flags, true);
93
94 *request = (ompi_request_t*) recvreq;
95
96 return OMPI_SUCCESS;
97 }
98
99 __opal_attribute_always_inline__ static inline int
100 mca_pml_cm_irecv(void *addr,
101 size_t count,
102 ompi_datatype_t * datatype,
103 int src,
104 int tag,
105 struct ompi_communicator_t *comm,
106 struct ompi_request_t **request)
107 {
108 int ret;
109 uint32_t flags = 0;
110 mca_pml_cm_thin_recv_request_t *recvreq;
111 #if OPAL_ENABLE_HETEROGENEOUS_SUPPORT
112 ompi_proc_t* ompi_proc = NULL;
113 #endif
114
115 MCA_PML_CM_THIN_RECV_REQUEST_ALLOC(recvreq);
116 if( OPAL_UNLIKELY(NULL == recvreq) ) return OMPI_ERR_OUT_OF_RESOURCE;
117
118 MCA_PML_CM_THIN_RECV_REQUEST_INIT(recvreq,
119 ompi_proc,
120 comm,
121 src,
122 datatype,
123 addr,
124 count,
125 flags);
126
127 MCA_PML_CM_THIN_RECV_REQUEST_START(recvreq, comm, tag, src, ret);
128
129 if( OPAL_LIKELY(OMPI_SUCCESS == ret) ) *request = (ompi_request_t*) recvreq;
130
131 return ret;
132 }
133
134 __opal_attribute_always_inline__ static inline void
135 mca_pml_cm_recv_fast_completion(struct mca_mtl_request_t *mtl_request)
136 {
137
138 ompi_request_complete(mtl_request->ompi_req, true);
139 return;
140 }
141
142 __opal_attribute_always_inline__ static inline int
143 mca_pml_cm_recv(void *addr,
144 size_t count,
145 ompi_datatype_t * datatype,
146 int src,
147 int tag,
148 struct ompi_communicator_t *comm,
149 ompi_status_public_t * status)
150 {
151 int ret;
152 uint32_t flags = 0;
153 #if OPAL_ENABLE_HETEROGENEOUS_SUPPORT
154 ompi_proc_t *ompi_proc;
155 #endif
156 opal_convertor_t convertor;
157 mca_pml_cm_request_t req;
158 mca_mtl_request_t *req_mtl =
159 alloca(sizeof(mca_mtl_request_t) + ompi_mtl->mtl_request_size);
160
161 OBJ_CONSTRUCT(&convertor, opal_convertor_t);
162 req_mtl->ompi_req = &req.req_ompi;
163 req_mtl->completion_callback = mca_pml_cm_recv_fast_completion;
164
165 req.req_pml_type = MCA_PML_CM_REQUEST_RECV_THIN;
166 req.req_free_called = false;
167 req.req_ompi.req_complete = false;
168 req.req_ompi.req_complete_cb = NULL;
169 req.req_ompi.req_state = OMPI_REQUEST_ACTIVE;
170 req.req_ompi.req_status.MPI_TAG = OMPI_ANY_TAG;
171 req.req_ompi.req_status.MPI_ERROR = OMPI_SUCCESS;
172 req.req_ompi.req_status._cancelled = 0;
173
174 #if OPAL_ENABLE_HETEROGENEOUS_SUPPORT
175 if( MPI_ANY_SOURCE == src ) {
176 ompi_proc = ompi_proc_local_proc;
177 } else {
178 ompi_proc = ompi_comm_peer_lookup( comm, src );
179 }
180
181 MCA_PML_CM_SWITCH_CUDA_CONVERTOR_OFF(flags, datatype, count);
182
183 opal_convertor_copy_and_prepare_for_recv(
184 ompi_proc->super.proc_convertor,
185 &(datatype->super),
186 count,
187 addr,
188 flags,
189 &convertor );
190 #else
191 MCA_PML_CM_SWITCH_CUDA_CONVERTOR_OFF(flags, datatype, count);
192
193 opal_convertor_copy_and_prepare_for_recv(
194 ompi_mpi_local_convertor,
195 &(datatype->super),
196 count,
197 addr,
198 flags,
199 &convertor );
200 #endif
201
202 ret = OMPI_MTL_CALL(irecv(ompi_mtl,
203 comm,
204 src,
205 tag,
206 &convertor,
207 req_mtl));
208 if( OPAL_UNLIKELY(OMPI_SUCCESS != ret) ) {
209 OBJ_DESTRUCT(&convertor);
210 return ret;
211 }
212
213 ompi_request_wait_completion(&req.req_ompi);
214
215 if (NULL != status) {
216 *status = req.req_ompi.req_status;
217 }
218 ret = req.req_ompi.req_status.MPI_ERROR;
219 OBJ_DESTRUCT(&convertor);
220 return ret;
221 }
222
223 __opal_attribute_always_inline__ static inline int
224 mca_pml_cm_isend_init(const void* buf,
225 size_t count,
226 ompi_datatype_t* datatype,
227 int dst,
228 int tag,
229 mca_pml_base_send_mode_t sendmode,
230 ompi_communicator_t* comm,
231 ompi_request_t** request)
232 {
233 mca_pml_cm_hvy_send_request_t *sendreq;
234 uint32_t flags = 0;
235 #if OPAL_ENABLE_HETEROGENEOUS_SUPPORT
236 ompi_proc_t* ompi_proc;
237 #endif
238
239 MCA_PML_CM_HVY_SEND_REQUEST_ALLOC(sendreq, comm, dst, ompi_proc);
240 if (OPAL_UNLIKELY(NULL == sendreq)) return OMPI_ERR_OUT_OF_RESOURCE;
241
242 MCA_PML_CM_HVY_SEND_REQUEST_INIT(sendreq, ompi_proc, comm, tag, dst,
243 datatype, sendmode, true, false, buf, count, flags);
244
245
246
247
248
249 sendreq->req_send.req_base.req_pml_complete = true;
250
251 *request = (ompi_request_t*) sendreq;
252
253 return OMPI_SUCCESS;
254 }
255
256 __opal_attribute_always_inline__ static inline int
257 mca_pml_cm_isend(const void* buf,
258 size_t count,
259 ompi_datatype_t* datatype,
260 int dst,
261 int tag,
262 mca_pml_base_send_mode_t sendmode,
263 ompi_communicator_t* comm,
264 ompi_request_t** request)
265 {
266 int ret;
267 uint32_t flags = 0;
268
269 if(sendmode == MCA_PML_BASE_SEND_BUFFERED ) {
270 mca_pml_cm_hvy_send_request_t* sendreq;
271 #if OPAL_ENABLE_HETEROGENEOUS_SUPPORT
272 ompi_proc_t* ompi_proc = NULL;
273 #endif
274
275 MCA_PML_CM_HVY_SEND_REQUEST_ALLOC(sendreq, comm, dst, ompi_proc);
276 if (OPAL_UNLIKELY(NULL == sendreq)) return OMPI_ERR_OUT_OF_RESOURCE;
277
278 MCA_PML_CM_HVY_SEND_REQUEST_INIT(sendreq,
279 ompi_proc,
280 comm,
281 tag,
282 dst,
283 datatype,
284 sendmode,
285 false,
286 false,
287 buf,
288 count,
289 flags);
290
291 MCA_PML_CM_HVY_SEND_REQUEST_START( sendreq, ret);
292
293 if (OPAL_LIKELY(OMPI_SUCCESS == ret)) *request = (ompi_request_t*) sendreq;
294
295 } else {
296 mca_pml_cm_thin_send_request_t* sendreq;
297 #if OPAL_ENABLE_HETEROGENEOUS_SUPPORT
298 ompi_proc_t* ompi_proc = NULL;
299 #endif
300 MCA_PML_CM_THIN_SEND_REQUEST_ALLOC(sendreq, comm, dst, ompi_proc);
301 if (OPAL_UNLIKELY(NULL == sendreq)) return OMPI_ERR_OUT_OF_RESOURCE;
302
303 MCA_PML_CM_THIN_SEND_REQUEST_INIT(sendreq,
304 ompi_proc,
305 comm,
306 tag,
307 dst,
308 datatype,
309 sendmode,
310 buf,
311 count,
312 flags);
313
314 MCA_PML_CM_THIN_SEND_REQUEST_START(
315 sendreq,
316 comm,
317 tag,
318 dst,
319 sendmode,
320 false,
321 ret);
322
323 if (OPAL_LIKELY(OMPI_SUCCESS == ret)) *request = (ompi_request_t*) sendreq;
324
325 }
326
327 return ret;
328 }
329
330 __opal_attribute_always_inline__ static inline int
331 mca_pml_cm_send(const void *buf,
332 size_t count,
333 ompi_datatype_t* datatype,
334 int dst,
335 int tag,
336 mca_pml_base_send_mode_t sendmode,
337 ompi_communicator_t* comm)
338 {
339 int ret = OMPI_ERROR;
340 uint32_t flags = 0;
341 ompi_proc_t * ompi_proc;
342
343 if(sendmode == MCA_PML_BASE_SEND_BUFFERED) {
344 mca_pml_cm_hvy_send_request_t *sendreq;
345
346 MCA_PML_CM_HVY_SEND_REQUEST_ALLOC(sendreq, comm, dst, ompi_proc);
347 if (OPAL_UNLIKELY(NULL == sendreq)) return OMPI_ERR_OUT_OF_RESOURCE;
348
349 MCA_PML_CM_HVY_SEND_REQUEST_INIT(sendreq,
350 ompi_proc,
351 comm,
352 tag,
353 dst,
354 datatype,
355 sendmode,
356 false,
357 false,
358 buf,
359 count,
360 flags);
361 MCA_PML_CM_HVY_SEND_REQUEST_START(sendreq, ret);
362 if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
363 MCA_PML_CM_HVY_SEND_REQUEST_RETURN(sendreq);
364 return ret;
365 }
366
367 ompi_request_free( (ompi_request_t**)&sendreq );
368 } else {
369 opal_convertor_t convertor;
370 OBJ_CONSTRUCT(&convertor, opal_convertor_t);
371 #if !(OPAL_ENABLE_HETEROGENEOUS_SUPPORT)
372 if (opal_datatype_is_contiguous_memory_layout(&datatype->super, count)) {
373
374 convertor.remoteArch = ompi_mpi_local_convertor->remoteArch;
375 convertor.flags = ompi_mpi_local_convertor->flags;
376 convertor.master = ompi_mpi_local_convertor->master;
377
378 convertor.local_size = count * datatype->super.size;
379 convertor.pBaseBuf = (unsigned char*)buf + datatype->super.true_lb;
380 convertor.count = count;
381 convertor.pDesc = &datatype->super;
382 } else
383 #endif
384 {
385 ompi_proc = ompi_comm_peer_lookup(comm, dst);
386
387 MCA_PML_CM_SWITCH_CUDA_CONVERTOR_OFF(flags, datatype, count);
388
389 opal_convertor_copy_and_prepare_for_send(
390 ompi_proc->super.proc_convertor,
391 &datatype->super, count, buf, flags,
392 &convertor);
393 }
394
395 ret = OMPI_MTL_CALL(send(ompi_mtl,
396 comm,
397 dst,
398 tag,
399 &convertor,
400 sendmode));
401 OBJ_DESTRUCT(&convertor);
402 }
403
404 return ret;
405 }
406
407 __opal_attribute_always_inline__ static inline int
408 mca_pml_cm_iprobe(int src, int tag,
409 struct ompi_communicator_t *comm,
410 int *matched, ompi_status_public_t * status)
411 {
412 return OMPI_MTL_CALL(iprobe(ompi_mtl,
413 comm, src, tag,
414 matched, status));
415 }
416
417 __opal_attribute_always_inline__ static inline int
418 mca_pml_cm_probe(int src, int tag,
419 struct ompi_communicator_t *comm,
420 ompi_status_public_t * status)
421 {
422 int ret, matched = 0;
423
424 while (true) {
425 ret = OMPI_MTL_CALL(iprobe(ompi_mtl,
426 comm, src, tag,
427 &matched, status));
428 if (OMPI_SUCCESS != ret) break;
429 if (matched) break;
430 opal_progress();
431 }
432
433 return ret;
434 }
435
436 __opal_attribute_always_inline__ static inline int
437 mca_pml_cm_improbe(int src,
438 int tag,
439 struct ompi_communicator_t* comm,
440 int *matched,
441 struct ompi_message_t **message,
442 ompi_status_public_t* status)
443 {
444 return OMPI_MTL_CALL(improbe(ompi_mtl,
445 comm, src, tag,
446 matched, message,
447 status));
448 }
449
450 __opal_attribute_always_inline__ static inline int
451 mca_pml_cm_mprobe(int src,
452 int tag,
453 struct ompi_communicator_t* comm,
454 struct ompi_message_t **message,
455 ompi_status_public_t* status)
456 {
457 int ret, matched = 0;
458
459 while (true) {
460 ret = OMPI_MTL_CALL(improbe(ompi_mtl,
461 comm, src, tag,
462 &matched, message,
463 status));
464 if (OMPI_SUCCESS != ret) break;
465 if (matched) break;
466 opal_progress();
467 }
468
469 return ret;
470 }
471
472 __opal_attribute_always_inline__ static inline int
473 mca_pml_cm_imrecv(void *buf,
474 size_t count,
475 ompi_datatype_t *datatype,
476 struct ompi_message_t **message,
477 struct ompi_request_t **request)
478 {
479 int ret;
480 uint32_t flags = 0;
481 mca_pml_cm_thin_recv_request_t *recvreq;
482 #if OPAL_ENABLE_HETEROGENEOUS_SUPPORT
483 ompi_proc_t* ompi_proc;
484 #endif
485 ompi_communicator_t *comm = (*message)->comm;
486
487 MCA_PML_CM_THIN_RECV_REQUEST_ALLOC(recvreq);
488 if( OPAL_UNLIKELY(NULL == recvreq) ) return OMPI_ERR_OUT_OF_RESOURCE;
489
490 MCA_PML_CM_THIN_RECV_REQUEST_INIT(recvreq,
491 ompi_proc,
492 comm,
493 (*message)->peer,
494 datatype,
495 buf,
496 count,
497 flags);
498
499 MCA_PML_CM_THIN_RECV_REQUEST_MATCHED_START(recvreq, message, ret);
500
501 if( OPAL_LIKELY(OMPI_SUCCESS == ret) ) *request = (ompi_request_t*) recvreq;
502
503 return ret;
504 }
505
506 __opal_attribute_always_inline__ static inline int
507 mca_pml_cm_mrecv(void *buf,
508 size_t count,
509 ompi_datatype_t *datatype,
510 struct ompi_message_t **message,
511 ompi_status_public_t* status)
512 {
513 int ret;
514 uint32_t flags = 0;
515 mca_pml_cm_thin_recv_request_t *recvreq;
516 #if OPAL_ENABLE_HETEROGENEOUS_SUPPORT
517 ompi_proc_t* ompi_proc;
518 #endif
519 ompi_communicator_t *comm = (*message)->comm;
520
521 MCA_PML_CM_THIN_RECV_REQUEST_ALLOC(recvreq);
522 if( OPAL_UNLIKELY(NULL == recvreq) ) return OMPI_ERR_OUT_OF_RESOURCE;
523
524 MCA_PML_CM_THIN_RECV_REQUEST_INIT(recvreq,
525 ompi_proc,
526 comm,
527 (*message)->peer,
528 datatype,
529 buf,
530 count,
531 flags);
532
533 MCA_PML_CM_THIN_RECV_REQUEST_MATCHED_START(recvreq,
534 message, ret);
535 if( OPAL_UNLIKELY(OMPI_SUCCESS != ret) ) {
536 MCA_PML_CM_THIN_RECV_REQUEST_RETURN(recvreq);
537 return ret;
538 }
539
540 ompi_request_wait_completion(&recvreq->req_base.req_ompi);
541
542 if (NULL != status) {
543 *status = recvreq->req_base.req_ompi.req_status;
544 }
545 ret = recvreq->req_base.req_ompi.req_status.MPI_ERROR;
546 ompi_request_free( (ompi_request_t**)&recvreq );
547
548 return ret;
549 }
550
551 OMPI_DECLSPEC extern int mca_pml_cm_start(size_t count, ompi_request_t** requests);
552
553
554 OMPI_DECLSPEC extern int mca_pml_cm_dump(struct ompi_communicator_t* comm,
555 int verbose);
556
557 OMPI_DECLSPEC extern int mca_pml_cm_cancel(struct ompi_request_t *request, int flag);
558
559 END_C_DECLS
560
561 #endif