1 // -*- c++ -*-
2 //
3 // Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
4 // University Research and Technology
5 // Corporation. All rights reserved.
6 // Copyright (c) 2004-2005 The University of Tennessee and The University
7 // of Tennessee Research Foundation. All rights
8 // reserved.
9 // Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
10 // University of Stuttgart. All rights reserved.
11 // Copyright (c) 2004-2005 The Regents of the University of California.
12 // All rights reserved.
13 // Copyright (c) 2006-2009 Cisco Systems, Inc. All rights reserved.
14 // Copyright (c) 2009 Sun Microsystems, Inc. All rights reserved.
15 // Copyright (c) 2016 Los Alamos National Security, LLC. All rights
16 // reserved.
17 // Copyright (c) 2017 Research Organization for Information Science
18 // and Technology (RIST). All rights reserved.
19 // $COPYRIGHT$
20 //
21 // Additional copyrights may follow
22 //
23 // $HEADER$
24 //
25
26
27 #include "mpicxx.h"
28 #include <cstdio>
29
30 #include "ompi_config.h"
31 #include "cxx_glue.h"
32
33 extern "C"
34 void ompi_mpi_cxx_throw_exception(int *errcode)
35 {
36 #if OMPI_HAVE_CXX_EXCEPTION_SUPPORT
37 throw(MPI::Exception(*errcode));
38 #else
39 // Ick. This is really ugly, but necesary if someone uses a C compiler
40 // and -lmpi++ (which can legally happen in the LAM MPI implementation,
41 // and probably in MPICH and others who include -lmpi++ by default in their
42 // wrapper compilers)
43 fprintf(stderr, "MPI 2 C++ exception throwing is disabled, MPI::mpi_errno has the error code\n");
44 MPI::mpi_errno = *errcode;
45 #endif
46 }
47
48 extern "C"
49 void ompi_mpi_cxx_comm_throw_excptn_fctn(MPI_Comm *, int *errcode, ...)
50 {
51 /* Portland compiler raises a warning if va_start is not used in a
52 * variable argument function */
53 va_list ap;
54 va_start(ap, errcode);
55 ompi_mpi_cxx_throw_exception(errcode);
56 va_end(ap);
57 }
58
59 extern "C"
60 void ompi_mpi_cxx_file_throw_excptn_fctn(MPI_File *, int *errcode, ...)
61 {
62 va_list ap;
63 va_start(ap, errcode);
64 ompi_mpi_cxx_throw_exception(errcode);
65 va_end(ap);
66 }
67
68 extern "C"
69 void ompi_mpi_cxx_win_throw_excptn_fctn(MPI_Win *, int *errcode, ...)
70 {
71 va_list ap;
72 va_start(ap, errcode);
73 ompi_mpi_cxx_throw_exception(errcode);
74 va_end(ap);
75 }
76
77
78 void
79 MPI::InitializeIntercepts()
80 {
81 ompi_cxx_errhandler_set_callbacks ((struct ompi_errhandler_t *) &ompi_mpi_errors_throw_exceptions,
82 ompi_mpi_cxx_comm_throw_excptn_fctn,
83 ompi_mpi_cxx_file_throw_excptn_fctn,
84 ompi_mpi_cxx_win_throw_excptn_fctn);
85 }
86
87
88 // This function uses OMPI types, and is invoked with C linkage for
89 // the express purpose of having a C++ entity call back the C++
90 // function (so that types can be converted, etc.).
91 extern "C"
92 void ompi_mpi_cxx_comm_errhandler_invoke(MPI_Comm *c_comm, int *err,
93 const char *message, void *comm_fn)
94 {
95 // MPI::Comm is an abstract base class; can't instantiate one of
96 // those. So fake it by instantiating an MPI::Intracomm and then
97 // casting it down to an (MPI::Comm&) when invoking the callback.
98 MPI::Intracomm cxx_comm(*c_comm);
99 MPI::Comm::Errhandler_function *cxx_fn =
100 (MPI::Comm::Errhandler_function*) comm_fn;
101
102 cxx_fn((MPI::Comm&) cxx_comm, err, message);
103 }
104
105 // This function uses OMPI types, and is invoked with C linkage for
106 // the express purpose of having a C++ entity call back the C++
107 // function (so that types can be converted, etc.).
108 extern "C"
109 void ompi_mpi_cxx_file_errhandler_invoke(MPI_File *c_file, int *err,
110 const char *message, void *file_fn)
111 {
112 MPI::File cxx_file(*c_file);
113 MPI::File::Errhandler_function *cxx_fn =
114 (MPI::File::Errhandler_function*) file_fn;
115
116 cxx_fn(cxx_file, err, message);
117 }
118
119 // This function uses OMPI types, and is invoked with C linkage for
120 // the express purpose of having a C++ entity call back the C++
121 // function (so that types can be converted, etc.).
122 extern "C"
123 void ompi_mpi_cxx_win_errhandler_invoke(MPI_Win *c_win, int *err,
124 const char *message, void *win_fn)
125 {
126 MPI::Win cxx_win(*c_win);
127 MPI::Win::Errhandler_function *cxx_fn =
128 (MPI::Win::Errhandler_function*) win_fn;
129
130 cxx_fn(cxx_win, err, message);
131 }
132
133 // This is a bit weird; bear with me. The user-supplied function for
134 // MPI::Op contains a C++ object reference. So it must be called from
135 // a C++-compiled function. However, libmpi does not contain any C++
136 // code because there are portability and bootstrapping issues
137 // involved if someone tries to make a 100% C application link against
138 // a libmpi that contains C++ code. At a minimum, the user will have
139 // to use the C++ compiler to link. LA-MPI has shown that users don't
140 // want to do this (there are other problems, but this one is easy to
141 // cite).
142 //
143 // Hence, there are two problems when trying to invoke the user's
144 // callback funcion from an MPI::Op:
145 //
146 // 1. The MPI_Datatype that the C library has must be converted to an
147 // (MPI::Datatype)
148 // 2. The C++ callback function must then be called with a
149 // (MPI::Datatype&)
150 //
151 // Some relevant facts for the discussion:
152 //
153 // - The main engine for invoking Op callback functions is in libmpi
154 // (i.e., in C code).
155 //
156 // - The C++ bindings are a thin layer on top of the C bindings.
157 //
158 // - The C++ bindings are a separate library from the C bindings
159 // (libmpi_cxx.la).
160 //
161 // - As a direct result, the mpiCC wrapper compiler must generate a
162 // link order thus: "... -lmpi_cxx -lmpi ...", meaning that we cannot
163 // have a direct function call from the libmpi to libmpi_cxx. We can
164 // only do it by function pointer.
165 //
166 // So the problem remains -- how to invoke a C++ MPI::Op callback
167 // function (which only occurrs for user-defined datatypes, BTW) from
168 // within the C Op callback engine in libmpi?
169 //
170 // It is easy to cache a function pointer to the
171 // ompi_mpi_cxx_op_intercept() function on the MPI_Op (that is located
172 // in the libmpi_cxx library, and is therefore compiled with a C++
173 // compiler). But the normal C callback MPI_User_function type
174 // signature is (void*, void*, int*, MPI_Datatype*) -- so if
175 // ompi_mpi_cxx_op_intercept() is invoked with these arguments, it has
176 // no way to deduce what the user-specified callback function is that
177 // is associated with the MPI::Op.
178 //
179 // One can easily imagine a scenario of caching the callback pointer
180 // of the current MPI::Op in a global variable somewhere, and when
181 // ompi_mpi_cxx_op_intercept() is invoked, simply use that global
182 // variable. This is unfortunately not thread safe.
183 //
184 // So what we do is as follows:
185 //
186 // 1. The C++ dispatch function ompi_mpi_cxx_op_intercept() is *not*
187 // of type (MPI_User_function*). More specifically, it takes an
188 // additional argument: a function pointer. its signature is (void*,
189 // void*, int*, MPI_Datatype*, MPI_Op*, MPI::User_function*). This
190 // last argument is the function pointer of the user callback function
191 // to be invoked.
192 //
193 // The careful reader will notice that it is impossible for the C Op
194 // dispatch code in libmpi to call this function properly because the
195 // last argument is of a type that is not defined in libmpi (i.e.,
196 // it's only in libmpi_cxx). Keep reading -- this is explained below.
197 //
198 // 2. When the MPI::Op is created (in MPI::Op::Init()), we call the
199 // back-end C MPI_Op_create() function as normal (just like the F77
200 // bindings, in fact), and pass it the ompi_mpi_cxx_op_intercept()
201 // function (casting it to (MPI_User_function*) -- it's a function
202 // pointer, so its size is guaranteed to be the same, even if the
203 // signature of the real function is different).
204 //
205 // 3. The function pointer to ompi_mpi_cxx_op_intercept() will be
206 // cached in the MPI_Op in op->o_func[0].cxx_intercept_fn.
207 //
208 // Recall that MPI_Op is implemented to have an array of function
209 // pointers so that optimized versions of reduction operations can be
210 // invoked based on the corresponding datatype. But when an MPI_Op
211 // represents a user-defined function operation, there is only one
212 // function, so it is always stored in function pointer array index 0.
213 //
214 // 4. When MPI_Op_create() returns, the C++ MPI::Op::Init function
215 // manually sets OMPI_OP_FLAGS_CXX_FUNC flag on the resulting MPI_Op
216 // (again, very similar to the F77 MPI_OP_CREATE wrapper). It also
217 // caches the user's C++ callback function in op->o_func[1].c_fn
218 // (recall that the array of function pointers is actually a union of
219 // multiple different function pointer types -- it doesn't matter
220 // which type the user's callback function pointer is stored in; since
221 // all the types in the union are function pointers, it's guaranteed
222 // to be large enough to hold what we need.
223 //
224 // Note that we don't have a member of the union for the C++ callback
225 // function because its signature includes a (MPI::Datatype&), which
226 // we can't put in the C library libmpi.
227 //
228 // 5. When the user invokes an function that uses the MPI::Op (or,
229 // more specifically, when the Op dispatch engine in ompi/op/op.c [in
230 // libmpi] tries to dispatch off to it), it will see the
231 // OMPI_OP_FLAGS_CXX_FUNC flag and know to use the
232 // op->o_func[0].cxx_intercept_fn and also pass as the 4th argument,
233 // op->o_func[1].c_fn.
234 //
235 // 6. ompi_mpi_cxx_op_intercept() is therefore invoked and receives
236 // both the (MPI_Datatype*) (which is easy to convert to
237 // (MPI::Datatype&)) and a pointer to the user's C++ callback function
238 // (albiet cast as the wrong type). So it casts the callback function
239 // pointer to (MPI::User_function*) and invokes it.
240 //
241 // Wasn't that simple?
242 //
243 extern "C" void
244 ompi_mpi_cxx_op_intercept(void *invec, void *outvec, int *len,
245 MPI_Datatype *datatype, MPI_User_function *c_fn)
246 {
247 MPI::Datatype cxx_datatype = *datatype;
248 MPI::User_function *cxx_callback = (MPI::User_function*) c_fn;
249 cxx_callback(invec, outvec, *len, cxx_datatype);
250 }
251
252 //
253 // Attribute copy functions -- comm, type, and win
254 //
255 extern "C" int
256 ompi_mpi_cxx_comm_copy_attr_intercept(MPI_Comm comm, int keyval,
257 void *extra_state,
258 void *attribute_val_in,
259 void *attribute_val_out, int *flag,
260 MPI_Comm newcomm)
261 {
262 int ret = 0;
263 MPI::Comm::keyval_intercept_data_t *kid =
264 (MPI::Comm::keyval_intercept_data_t*) extra_state;
265
266 // The callback may be in C or C++. If it's in C, it's easy - just
267 // call it with no extra C++ machinery.
268
269 if (NULL != kid->c_copy_fn) {
270 return kid->c_copy_fn(comm, keyval, kid->extra_state, attribute_val_in,
271 attribute_val_out, flag);
272 }
273
274 // If the callback was C++, we have to do a little more work
275
276 MPI::Intracomm intracomm;
277 MPI::Intercomm intercomm;
278 MPI::Graphcomm graphcomm;
279 MPI::Cartcomm cartcomm;
280
281 bool bflag = OPAL_INT_TO_BOOL(*flag);
282
283 if (NULL != kid->cxx_copy_fn) {
284 ompi_cxx_communicator_type_t comm_type =
285 ompi_cxx_comm_get_type (comm);
286 switch (comm_type) {
287 case OMPI_CXX_COMM_TYPE_GRAPH:
288 graphcomm = MPI::Graphcomm(comm);
289 ret = kid->cxx_copy_fn(graphcomm, keyval, kid->extra_state,
290 attribute_val_in, attribute_val_out,
291 bflag);
292 break;
293 case OMPI_CXX_COMM_TYPE_CART:
294 cartcomm = MPI::Cartcomm(comm);
295 ret = kid->cxx_copy_fn(cartcomm, keyval, kid->extra_state,
296 attribute_val_in, attribute_val_out,
297 bflag);
298 break;
299 case OMPI_CXX_COMM_TYPE_INTRACOMM:
300 intracomm = MPI::Intracomm(comm);
301 ret = kid->cxx_copy_fn(intracomm, keyval, kid->extra_state,
302 attribute_val_in, attribute_val_out,
303 bflag);
304 break;
305 case OMPI_CXX_COMM_TYPE_INTERCOMM:
306 intercomm = MPI::Intercomm(comm);
307 ret = kid->cxx_copy_fn(intercomm, keyval, kid->extra_state,
308 attribute_val_in, attribute_val_out,
309 bflag);
310 break;
311 default:
312 ret = MPI::ERR_COMM;
313 }
314 } else {
315 ret = MPI::ERR_OTHER;
316 }
317
318 *flag = (int)bflag;
319 return ret;
320 }
321
322 extern "C" int
323 ompi_mpi_cxx_comm_delete_attr_intercept(MPI_Comm comm, int keyval,
324 void *attribute_val, void *extra_state)
325 {
326 int ret = 0;
327 MPI::Comm::keyval_intercept_data_t *kid =
328 (MPI::Comm::keyval_intercept_data_t*) extra_state;
329
330 // The callback may be in C or C++. If it's in C, it's easy - just
331 // call it with no extra C++ machinery.
332
333 if (NULL != kid->c_delete_fn) {
334 return kid->c_delete_fn(comm, keyval, attribute_val, kid->extra_state);
335 }
336
337 // If the callback was C++, we have to do a little more work
338
339 MPI::Intracomm intracomm;
340 MPI::Intercomm intercomm;
341 MPI::Graphcomm graphcomm;
342 MPI::Cartcomm cartcomm;
343
344 if (NULL != kid->cxx_delete_fn) {
345 ompi_cxx_communicator_type_t comm_type =
346 ompi_cxx_comm_get_type (comm);
347 switch (comm_type) {
348 case OMPI_CXX_COMM_TYPE_GRAPH:
349 graphcomm = MPI::Graphcomm(comm);
350 ret = kid->cxx_delete_fn(graphcomm, keyval, attribute_val,
351 kid->extra_state);
352 break;
353 case OMPI_CXX_COMM_TYPE_CART:
354 cartcomm = MPI::Cartcomm(comm);
355 ret = kid->cxx_delete_fn(cartcomm, keyval, attribute_val,
356 kid->extra_state);
357 break;
358 case OMPI_CXX_COMM_TYPE_INTRACOMM:
359 intracomm = MPI::Intracomm(comm);
360 ret = kid->cxx_delete_fn(intracomm, keyval, attribute_val,
361 kid->extra_state);
362 break;
363 case OMPI_CXX_COMM_TYPE_INTERCOMM:
364 intercomm = MPI::Intercomm(comm);
365 ret = kid->cxx_delete_fn(intercomm, keyval, attribute_val,
366 kid->extra_state);
367 break;
368 default:
369 ret = MPI::ERR_COMM;
370 }
371 } else {
372 ret = MPI::ERR_OTHER;
373 }
374
375 return ret;
376 }
377
378 extern "C" int
379 ompi_mpi_cxx_type_copy_attr_intercept(MPI_Datatype oldtype, int keyval,
380 void *extra_state, void *attribute_val_in,
381 void *attribute_val_out, int *flag)
382 {
383 int ret = 0;
384 MPI::Datatype::keyval_intercept_data_t *kid =
385 (MPI::Datatype::keyval_intercept_data_t*) extra_state;
386
387
388 if (NULL != kid->c_copy_fn) {
389 // The callback may be in C or C++. If it's in C, it's easy - just
390 // call it with no extra C++ machinery.
391 ret = kid->c_copy_fn(oldtype, keyval, kid->extra_state, attribute_val_in,
392 attribute_val_out, flag);
393 } else if (NULL != kid->cxx_copy_fn) {
394 // If the callback was C++, we have to do a little more work
395 bool bflag = OPAL_INT_TO_BOOL(*flag);
396 MPI::Datatype cxx_datatype(oldtype);
397 ret = kid->cxx_copy_fn(cxx_datatype, keyval, kid->extra_state,
398 attribute_val_in, attribute_val_out, bflag);
399 *flag = (int)bflag;
400 } else {
401 ret = MPI::ERR_TYPE;
402 }
403
404 return ret;
405 }
406
407 extern "C" int
408 ompi_mpi_cxx_type_delete_attr_intercept(MPI_Datatype type, int keyval,
409 void *attribute_val, void *extra_state)
410 {
411 int ret = 0;
412 MPI::Datatype::keyval_intercept_data_t *kid =
413 (MPI::Datatype::keyval_intercept_data_t*) extra_state;
414
415 if (NULL != kid->c_delete_fn) {
416 return kid->c_delete_fn(type, keyval, attribute_val, kid->extra_state);
417 } else if (NULL != kid->cxx_delete_fn) {
418 MPI::Datatype cxx_datatype(type);
419 return kid->cxx_delete_fn(cxx_datatype, keyval, attribute_val,
420 kid->extra_state);
421 } else {
422 ret = MPI::ERR_TYPE;
423 }
424
425 return ret;
426 }
427
428 extern "C" int
429 ompi_mpi_cxx_win_copy_attr_intercept(MPI_Win oldwin, int keyval,
430 void *extra_state, void *attribute_val_in,
431 void *attribute_val_out, int *flag)
432 {
433 int ret = 0;
434 MPI::Win::keyval_intercept_data_t *kid =
435 (MPI::Win::keyval_intercept_data_t*) extra_state;
436
437 if (NULL != kid->c_copy_fn) {
438 // The callback may be in C or C++. If it's in C, it's easy - just
439 // call it with no extra C++ machinery.
440 ret = kid->c_copy_fn(oldwin, keyval, kid->extra_state, attribute_val_in,
441 attribute_val_out, flag);
442 } else if (NULL != kid->cxx_copy_fn) {
443 // If the callback was C++, we have to do a little more work
444 bool bflag = OPAL_INT_TO_BOOL(*flag);
445 MPI::Win cxx_win(oldwin);
446 ret = kid->cxx_copy_fn(cxx_win, keyval, kid->extra_state,
447 attribute_val_in, attribute_val_out, bflag);
448 *flag = (int)bflag;
449 } else {
450 ret = MPI::ERR_WIN;
451 }
452
453 return ret;
454 }
455
456 extern "C" int
457 ompi_mpi_cxx_win_delete_attr_intercept(MPI_Win win, int keyval,
458 void *attribute_val, void *extra_state)
459 {
460 int ret = 0;
461 MPI::Win::keyval_intercept_data_t *kid =
462 (MPI::Win::keyval_intercept_data_t*) extra_state;
463
464 if (NULL != kid->c_delete_fn) {
465 return kid->c_delete_fn(win, keyval, attribute_val, kid->extra_state);
466 } else if (NULL != kid->cxx_delete_fn) {
467 MPI::Win cxx_win(win);
468 return kid->cxx_delete_fn(cxx_win, keyval, attribute_val,
469 kid->extra_state);
470 } else {
471 ret = MPI::ERR_WIN;
472 }
473
474 return ret;
475 }
476
477 // For similar reasons as above, we need to intercept calls for the 3
478 // generalized request callbacks (convert arguments to C++ types and
479 // invoke the C++ callback signature).
480
481 extern "C" int
482 ompi_mpi_cxx_grequest_query_fn_intercept(void *state, MPI_Status *status)
483 {
484 MPI::Grequest::Intercept_data_t *data =
485 (MPI::Grequest::Intercept_data_t *) state;
486
487 MPI::Status s(*status);
488 int ret = data->id_cxx_query_fn(data->id_extra, s);
489 *status = s;
490 return ret;
491 }
492
493 extern "C" int
494 ompi_mpi_cxx_grequest_free_fn_intercept(void *state)
495 {
496 MPI::Grequest::Intercept_data_t *data =
497 (MPI::Grequest::Intercept_data_t *) state;
498 int ret = data->id_cxx_free_fn(data->id_extra);
499 // Delete the struct that was "new"ed in MPI::Grequest::Start()
500 delete data;
501 return ret;
502 }
503
504 extern "C" int
505 ompi_mpi_cxx_grequest_cancel_fn_intercept(void *state, int cancelled)
506 {
507 MPI::Grequest::Intercept_data_t *data =
508 (MPI::Grequest::Intercept_data_t *) state;
509 return data->id_cxx_cancel_fn(data->id_extra,
510 (0 != cancelled ? true : false));
511 }