root/ompi/mpi/cxx/intercepts.cc

/* [<][>][^][v][top][bottom][index][help] */

DEFINITIONS

This source file includes following definitions.
  1. ompi_mpi_cxx_throw_exception
  2. ompi_mpi_cxx_comm_throw_excptn_fctn
  3. ompi_mpi_cxx_file_throw_excptn_fctn
  4. ompi_mpi_cxx_win_throw_excptn_fctn
  5. InitializeIntercepts
  6. ompi_mpi_cxx_comm_errhandler_invoke
  7. ompi_mpi_cxx_file_errhandler_invoke
  8. ompi_mpi_cxx_win_errhandler_invoke
  9. ompi_mpi_cxx_op_intercept
  10. ompi_mpi_cxx_comm_copy_attr_intercept
  11. ompi_mpi_cxx_comm_delete_attr_intercept
  12. ompi_mpi_cxx_type_copy_attr_intercept
  13. ompi_mpi_cxx_type_delete_attr_intercept
  14. ompi_mpi_cxx_win_copy_attr_intercept
  15. ompi_mpi_cxx_win_delete_attr_intercept
  16. ompi_mpi_cxx_grequest_query_fn_intercept
  17. ompi_mpi_cxx_grequest_free_fn_intercept
  18. ompi_mpi_cxx_grequest_cancel_fn_intercept

   1 // -*- c++ -*-
   2 //
   3 // Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
   4 //                         University Research and Technology
   5 //                         Corporation.  All rights reserved.
   6 // Copyright (c) 2004-2005 The University of Tennessee and The University
   7 //                         of Tennessee Research Foundation.  All rights
   8 //                         reserved.
   9 // Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
  10 //                         University of Stuttgart.  All rights reserved.
  11 // Copyright (c) 2004-2005 The Regents of the University of California.
  12 //                         All rights reserved.
  13 // Copyright (c) 2006-2009 Cisco Systems, Inc.  All rights reserved.
  14 // Copyright (c) 2009      Sun Microsystems, Inc.  All rights reserved.
  15 // Copyright (c) 2016      Los Alamos National Security, LLC. All rights
  16 //                         reserved.
  17 // Copyright (c) 2017      Research Organization for Information Science
  18 //                         and Technology (RIST). All rights reserved.
  19 // $COPYRIGHT$
  20 //
  21 // Additional copyrights may follow
  22 //
  23 // $HEADER$
  24 //
  25 
  26 
  27 #include "mpicxx.h"
  28 #include <cstdio>
  29 
  30 #include "ompi_config.h"
  31 #include "cxx_glue.h"
  32 
  33 extern "C"
  34 void ompi_mpi_cxx_throw_exception(int *errcode)
  35 {
  36 #if OMPI_HAVE_CXX_EXCEPTION_SUPPORT
  37     throw(MPI::Exception(*errcode));
  38 #else
  39   // Ick.  This is really ugly, but necesary if someone uses a C compiler
  40   // and -lmpi++ (which can legally happen in the LAM MPI implementation,
  41   // and probably in MPICH and others who include -lmpi++ by default in their
  42   // wrapper compilers)
  43   fprintf(stderr, "MPI 2 C++ exception throwing is disabled, MPI::mpi_errno has the error code\n");
  44   MPI::mpi_errno = *errcode;
  45 #endif
  46 }
  47 
  48 extern "C"
  49 void ompi_mpi_cxx_comm_throw_excptn_fctn(MPI_Comm *, int *errcode, ...)
  50 {
  51     /* Portland compiler raises a warning if va_start is not used in a
  52      * variable argument function */
  53     va_list ap;
  54     va_start(ap, errcode);
  55     ompi_mpi_cxx_throw_exception(errcode);
  56     va_end(ap);
  57 }
  58 
  59 extern "C"
  60 void ompi_mpi_cxx_file_throw_excptn_fctn(MPI_File *, int *errcode, ...)
  61 {
  62     va_list ap;
  63     va_start(ap, errcode);
  64     ompi_mpi_cxx_throw_exception(errcode);
  65     va_end(ap);
  66 }
  67 
  68 extern "C"
  69 void ompi_mpi_cxx_win_throw_excptn_fctn(MPI_Win *, int *errcode, ...)
  70 {
  71     va_list ap;
  72     va_start(ap, errcode);
  73     ompi_mpi_cxx_throw_exception(errcode);
  74     va_end(ap);
  75 }
  76 
  77 
  78 void
  79 MPI::InitializeIntercepts()
  80 {
  81     ompi_cxx_errhandler_set_callbacks ((struct ompi_errhandler_t *) &ompi_mpi_errors_throw_exceptions,
  82                                        ompi_mpi_cxx_comm_throw_excptn_fctn,
  83                                        ompi_mpi_cxx_file_throw_excptn_fctn,
  84                                        ompi_mpi_cxx_win_throw_excptn_fctn);
  85 }
  86 
  87 
  88 // This function uses OMPI types, and is invoked with C linkage for
  89 // the express purpose of having a C++ entity call back the C++
  90 // function (so that types can be converted, etc.).
  91 extern "C"
  92 void ompi_mpi_cxx_comm_errhandler_invoke(MPI_Comm *c_comm, int *err,
  93                                          const char *message, void *comm_fn)
  94 {
  95     // MPI::Comm is an abstract base class; can't instantiate one of
  96     // those.  So fake it by instantiating an MPI::Intracomm and then
  97     // casting it down to an (MPI::Comm&) when invoking the callback.
  98     MPI::Intracomm cxx_comm(*c_comm);
  99     MPI::Comm::Errhandler_function *cxx_fn =
 100         (MPI::Comm::Errhandler_function*) comm_fn;
 101 
 102     cxx_fn((MPI::Comm&) cxx_comm, err, message);
 103 }
 104 
 105 // This function uses OMPI types, and is invoked with C linkage for
 106 // the express purpose of having a C++ entity call back the C++
 107 // function (so that types can be converted, etc.).
 108 extern "C"
 109 void ompi_mpi_cxx_file_errhandler_invoke(MPI_File *c_file, int *err,
 110                                          const char *message, void *file_fn)
 111 {
 112     MPI::File cxx_file(*c_file);
 113     MPI::File::Errhandler_function *cxx_fn =
 114         (MPI::File::Errhandler_function*) file_fn;
 115 
 116     cxx_fn(cxx_file, err, message);
 117 }
 118 
 119 // This function uses OMPI types, and is invoked with C linkage for
 120 // the express purpose of having a C++ entity call back the C++
 121 // function (so that types can be converted, etc.).
 122 extern "C"
 123 void ompi_mpi_cxx_win_errhandler_invoke(MPI_Win *c_win, int *err,
 124                                         const char *message, void *win_fn)
 125 {
 126     MPI::Win cxx_win(*c_win);
 127     MPI::Win::Errhandler_function *cxx_fn =
 128         (MPI::Win::Errhandler_function*) win_fn;
 129 
 130     cxx_fn(cxx_win, err, message);
 131 }
 132 
 133 // This is a bit weird; bear with me.  The user-supplied function for
 134 // MPI::Op contains a C++ object reference.  So it must be called from
 135 // a C++-compiled function.  However, libmpi does not contain any C++
 136 // code because there are portability and bootstrapping issues
 137 // involved if someone tries to make a 100% C application link against
 138 // a libmpi that contains C++ code.  At a minimum, the user will have
 139 // to use the C++ compiler to link.  LA-MPI has shown that users don't
 140 // want to do this (there are other problems, but this one is easy to
 141 // cite).
 142 //
 143 // Hence, there are two problems when trying to invoke the user's
 144 // callback funcion from an MPI::Op:
 145 //
 146 // 1. The MPI_Datatype that the C library has must be converted to an
 147 // (MPI::Datatype)
 148 // 2. The C++ callback function must then be called with a
 149 // (MPI::Datatype&)
 150 //
 151 // Some relevant facts for the discussion:
 152 //
 153 // - The main engine for invoking Op callback functions is in libmpi
 154 // (i.e., in C code).
 155 //
 156 // - The C++ bindings are a thin layer on top of the C bindings.
 157 //
 158 // - The C++ bindings are a separate library from the C bindings
 159 // (libmpi_cxx.la).
 160 //
 161 // - As a direct result, the mpiCC wrapper compiler must generate a
 162 // link order thus: "... -lmpi_cxx -lmpi ...", meaning that we cannot
 163 // have a direct function call from the libmpi to libmpi_cxx.  We can
 164 // only do it by function pointer.
 165 //
 166 // So the problem remains -- how to invoke a C++ MPI::Op callback
 167 // function (which only occurrs for user-defined datatypes, BTW) from
 168 // within the C Op callback engine in libmpi?
 169 //
 170 // It is easy to cache a function pointer to the
 171 // ompi_mpi_cxx_op_intercept() function on the MPI_Op (that is located
 172 // in the libmpi_cxx library, and is therefore compiled with a C++
 173 // compiler).  But the normal C callback MPI_User_function type
 174 // signature is (void*, void*, int*, MPI_Datatype*) -- so if
 175 // ompi_mpi_cxx_op_intercept() is invoked with these arguments, it has
 176 // no way to deduce what the user-specified callback function is that
 177 // is associated with the MPI::Op.
 178 //
 179 // One can easily imagine a scenario of caching the callback pointer
 180 // of the current MPI::Op in a global variable somewhere, and when
 181 // ompi_mpi_cxx_op_intercept() is invoked, simply use that global
 182 // variable.  This is unfortunately not thread safe.
 183 //
 184 // So what we do is as follows:
 185 //
 186 // 1. The C++ dispatch function ompi_mpi_cxx_op_intercept() is *not*
 187 // of type (MPI_User_function*).  More specifically, it takes an
 188 // additional argument: a function pointer.  its signature is (void*,
 189 // void*, int*, MPI_Datatype*, MPI_Op*, MPI::User_function*).  This
 190 // last argument is the function pointer of the user callback function
 191 // to be invoked.
 192 //
 193 // The careful reader will notice that it is impossible for the C Op
 194 // dispatch code in libmpi to call this function properly because the
 195 // last argument is of a type that is not defined in libmpi (i.e.,
 196 // it's only in libmpi_cxx).  Keep reading -- this is explained below.
 197 //
 198 // 2. When the MPI::Op is created (in MPI::Op::Init()), we call the
 199 // back-end C MPI_Op_create() function as normal (just like the F77
 200 // bindings, in fact), and pass it the ompi_mpi_cxx_op_intercept()
 201 // function (casting it to (MPI_User_function*) -- it's a function
 202 // pointer, so its size is guaranteed to be the same, even if the
 203 // signature of the real function is different).
 204 //
 205 // 3. The function pointer to ompi_mpi_cxx_op_intercept() will be
 206 // cached in the MPI_Op in op->o_func[0].cxx_intercept_fn.
 207 //
 208 // Recall that MPI_Op is implemented to have an array of function
 209 // pointers so that optimized versions of reduction operations can be
 210 // invoked based on the corresponding datatype.  But when an MPI_Op
 211 // represents a user-defined function operation, there is only one
 212 // function, so it is always stored in function pointer array index 0.
 213 //
 214 // 4. When MPI_Op_create() returns, the C++ MPI::Op::Init function
 215 // manually sets OMPI_OP_FLAGS_CXX_FUNC flag on the resulting MPI_Op
 216 // (again, very similar to the F77 MPI_OP_CREATE wrapper).  It also
 217 // caches the user's C++ callback function in op->o_func[1].c_fn
 218 // (recall that the array of function pointers is actually a union of
 219 // multiple different function pointer types -- it doesn't matter
 220 // which type the user's callback function pointer is stored in; since
 221 // all the types in the union are function pointers, it's guaranteed
 222 // to be large enough to hold what we need.
 223 //
 224 // Note that we don't have a member of the union for the C++ callback
 225 // function because its signature includes a (MPI::Datatype&), which
 226 // we can't put in the C library libmpi.
 227 //
 228 // 5. When the user invokes an function that uses the MPI::Op (or,
 229 // more specifically, when the Op dispatch engine in ompi/op/op.c [in
 230 // libmpi] tries to dispatch off to it), it will see the
 231 // OMPI_OP_FLAGS_CXX_FUNC flag and know to use the
 232 // op->o_func[0].cxx_intercept_fn and also pass as the 4th argument,
 233 // op->o_func[1].c_fn.
 234 //
 235 // 6. ompi_mpi_cxx_op_intercept() is therefore invoked and receives
 236 // both the (MPI_Datatype*) (which is easy to convert to
 237 // (MPI::Datatype&)) and a pointer to the user's C++ callback function
 238 // (albiet cast as the wrong type).  So it casts the callback function
 239 // pointer to (MPI::User_function*) and invokes it.
 240 //
 241 // Wasn't that simple?
 242 //
 243 extern "C" void
 244 ompi_mpi_cxx_op_intercept(void *invec, void *outvec, int *len,
 245                           MPI_Datatype *datatype, MPI_User_function *c_fn)
 246 {
 247     MPI::Datatype cxx_datatype = *datatype;
 248     MPI::User_function *cxx_callback = (MPI::User_function*) c_fn;
 249     cxx_callback(invec, outvec, *len, cxx_datatype);
 250 }
 251 
 252 //
 253 // Attribute copy functions -- comm, type, and win
 254 //
 255 extern "C" int
 256 ompi_mpi_cxx_comm_copy_attr_intercept(MPI_Comm comm, int keyval,
 257                                       void *extra_state,
 258                                       void *attribute_val_in,
 259                                       void *attribute_val_out, int *flag,
 260                                       MPI_Comm newcomm)
 261 {
 262   int ret = 0;
 263   MPI::Comm::keyval_intercept_data_t *kid =
 264       (MPI::Comm::keyval_intercept_data_t*) extra_state;
 265 
 266   // The callback may be in C or C++.  If it's in C, it's easy - just
 267   // call it with no extra C++ machinery.
 268 
 269   if (NULL != kid->c_copy_fn) {
 270       return kid->c_copy_fn(comm, keyval, kid->extra_state, attribute_val_in,
 271                             attribute_val_out, flag);
 272   }
 273 
 274   // If the callback was C++, we have to do a little more work
 275 
 276   MPI::Intracomm intracomm;
 277   MPI::Intercomm intercomm;
 278   MPI::Graphcomm graphcomm;
 279   MPI::Cartcomm cartcomm;
 280 
 281   bool bflag = OPAL_INT_TO_BOOL(*flag);
 282 
 283   if (NULL != kid->cxx_copy_fn) {
 284       ompi_cxx_communicator_type_t comm_type =
 285           ompi_cxx_comm_get_type (comm);
 286       switch (comm_type) {
 287       case OMPI_CXX_COMM_TYPE_GRAPH:
 288           graphcomm = MPI::Graphcomm(comm);
 289           ret = kid->cxx_copy_fn(graphcomm, keyval, kid->extra_state,
 290                                  attribute_val_in, attribute_val_out,
 291                                  bflag);
 292           break;
 293       case OMPI_CXX_COMM_TYPE_CART:
 294           cartcomm = MPI::Cartcomm(comm);
 295           ret = kid->cxx_copy_fn(cartcomm, keyval, kid->extra_state,
 296                                  attribute_val_in, attribute_val_out,
 297                                  bflag);
 298           break;
 299       case OMPI_CXX_COMM_TYPE_INTRACOMM:
 300           intracomm = MPI::Intracomm(comm);
 301           ret = kid->cxx_copy_fn(intracomm, keyval, kid->extra_state,
 302                                  attribute_val_in, attribute_val_out,
 303                                  bflag);
 304           break;
 305       case OMPI_CXX_COMM_TYPE_INTERCOMM:
 306           intercomm = MPI::Intercomm(comm);
 307           ret = kid->cxx_copy_fn(intercomm, keyval, kid->extra_state,
 308                                  attribute_val_in, attribute_val_out,
 309                                  bflag);
 310           break;
 311       default:
 312           ret = MPI::ERR_COMM;
 313       }
 314   } else {
 315       ret = MPI::ERR_OTHER;
 316   }
 317 
 318   *flag = (int)bflag;
 319   return ret;
 320 }
 321 
 322 extern "C" int
 323 ompi_mpi_cxx_comm_delete_attr_intercept(MPI_Comm comm, int keyval,
 324                                         void *attribute_val, void *extra_state)
 325 {
 326   int ret = 0;
 327   MPI::Comm::keyval_intercept_data_t *kid =
 328       (MPI::Comm::keyval_intercept_data_t*) extra_state;
 329 
 330   // The callback may be in C or C++.  If it's in C, it's easy - just
 331   // call it with no extra C++ machinery.
 332 
 333   if (NULL != kid->c_delete_fn) {
 334       return kid->c_delete_fn(comm, keyval, attribute_val, kid->extra_state);
 335   }
 336 
 337   // If the callback was C++, we have to do a little more work
 338 
 339   MPI::Intracomm intracomm;
 340   MPI::Intercomm intercomm;
 341   MPI::Graphcomm graphcomm;
 342   MPI::Cartcomm cartcomm;
 343 
 344   if (NULL != kid->cxx_delete_fn) {
 345       ompi_cxx_communicator_type_t comm_type =
 346           ompi_cxx_comm_get_type (comm);
 347       switch (comm_type) {
 348       case OMPI_CXX_COMM_TYPE_GRAPH:
 349           graphcomm = MPI::Graphcomm(comm);
 350           ret = kid->cxx_delete_fn(graphcomm, keyval, attribute_val,
 351                                    kid->extra_state);
 352           break;
 353       case OMPI_CXX_COMM_TYPE_CART:
 354           cartcomm = MPI::Cartcomm(comm);
 355           ret = kid->cxx_delete_fn(cartcomm, keyval, attribute_val,
 356                                    kid->extra_state);
 357           break;
 358       case OMPI_CXX_COMM_TYPE_INTRACOMM:
 359           intracomm = MPI::Intracomm(comm);
 360           ret = kid->cxx_delete_fn(intracomm, keyval, attribute_val,
 361                                    kid->extra_state);
 362           break;
 363       case OMPI_CXX_COMM_TYPE_INTERCOMM:
 364           intercomm = MPI::Intercomm(comm);
 365           ret = kid->cxx_delete_fn(intercomm, keyval, attribute_val,
 366                                    kid->extra_state);
 367           break;
 368       default:
 369           ret = MPI::ERR_COMM;
 370       }
 371   } else {
 372       ret = MPI::ERR_OTHER;
 373   }
 374 
 375   return ret;
 376 }
 377 
 378 extern "C" int
 379 ompi_mpi_cxx_type_copy_attr_intercept(MPI_Datatype oldtype, int keyval,
 380                                       void *extra_state, void *attribute_val_in,
 381                                       void *attribute_val_out, int *flag)
 382 {
 383   int ret = 0;
 384   MPI::Datatype::keyval_intercept_data_t *kid =
 385       (MPI::Datatype::keyval_intercept_data_t*) extra_state;
 386 
 387 
 388   if (NULL != kid->c_copy_fn) {
 389       // The callback may be in C or C++.  If it's in C, it's easy - just
 390       // call it with no extra C++ machinery.
 391       ret = kid->c_copy_fn(oldtype, keyval, kid->extra_state, attribute_val_in,
 392                            attribute_val_out, flag);
 393   } else if (NULL != kid->cxx_copy_fn) {
 394       // If the callback was C++, we have to do a little more work
 395       bool bflag = OPAL_INT_TO_BOOL(*flag);
 396       MPI::Datatype cxx_datatype(oldtype);
 397       ret = kid->cxx_copy_fn(cxx_datatype, keyval, kid->extra_state,
 398                              attribute_val_in, attribute_val_out, bflag);
 399       *flag = (int)bflag;
 400   } else {
 401     ret = MPI::ERR_TYPE;
 402   }
 403 
 404   return ret;
 405 }
 406 
 407 extern "C" int
 408 ompi_mpi_cxx_type_delete_attr_intercept(MPI_Datatype type, int keyval,
 409                                         void *attribute_val, void *extra_state)
 410 {
 411   int ret = 0;
 412   MPI::Datatype::keyval_intercept_data_t *kid =
 413       (MPI::Datatype::keyval_intercept_data_t*) extra_state;
 414 
 415   if (NULL != kid->c_delete_fn) {
 416       return kid->c_delete_fn(type, keyval, attribute_val, kid->extra_state);
 417   } else if (NULL != kid->cxx_delete_fn) {
 418       MPI::Datatype cxx_datatype(type);
 419       return kid->cxx_delete_fn(cxx_datatype, keyval, attribute_val,
 420                                 kid->extra_state);
 421   } else {
 422     ret = MPI::ERR_TYPE;
 423   }
 424 
 425   return ret;
 426 }
 427 
 428 extern "C" int
 429 ompi_mpi_cxx_win_copy_attr_intercept(MPI_Win oldwin, int keyval,
 430                                       void *extra_state, void *attribute_val_in,
 431                                       void *attribute_val_out, int *flag)
 432 {
 433   int ret = 0;
 434   MPI::Win::keyval_intercept_data_t *kid =
 435     (MPI::Win::keyval_intercept_data_t*) extra_state;
 436 
 437   if (NULL != kid->c_copy_fn) {
 438       // The callback may be in C or C++.  If it's in C, it's easy - just
 439       // call it with no extra C++ machinery.
 440       ret = kid->c_copy_fn(oldwin, keyval, kid->extra_state, attribute_val_in,
 441                            attribute_val_out, flag);
 442   } else if (NULL != kid->cxx_copy_fn) {
 443       // If the callback was C++, we have to do a little more work
 444       bool bflag = OPAL_INT_TO_BOOL(*flag);
 445       MPI::Win cxx_win(oldwin);
 446       ret = kid->cxx_copy_fn(cxx_win, keyval, kid->extra_state,
 447                              attribute_val_in, attribute_val_out, bflag);
 448       *flag = (int)bflag;
 449   } else {
 450       ret = MPI::ERR_WIN;
 451   }
 452 
 453   return ret;
 454 }
 455 
 456 extern "C" int
 457 ompi_mpi_cxx_win_delete_attr_intercept(MPI_Win win, int keyval,
 458                                         void *attribute_val, void *extra_state)
 459 {
 460   int ret = 0;
 461   MPI::Win::keyval_intercept_data_t *kid =
 462       (MPI::Win::keyval_intercept_data_t*) extra_state;
 463 
 464   if (NULL != kid->c_delete_fn) {
 465       return kid->c_delete_fn(win, keyval, attribute_val, kid->extra_state);
 466   } else if (NULL != kid->cxx_delete_fn) {
 467       MPI::Win cxx_win(win);
 468       return kid->cxx_delete_fn(cxx_win, keyval, attribute_val,
 469                                 kid->extra_state);
 470   } else {
 471       ret = MPI::ERR_WIN;
 472   }
 473 
 474   return ret;
 475 }
 476 
 477 // For similar reasons as above, we need to intercept calls for the 3
 478 // generalized request callbacks (convert arguments to C++ types and
 479 // invoke the C++ callback signature).
 480 
 481 extern "C" int
 482 ompi_mpi_cxx_grequest_query_fn_intercept(void *state, MPI_Status *status)
 483 {
 484     MPI::Grequest::Intercept_data_t *data =
 485         (MPI::Grequest::Intercept_data_t *) state;
 486 
 487     MPI::Status s(*status);
 488     int ret = data->id_cxx_query_fn(data->id_extra, s);
 489     *status = s;
 490     return ret;
 491 }
 492 
 493 extern "C" int
 494 ompi_mpi_cxx_grequest_free_fn_intercept(void *state)
 495 {
 496     MPI::Grequest::Intercept_data_t *data =
 497         (MPI::Grequest::Intercept_data_t *) state;
 498     int ret = data->id_cxx_free_fn(data->id_extra);
 499     // Delete the struct that was "new"ed in MPI::Grequest::Start()
 500     delete data;
 501     return ret;
 502 }
 503 
 504 extern "C" int
 505 ompi_mpi_cxx_grequest_cancel_fn_intercept(void *state, int cancelled)
 506 {
 507     MPI::Grequest::Intercept_data_t *data =
 508         (MPI::Grequest::Intercept_data_t *) state;
 509     return data->id_cxx_cancel_fn(data->id_extra,
 510                                   (0 != cancelled ? true : false));
 511 }

/* [<][>][^][v][top][bottom][index][help] */