root/ompi/mca/coll/libnbc/coll_libnbc.h

INCLUDED FROM

   1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
   2 /*
   3  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
   4  *                         University Research and Technology
   5  *                         Corporation.  All rights reserved.
   6  * Copyright (c) 2004-2013 The University of Tennessee and The University
   7  *                         of Tennessee Research Foundation.  All rights
   8  *                         reserved.
   9  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
  10  *                         University of Stuttgart.  All rights reserved.
  11  * Copyright (c) 2004-2005 The Regents of the University of California.
  12  *                         All rights reserved.
  13  * Copyright (c) 2008      Cisco Systems, Inc.  All rights reserved.
  14  * Copyright (c) 2013-2015 Los Alamos National Security, LLC. All rights
  15  *                         reserved.
  16  * Copyright (c) 2014-2017 Research Organization for Information Science
  17  *                         and Technology (RIST). All rights reserved.
  18  * Copyright (c) 2016-2017 IBM Corporation.  All rights reserved.
  19  * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
  20  * $COPYRIGHT$
  21  *
  22  * Additional copyrights may follow
  23  *
  24  * $HEADER$
  25  */
  26 
  27 #ifndef MCA_COLL_LIBNBC_EXPORT_H
  28 #define MCA_COLL_LIBNBC_EXPORT_H
  29 
  30 #include "ompi/mca/coll/coll.h"
  31 #include "ompi/request/request.h"
  32 #include "opal/sys/atomic.h"
  33 
  34 BEGIN_C_DECLS
  35 
  36 /*********************** LibNBC tuning parameters ************************/
  37 
  38 /* the debug level */
  39 #define NBC_DLEVEL 0
  40 
  41 /* enable schedule caching - undef NBC_CACHE_SCHEDULE to deactivate it */
  42 /* TODO: this whole schedule cache stuff does not work with the tmbuf
  43  * :-( - first, the tmpbuf must not be freed if a schedule using it is
  44  * still in the cache and second, the tmpbuf used by the schedule must
  45  * be attached to the handle that uses this schedule !!!!
  46  * I.E., THIS IS EXPERIMENTAL AND MIGHT NOT WORK */
  47 /* It also leaks memory because the schedule is never cleaned up when
  48    the communicator is destroyed, so don't use it for now */
  49 #ifdef NBC_CACHE_SCHEDULE
  50 #undef NBC_CACHE_SCHEDULE
  51 #endif
  52 #define NBC_SCHED_DICT_UPPER 1024 /* max. number of dict entries */
  53 #define NBC_SCHED_DICT_LOWER 512  /* nuber of dict entries after wipe, if SCHED_DICT_UPPER is reached */
  54 
  55 /********************* end of LibNBC tuning parameters ************************/
  56 
  57 /* Function return codes  */
  58 #define NBC_OK 0 /* everything went fine */
  59 #define NBC_SUCCESS 0 /* everything went fine (MPI compliant :) */
  60 #define NBC_OOR 1 /* out of resources */
  61 #define NBC_BAD_SCHED 2 /* bad schedule */
  62 #define NBC_CONTINUE 3 /* progress not done */
  63 #define NBC_DATATYPE_NOT_SUPPORTED 4 /* datatype not supported or not valid */
  64 #define NBC_OP_NOT_SUPPORTED 5 /* operation not supported or not valid */
  65 #define NBC_NOT_IMPLEMENTED 6
  66 #define NBC_INVALID_PARAM 7 /* invalid parameters */
  67 #define NBC_INVALID_TOPOLOGY_COMM 8 /* invalid topology attached to communicator */
  68 
  69 /* number of implemented collective functions */
  70 #define NBC_NUM_COLL 17
  71 
  72 extern bool libnbc_ibcast_skip_dt_decision;
  73 extern int libnbc_iallgather_algorithm;
  74 extern int libnbc_iallreduce_algorithm;
  75 extern int libnbc_ibcast_algorithm;
  76 extern int libnbc_ibcast_knomial_radix;
  77 extern int libnbc_iexscan_algorithm;
  78 extern int libnbc_ireduce_algorithm;
  79 extern int libnbc_iscan_algorithm;
  80 
  81 struct ompi_coll_libnbc_component_t {
  82     mca_coll_base_component_2_0_0_t super;
  83     opal_free_list_t requests;
  84     opal_list_t active_requests;
  85     opal_atomic_int32_t active_comms;
  86     opal_mutex_t lock;                /* protect access to the active_requests list */
  87 };
  88 typedef struct ompi_coll_libnbc_component_t ompi_coll_libnbc_component_t;
  89 
  90 /* Globally exported variables */
  91 OMPI_MODULE_DECLSPEC extern ompi_coll_libnbc_component_t mca_coll_libnbc_component;
  92 
  93 struct ompi_coll_libnbc_module_t {
  94     mca_coll_base_module_t super;
  95     opal_mutex_t mutex;
  96     bool comm_registered;
  97     int tag;
  98 #ifdef NBC_CACHE_SCHEDULE
  99   void *NBC_Dict[NBC_NUM_COLL]; /* this should point to a struct
 100                                       hb_tree, but since this is a
 101                                       public header-file, this would be
 102                                       an include mess :-(. So let's void
 103                                       it ...*/
 104   int NBC_Dict_size[NBC_NUM_COLL];
 105 #endif
 106 };
 107 typedef struct ompi_coll_libnbc_module_t ompi_coll_libnbc_module_t;
 108 OBJ_CLASS_DECLARATION(ompi_coll_libnbc_module_t);
 109 
 110 typedef ompi_coll_libnbc_module_t NBC_Comminfo;
 111 
 112 struct NBC_Schedule {
 113     opal_object_t super;
 114     volatile int size;
 115     volatile int current_round_offset;
 116     char *data;
 117 };
 118 
 119 typedef struct NBC_Schedule NBC_Schedule;
 120 
 121 OBJ_CLASS_DECLARATION(NBC_Schedule);
 122 
 123 struct ompi_coll_libnbc_request_t {
 124     ompi_request_t super;
 125     MPI_Comm comm;
 126     long row_offset;
 127     bool nbc_complete; /* status in libnbc level */
 128     int tag;
 129     volatile int req_count;
 130     ompi_request_t **req_array;
 131     NBC_Comminfo *comminfo;
 132     NBC_Schedule *schedule;
 133     void *tmpbuf; /* temporary buffer e.g. used for Reduce */
 134     /* TODO: we should make a handle pointer to a state later (that the user
 135      * can move request handles) */
 136 };
 137 typedef struct ompi_coll_libnbc_request_t ompi_coll_libnbc_request_t;
 138 OBJ_CLASS_DECLARATION(ompi_coll_libnbc_request_t);
 139 
 140 typedef ompi_coll_libnbc_request_t NBC_Handle;
 141 
 142 
 143 #define OMPI_COLL_LIBNBC_REQUEST_ALLOC(comm, persistent, req)           \
 144     do {                                                                \
 145         opal_free_list_item_t *item;                                    \
 146         item = opal_free_list_wait (&mca_coll_libnbc_component.requests); \
 147         req = (ompi_coll_libnbc_request_t*) item;                       \
 148         OMPI_REQUEST_INIT(&req->super, persistent);                     \
 149         req->super.req_mpi_object.comm = comm;                          \
 150     } while (0)
 151 
 152 #define OMPI_COLL_LIBNBC_REQUEST_RETURN(req)                            \
 153     do {                                                                \
 154         OMPI_REQUEST_FINI(&(req)->super);                               \
 155         opal_free_list_return (&mca_coll_libnbc_component.requests,     \
 156                                (opal_free_list_item_t*) (req));         \
 157     } while (0)
 158 
 159 int ompi_coll_libnbc_progress(void);
 160 
 161 int NBC_Init_comm(MPI_Comm comm, ompi_coll_libnbc_module_t *module);
 162 int NBC_Progress(NBC_Handle *handle);
 163 
 164 
 165 int ompi_coll_libnbc_iallgather(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
 166                                 MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
 167                                 struct mca_coll_base_module_2_3_0_t *module);
 168 int ompi_coll_libnbc_iallgatherv(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *displs,
 169                                  MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
 170                                  struct mca_coll_base_module_2_3_0_t *module);
 171 int ompi_coll_libnbc_iallreduce(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op,
 172                                 struct ompi_communicator_t *comm, ompi_request_t ** request,
 173                                 struct mca_coll_base_module_2_3_0_t *module);
 174 int ompi_coll_libnbc_ialltoall(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
 175                                MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
 176                                struct mca_coll_base_module_2_3_0_t *module);
 177 int ompi_coll_libnbc_ialltoallv(const void* sendbuf, const int *sendcounts, const int *sdispls,
 178                                 MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *rdispls,
 179                                 MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
 180                                 struct mca_coll_base_module_2_3_0_t *module);
 181 int ompi_coll_libnbc_ialltoallw(const void *sbuf, const int *scounts, const int *sdisps, struct ompi_datatype_t * const *sdtypes,
 182                                 void *rbuf, const int *rcounts, const int *rdisps, struct ompi_datatype_t * const *rdtypes,
 183                                 struct ompi_communicator_t *comm, ompi_request_t **request,
 184                                 struct mca_coll_base_module_2_3_0_t *module);
 185 int ompi_coll_libnbc_ibarrier(struct ompi_communicator_t *comm, ompi_request_t ** request,
 186                               struct mca_coll_base_module_2_3_0_t *module);
 187 int ompi_coll_libnbc_ibcast(void *buffer, int count, MPI_Datatype datatype, int root,
 188                             struct ompi_communicator_t *comm, ompi_request_t ** request,
 189                             struct mca_coll_base_module_2_3_0_t *module);
 190 int ompi_coll_libnbc_iexscan(const void *sbuf, void *rbuf, int count, struct ompi_datatype_t *dtype,
 191                              struct ompi_op_t *op, struct ompi_communicator_t *comm, ompi_request_t **request,
 192                              struct mca_coll_base_module_2_3_0_t *module);
 193 int ompi_coll_libnbc_igather(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
 194                              MPI_Datatype recvtype, int root, struct ompi_communicator_t *comm, ompi_request_t ** request,
 195                              struct mca_coll_base_module_2_3_0_t *module);
 196 int ompi_coll_libnbc_igatherv(const void* sendbuf, int sendcount, MPI_Datatype sendtype,
 197                               void* recvbuf, const int *recvcounts, const int *displs, MPI_Datatype recvtype,
 198                               int root, struct ompi_communicator_t *comm, ompi_request_t ** request,
 199                               struct mca_coll_base_module_2_3_0_t *module);
 200 int ompi_coll_libnbc_ireduce(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype,
 201                              MPI_Op op, int root, struct ompi_communicator_t *comm, ompi_request_t ** request,
 202                              struct mca_coll_base_module_2_3_0_t *module);
 203 int ompi_coll_libnbc_ireduce_scatter(const void* sendbuf, void* recvbuf, const int *recvcounts, MPI_Datatype datatype,
 204                                      MPI_Op op, struct ompi_communicator_t *comm, ompi_request_t ** request,
 205                                      struct mca_coll_base_module_2_3_0_t *module);
 206 int ompi_coll_libnbc_ireduce_scatter_block(const void *sbuf, void *rbuf, int rcount, struct ompi_datatype_t *dtype,
 207                                            struct ompi_op_t *op, struct ompi_communicator_t *comm,
 208                                            ompi_request_t **request,
 209                                            struct mca_coll_base_module_2_3_0_t *module);
 210 int ompi_coll_libnbc_iscan(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op,
 211                            struct ompi_communicator_t *comm, ompi_request_t ** request,
 212                            struct mca_coll_base_module_2_3_0_t *module);
 213 int ompi_coll_libnbc_iscatter(const void* sendbuf, int sendcount, MPI_Datatype sendtype,
 214                               void* recvbuf, int recvcount, MPI_Datatype recvtype, int root,
 215                               struct ompi_communicator_t *comm, ompi_request_t ** request,
 216                               struct mca_coll_base_module_2_3_0_t *module);
 217 int ompi_coll_libnbc_iscatterv(const void* sendbuf, const int *sendcounts, const int *displs, MPI_Datatype sendtype,
 218                                void* recvbuf, int recvcount, MPI_Datatype recvtype, int root,
 219                                struct ompi_communicator_t *comm, ompi_request_t ** request,
 220                                struct mca_coll_base_module_2_3_0_t *module);
 221 
 222 
 223 int ompi_coll_libnbc_iallgather_inter(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
 224                                 MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
 225                                 struct mca_coll_base_module_2_3_0_t *module);
 226 int ompi_coll_libnbc_iallgatherv_inter(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *displs,
 227                                  MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
 228                                  struct mca_coll_base_module_2_3_0_t *module);
 229 int ompi_coll_libnbc_iallreduce_inter(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op,
 230                                 struct ompi_communicator_t *comm, ompi_request_t ** request,
 231                                 struct mca_coll_base_module_2_3_0_t *module);
 232 int ompi_coll_libnbc_ialltoall_inter(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
 233                                MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
 234                                struct mca_coll_base_module_2_3_0_t *module);
 235 int ompi_coll_libnbc_ialltoallv_inter(const void* sendbuf, const int *sendcounts, const int *sdispls,
 236                                 MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *rdispls,
 237                                 MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
 238                                 struct mca_coll_base_module_2_3_0_t *module);
 239 int ompi_coll_libnbc_ialltoallw_inter(const void *sbuf, const int *scounts, const int *sdisps, struct ompi_datatype_t * const *sdtypes,
 240                                       void *rbuf, const int *rcounts, const int *rdisps, struct ompi_datatype_t * const *rdtypes,
 241                                       struct ompi_communicator_t *comm, ompi_request_t **request,
 242                                       struct mca_coll_base_module_2_3_0_t *module);
 243 int ompi_coll_libnbc_ibarrier_inter(struct ompi_communicator_t *comm, ompi_request_t ** request,
 244                               struct mca_coll_base_module_2_3_0_t *module);
 245 int ompi_coll_libnbc_ibcast_inter(void *buffer, int count, MPI_Datatype datatype, int root,
 246                             struct ompi_communicator_t *comm, ompi_request_t ** request,
 247                             struct mca_coll_base_module_2_3_0_t *module);
 248 int ompi_coll_libnbc_igather_inter(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
 249                              MPI_Datatype recvtype, int root, struct ompi_communicator_t *comm, ompi_request_t ** request,
 250                              struct mca_coll_base_module_2_3_0_t *module);
 251 int ompi_coll_libnbc_igatherv_inter(const void* sendbuf, int sendcount, MPI_Datatype sendtype,
 252                               void* recvbuf, const int *recvcounts, const int *displs, MPI_Datatype recvtype,
 253                               int root, struct ompi_communicator_t *comm, ompi_request_t ** request,
 254                               struct mca_coll_base_module_2_3_0_t *module);
 255 int ompi_coll_libnbc_ireduce_inter(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype,
 256                              MPI_Op op, int root, struct ompi_communicator_t *comm, ompi_request_t ** request,
 257                              struct mca_coll_base_module_2_3_0_t *module);
 258 int ompi_coll_libnbc_ireduce_scatter_inter(const void* sendbuf, void* recvbuf, const int *recvcounts, MPI_Datatype datatype,
 259                                      MPI_Op op, struct ompi_communicator_t *comm, ompi_request_t ** request,
 260                                      struct mca_coll_base_module_2_3_0_t *module);
 261 int ompi_coll_libnbc_ireduce_scatter_block_inter(const void *sbuf, void *rbuf, int rcount, struct ompi_datatype_t *dtype,
 262                                                  struct ompi_op_t *op, struct ompi_communicator_t *comm,
 263                                                  ompi_request_t **request,
 264                                                  struct mca_coll_base_module_2_3_0_t *module);
 265 int ompi_coll_libnbc_iscatter_inter(const void* sendbuf, int sendcount, MPI_Datatype sendtype,
 266                               void* recvbuf, int recvcount, MPI_Datatype recvtype, int root,
 267                               struct ompi_communicator_t *comm, ompi_request_t ** request,
 268                               struct mca_coll_base_module_2_3_0_t *module);
 269 int ompi_coll_libnbc_iscatterv_inter(const void* sendbuf, const int *sendcounts, const int *displs, MPI_Datatype sendtype,
 270                                void* recvbuf, int recvcount, MPI_Datatype recvtype, int root,
 271                                struct ompi_communicator_t *comm, ompi_request_t ** request,
 272                                struct mca_coll_base_module_2_3_0_t *module);
 273 
 274 
 275 int ompi_coll_libnbc_ineighbor_allgather(const void *sbuf, int scount, MPI_Datatype stype, void *rbuf,
 276                                          int rcount, MPI_Datatype rtype, struct ompi_communicator_t *comm,
 277                                          ompi_request_t ** request, struct mca_coll_base_module_2_3_0_t *module);
 278 int ompi_coll_libnbc_ineighbor_allgatherv(const void *sbuf, int scount, MPI_Datatype stype, void *rbuf,
 279                                           const int *rcounts, const int *displs, MPI_Datatype rtype,
 280                                           struct ompi_communicator_t *comm, ompi_request_t ** request,
 281                                           struct mca_coll_base_module_2_3_0_t *module);
 282 int ompi_coll_libnbc_ineighbor_alltoall(const void *sbuf, int scount, MPI_Datatype stype, void *rbuf,
 283                                         int rcount, MPI_Datatype rtype, struct ompi_communicator_t *comm,
 284                                         ompi_request_t ** request, struct mca_coll_base_module_2_3_0_t *module);
 285 int ompi_coll_libnbc_ineighbor_alltoallv(const void *sbuf, const int *scounts, const int *sdispls, MPI_Datatype stype,
 286                                          void *rbuf, const int *rcounts, const int *rdispls, MPI_Datatype rtype,
 287                                          struct ompi_communicator_t *comm, ompi_request_t ** request,
 288                                          struct mca_coll_base_module_2_3_0_t *module);
 289 int ompi_coll_libnbc_ineighbor_alltoallw(const void *sbuf, const int *scounts, const MPI_Aint *sdisps, struct ompi_datatype_t * const *stypes,
 290                                          void *rbuf, const int *rcounts, const MPI_Aint *rdisps, struct ompi_datatype_t * const *rtypes,
 291                                          struct ompi_communicator_t *comm, ompi_request_t ** request,
 292                                          struct mca_coll_base_module_2_3_0_t *module);
 293 
 294 int ompi_coll_libnbc_allgather_init(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
 295                                     MPI_Datatype recvtype, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
 296                                     struct mca_coll_base_module_2_3_0_t *module);
 297 int ompi_coll_libnbc_allgatherv_init(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *displs,
 298                                      MPI_Datatype recvtype, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
 299                                      struct mca_coll_base_module_2_3_0_t *module);
 300 int ompi_coll_libnbc_allreduce_init(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op,
 301                                     struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
 302                                     struct mca_coll_base_module_2_3_0_t *module);
 303 int ompi_coll_libnbc_alltoall_init(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
 304                                    MPI_Datatype recvtype, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
 305                                    struct mca_coll_base_module_2_3_0_t *module);
 306 int ompi_coll_libnbc_alltoallv_init(const void* sendbuf, const int *sendcounts, const int *sdispls,
 307                                     MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *rdispls,
 308                                     MPI_Datatype recvtype, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
 309                                     struct mca_coll_base_module_2_3_0_t *module);
 310 int ompi_coll_libnbc_alltoallw_init(const void *sbuf, const int *scounts, const int *sdisps, struct ompi_datatype_t * const *sdtypes,
 311                                     void *rbuf, const int *rcounts, const int *rdisps, struct ompi_datatype_t * const *rdtypes,
 312                                     struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t **request,
 313                                     struct mca_coll_base_module_2_3_0_t *module);
 314 int ompi_coll_libnbc_barrier_init(struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
 315                                   struct mca_coll_base_module_2_3_0_t *module);
 316 int ompi_coll_libnbc_bcast_init(void *buffer, int count, MPI_Datatype datatype, int root,
 317                                 struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
 318                                 struct mca_coll_base_module_2_3_0_t *module);
 319 int ompi_coll_libnbc_exscan_init(const void *sbuf, void *rbuf, int count, struct ompi_datatype_t *dtype,
 320                                  struct ompi_op_t *op, struct ompi_communicator_t *comm, MPI_Info info,  ompi_request_t **request,
 321                                  struct mca_coll_base_module_2_3_0_t *module);
 322 int ompi_coll_libnbc_gather_init(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
 323                                  MPI_Datatype recvtype, int root, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
 324                                  struct mca_coll_base_module_2_3_0_t *module);
 325 int ompi_coll_libnbc_gatherv_init(const void* sendbuf, int sendcount, MPI_Datatype sendtype,
 326                                   void* recvbuf, const int *recvcounts, const int *displs, MPI_Datatype recvtype,
 327                                   int root, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
 328                                   struct mca_coll_base_module_2_3_0_t *module);
 329 int ompi_coll_libnbc_reduce_init(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype,
 330                                  MPI_Op op, int root, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
 331                                  struct mca_coll_base_module_2_3_0_t *module);
 332 int ompi_coll_libnbc_reduce_scatter_init(const void* sendbuf, void* recvbuf, const int *recvcounts, MPI_Datatype datatype,
 333                                          MPI_Op op, struct ompi_communicator_t *comm, MPI_Info info,  ompi_request_t ** request,
 334                                          struct mca_coll_base_module_2_3_0_t *module);
 335 int ompi_coll_libnbc_reduce_scatter_block_init(const void *sbuf, void *rbuf, int rcount, struct ompi_datatype_t *dtype,
 336                                                struct ompi_op_t *op, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t **request,
 337                                                struct mca_coll_base_module_2_3_0_t *module);
 338 int ompi_coll_libnbc_scan_init(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op,
 339                                struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
 340                                struct mca_coll_base_module_2_3_0_t *module);
 341 int ompi_coll_libnbc_scatter_init(const void* sendbuf, int sendcount, MPI_Datatype sendtype,
 342                                   void* recvbuf, int recvcount, MPI_Datatype recvtype, int root,
 343                                   struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
 344                                   struct mca_coll_base_module_2_3_0_t *module);
 345 int ompi_coll_libnbc_scatterv_init(const void* sendbuf, const int *sendcounts, const int *displs, MPI_Datatype sendtype,
 346                                    void* recvbuf, int recvcount, MPI_Datatype recvtype, int root,
 347                                    struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
 348                                    struct mca_coll_base_module_2_3_0_t *module);
 349 
 350 int ompi_coll_libnbc_allgather_inter_init(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
 351                                           MPI_Datatype recvtype, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
 352                                           struct mca_coll_base_module_2_3_0_t *module);
 353 int ompi_coll_libnbc_allgatherv_inter_init(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *displs,
 354                                            MPI_Datatype recvtype, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
 355                                            struct mca_coll_base_module_2_3_0_t *module);
 356 int ompi_coll_libnbc_allreduce_inter_init(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op,
 357                                           struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
 358                                           struct mca_coll_base_module_2_3_0_t *module);
 359 int ompi_coll_libnbc_alltoall_inter_init(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
 360                                          MPI_Datatype recvtype, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
 361                                          struct mca_coll_base_module_2_3_0_t *module);
 362 int ompi_coll_libnbc_alltoallv_inter_init(const void* sendbuf, const int *sendcounts, const int *sdispls,
 363                                           MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *rdispls,
 364                                           MPI_Datatype recvtype, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
 365                                           struct mca_coll_base_module_2_3_0_t *module);
 366 int ompi_coll_libnbc_alltoallw_inter_init(const void *sbuf, const int *scounts, const int *sdisps, struct ompi_datatype_t * const *sdtypes,
 367                                           void *rbuf, const int *rcounts, const int *rdisps, struct ompi_datatype_t * const *rdtypes,
 368                                           struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t **request,
 369                                           struct mca_coll_base_module_2_3_0_t *module);
 370 int ompi_coll_libnbc_barrier_inter_init(struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
 371                                         struct mca_coll_base_module_2_3_0_t *module);
 372 int ompi_coll_libnbc_bcast_inter_init(void *buffer, int count, MPI_Datatype datatype, int root,
 373                                       struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
 374                                       struct mca_coll_base_module_2_3_0_t *module);
 375 int ompi_coll_libnbc_gather_inter_init(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
 376                                        MPI_Datatype recvtype, int root, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
 377                                        struct mca_coll_base_module_2_3_0_t *module);
 378 int ompi_coll_libnbc_gatherv_inter_init(const void* sendbuf, int sendcount, MPI_Datatype sendtype,
 379                                         void* recvbuf, const int *recvcounts, const int *displs, MPI_Datatype recvtype,
 380                                         int root, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
 381                                         struct mca_coll_base_module_2_3_0_t *module);
 382 int ompi_coll_libnbc_reduce_inter_init(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype,
 383                                        MPI_Op op, int root, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
 384                                        struct mca_coll_base_module_2_3_0_t *module);
 385 int ompi_coll_libnbc_reduce_scatter_inter_init(const void* sendbuf, void* recvbuf, const int *recvcounts, MPI_Datatype datatype,
 386                                                MPI_Op op, struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
 387                                                struct mca_coll_base_module_2_3_0_t *module);
 388 int ompi_coll_libnbc_reduce_scatter_block_inter_init(const void *sbuf, void *rbuf, int rcount, struct ompi_datatype_t *dtype,
 389                                                      struct ompi_op_t *op, struct ompi_communicator_t *comm,
 390                                                      MPI_Info info, ompi_request_t **request,
 391                                                      struct mca_coll_base_module_2_3_0_t *module);
 392 int ompi_coll_libnbc_scatter_inter_init(const void* sendbuf, int sendcount, MPI_Datatype sendtype,
 393                                         void* recvbuf, int recvcount, MPI_Datatype recvtype, int root,
 394                                         struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
 395                                         struct mca_coll_base_module_2_3_0_t *module);
 396 int ompi_coll_libnbc_scatterv_inter_init(const void* sendbuf, const int *sendcounts, const int *displs, MPI_Datatype sendtype,
 397                                          void* recvbuf, int recvcount, MPI_Datatype recvtype, int root,
 398                                          struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
 399                                          struct mca_coll_base_module_2_3_0_t *module);
 400 
 401 int ompi_coll_libnbc_neighbor_allgather_init(const void *sbuf, int scount, MPI_Datatype stype, void *rbuf,
 402                                              int rcount, MPI_Datatype rtype, struct ompi_communicator_t *comm,
 403                                              MPI_Info info, ompi_request_t ** request, struct mca_coll_base_module_2_3_0_t *module);
 404 int ompi_coll_libnbc_neighbor_allgatherv_init(const void *sbuf, int scount, MPI_Datatype stype, void *rbuf,
 405                                               const int *rcounts, const int *displs, MPI_Datatype rtype,
 406                                               struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
 407                                               struct mca_coll_base_module_2_3_0_t *module);
 408 int ompi_coll_libnbc_neighbor_alltoall_init(const void *sbuf, int scount, MPI_Datatype stype, void *rbuf,
 409                                             int rcount, MPI_Datatype rtype, struct ompi_communicator_t *comm, MPI_Info info,
 410                                             ompi_request_t ** request, struct mca_coll_base_module_2_3_0_t *module);
 411 int ompi_coll_libnbc_neighbor_alltoallv_init(const void *sbuf, const int *scounts, const int *sdispls, MPI_Datatype stype,
 412                                              void *rbuf, const int *rcounts, const int *rdispls, MPI_Datatype rtype,
 413                                              struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
 414                                              struct mca_coll_base_module_2_3_0_t *module);
 415 int ompi_coll_libnbc_neighbor_alltoallw_init(const void *sbuf, const int *scounts, const MPI_Aint *sdisps, struct ompi_datatype_t * const *stypes,
 416                                              void *rbuf, const int *rcounts, const MPI_Aint *rdisps, struct ompi_datatype_t * const *rtypes,
 417                                              struct ompi_communicator_t *comm, MPI_Info info, ompi_request_t ** request,
 418                                              struct mca_coll_base_module_2_3_0_t *module);
 419 
 420 
 421 END_C_DECLS
 422 
 423 #endif /* MCA_COLL_LIBNBC_EXPORT_H */