ompi/mca/osc/rdma/osc_rdma

/* [<][>][^][v][top][bottom][index][help] */
This source file includes following definitions.
ompi_osc_rdma_peer_accumulate_cleanup
ompi_osc_rdma_event_put
ompi_osc_rdma_event_queue
ompi_osc_rdma_gacc_local
ompi_osc_rdma_cas_local
ompi_osc_rdma_gacc_contig
ompi_osc_rdma_gacc_master_cleanup
ompi_osc_rdma_gacc_master
ompi_osc_rdma_cas_atomic
ompi_osc_rdma_fetch_and_op_atomic
ompi_osc_rdma_fetch_and_op_cas
ompi_osc_rdma_acc_single_atomic
ompi_osc_rdma_cas_put_complete
cas_rdma
ompi_osc_rdma_compare_and_swap
ompi_osc_rdma_rget_accumulate_internal
ompi_osc_rdma_get_accumulate
ompi_osc_rdma_rget_accumulate
ompi_osc_rdma_raccumulate
ompi_osc_rdma_accumulate
ompi_osc_rdma_fetch_and_op
   1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
   2 /*
   3  * Copyright (c) 2014-2018 Los Alamos National Security, LLC.  All rights
   4  *                         reserved.
   5  * Copyright (c) 2016-2017 Research Organization for Information Science
   6  *                         and Technology (RIST). All rights reserved.
   7  * Copyright (c) 2016-2018 Intel, Inc. All rights reserved.
   8  * Copyright (c) 2019      Triad National Security, LLC. All rights
   9  *                         reserved.
  10  * Copyright (c) 2019      Google, LLC. All rights reserved.
  11  * $COPYRIGHT$
  12  *
  13  * Additional copyrights may follow
  14  *
  15  * $HEADER$
  16  */
  17 
  18 #include "osc_rdma_accumulate.h"
  19 #include "osc_rdma_request.h"
  20 #include "osc_rdma_comm.h"
  21 
  22 #include "ompi/mca/osc/base/osc_base_obj_convert.h"
  23 
  24 static inline void ompi_osc_rdma_peer_accumulate_cleanup (ompi_osc_rdma_module_t *module, ompi_osc_rdma_peer_t *peer, bool lock_acquired)
  25 {
  26     if (lock_acquired) {
  27         (void) ompi_osc_rdma_lock_release_exclusive (module, peer, offsetof (ompi_osc_rdma_state_t, accumulate_lock));
  28     }
  29 
  30     /* clear out the accumulation flag */
  31     ompi_osc_rdma_peer_clear_flag (peer, OMPI_OSC_RDMA_PEER_ACCUMULATING);
  32 }
  33 
  34 enum ompi_osc_rdma_event_type_t {
  35     OMPI_OSC_RDMA_EVENT_TYPE_PUT,
  36 };
  37 
  38 typedef enum ompi_osc_rdma_event_type_t ompi_osc_rdma_event_type_t;
  39 
  40 struct ompi_osc_rdma_event_t {
  41     opal_event_t super;
  42     ompi_osc_rdma_module_t *module;
  43     struct mca_btl_base_endpoint_t *endpoint;
  44     void *local_address;
  45     mca_btl_base_registration_handle_t *local_handle;
  46     uint64_t remote_address;
  47     mca_btl_base_registration_handle_t *remote_handle;
  48     uint64_t length;
  49     mca_btl_base_rdma_completion_fn_t cbfunc;
  50     void *cbcontext;
  51     void *cbdata;
  52 };
  53 
  54 typedef struct ompi_osc_rdma_event_t ompi_osc_rdma_event_t;
  55 
  56 #if 0
  57 static void *ompi_osc_rdma_event_put (int fd, int flags, void *context)
  58 {
  59     ompi_osc_rdma_event_t *event = (ompi_osc_rdma_event_t *) context;
  60     int ret;
  61 
  62     ret = event->module->selected_btl->btl_put (event->module->selected_btl, event->endpoint, event->local_address,
  63                                                 event->remote_address, event->local_handle, event->remote_handle,
  64                                                 event->length, 0, MCA_BTL_NO_ORDER, event->cbfunc, event->cbcontext,
  65                                                 event->cbdata);
  66     if (OPAL_LIKELY(OPAL_SUCCESS == ret)) {
  67         /* done with this event */
  68         opal_event_del (&event->super);
  69         free (event);
  70     } else {
  71         /* re-activate the event */
  72         opal_event_active (&event->super, OPAL_EV_READ, 1);
  73     }
  74 
  75     return NULL;
  76 }
  77 
  78 static int ompi_osc_rdma_event_queue (ompi_osc_rdma_module_t *module, struct mca_btl_base_endpoint_t *endpoint,
  79                                       ompi_osc_rdma_event_type_t event_type, void *local_address, mca_btl_base_registration_handle_t *local_handle,
  80                                       uint64_t remote_address, mca_btl_base_registration_handle_t *remote_handle,
  81                                       uint64_t length, mca_btl_base_rdma_completion_fn_t cbfunc,  void *cbcontext,
  82                                       void *cbdata)
  83 {
  84     ompi_osc_rdma_event_t *event = malloc (sizeof (*event));
  85     void *(*event_func) (int, int, void *);
  86 
  87     OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "queueing event type %d", event_type);
  88 
  89     if (OPAL_UNLIKELY(NULL == event)) {
  90         return OMPI_ERR_OUT_OF_RESOURCE;
  91     }
  92 
  93     event->module = module;
  94     event->endpoint = endpoint;
  95     event->local_address = local_address;
  96     event->local_handle = local_handle;
  97     event->remote_address = remote_address;
  98     event->remote_handle = remote_handle;
  99     event->length = length;
 100     event->cbfunc = cbfunc;
 101     event->cbcontext = cbcontext;
 102     event->cbdata = cbdata;
 103 
 104     switch (event_type) {
 105     case OMPI_OSC_RDMA_EVENT_TYPE_PUT:
 106         event_func = ompi_osc_rdma_event_put;
 107         break;
 108     default:
 109         opal_output(0, "osc/rdma: cannot queue unknown event type %d", event_type);
 110         abort ();
 111     }
 112 
 113     opal_event_set (opal_sync_event_base, &event->super, -1, OPAL_EV_READ,
 114                     event_func, event);
 115     opal_event_active (&event->super, OPAL_EV_READ, 1);
 116 
 117     return OMPI_SUCCESS;
 118 }
 119 #endif
 120 
 121 static int ompi_osc_rdma_gacc_local (const void *source_buffer, int source_count, ompi_datatype_t *source_datatype,
 122                                      void *result_buffer, int result_count, ompi_datatype_t *result_datatype,
 123                                      ompi_osc_rdma_peer_t *peer, uint64_t target_address,
 124                                      mca_btl_base_registration_handle_t *target_handle, int target_count,
 125                                      ompi_datatype_t *target_datatype, ompi_op_t *op, ompi_osc_rdma_module_t *module,
 126                                      ompi_osc_rdma_request_t *request, bool lock_acquired)
 127 {
 128     int ret = OMPI_SUCCESS;
 129 
 130     do {
 131         OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "performing accumulate with local region(s)");
 132 
 133         if (NULL != result_buffer) {
 134             /* get accumulate */
 135 
 136             ret = ompi_datatype_sndrcv ((void *) (intptr_t) target_address, target_count, target_datatype,
 137                                         result_buffer, result_count, result_datatype);
 138 
 139             if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
 140                 break;
 141             }
 142         }
 143 
 144         if (&ompi_mpi_op_no_op.op != op) {
 145             if (&ompi_mpi_op_replace.op != op) {
 146                 ret = ompi_osc_base_sndrcv_op (source_buffer, source_count, source_datatype, (void *) (intptr_t) target_address,
 147                                                target_count, target_datatype, op);
 148             } else {
 149                 ret = ompi_datatype_sndrcv (source_buffer, source_count, source_datatype, (void *) (intptr_t) target_address,
 150                                             target_count, target_datatype);
 151             }
 152         }
 153     } while (0);
 154 
 155     ompi_osc_rdma_peer_accumulate_cleanup (module, peer, lock_acquired);
 156 
 157     if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
 158         OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_ERROR, "local accumulate failed with ompi error code %d", ret);
 159         return ret;
 160     }
 161 
 162     if (request) {
 163         /* NTH: is it ok to use an ompi error code here? */
 164         ompi_osc_rdma_request_complete (request, ret);
 165     }
 166 
 167     return ret;
 168 }
 169 
 170 static inline int ompi_osc_rdma_cas_local (const void *source_addr, const void *compare_addr, void *result_addr,
 171                                            ompi_datatype_t *datatype, ompi_osc_rdma_peer_t *peer,
 172                                            uint64_t target_address, mca_btl_base_registration_handle_t *target_handle,
 173                                            ompi_osc_rdma_module_t *module, bool lock_acquired)
 174 {
 175     OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "performing compare-and-swap with local regions");
 176 
 177     memcpy (result_addr, (void *) (uintptr_t) target_address, datatype->super.size);
 178 
 179     if (0 == memcmp (compare_addr, result_addr, datatype->super.size)) {
 180         memcpy ((void *) (uintptr_t) target_address, source_addr, datatype->super.size);
 181     }
 182 
 183     ompi_osc_rdma_peer_accumulate_cleanup (module, peer, lock_acquired);
 184 
 185     return OMPI_SUCCESS;
 186 }
 187 
 188 static inline int ompi_osc_rdma_gacc_contig (ompi_osc_rdma_sync_t *sync, const void *source, int source_count,
 189                                              ompi_datatype_t *source_datatype, void *result, int result_count,
 190                                              ompi_datatype_t *result_datatype, ompi_osc_rdma_peer_t *peer, uint64_t target_address,
 191                                              mca_btl_base_registration_handle_t *target_handle, int target_count,
 192                                              ompi_datatype_t *target_datatype, ompi_op_t *op, ompi_osc_rdma_request_t *request)
 193 {
 194     ompi_osc_rdma_module_t *module = sync->module;
 195     unsigned long len = target_count * target_datatype->super.size;
 196     char *ptr = NULL;
 197     int ret;
 198 
 199     OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "initiating accumulate on contiguous region of %lu bytes to remote address %" PRIx64
 200                      ", sync %p", len, target_address, (void *) sync);
 201 
 202     if (&ompi_mpi_op_replace.op != op || OMPI_OSC_RDMA_TYPE_GET_ACC == request->type) {
 203         ptr = malloc (len);
 204         if (OPAL_UNLIKELY(NULL == ptr)) {
 205             OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_WARN, "could not allocate a temporary buffer for accumulate");
 206             return OMPI_ERR_OUT_OF_RESOURCE;
 207         }
 208 
 209         /* set up the request */
 210         request->to_free = ptr;
 211 
 212         ret = ompi_osc_get_data_blocking (module, peer->data_endpoint, target_address, target_handle, ptr, len);
 213         if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
 214             return ret;
 215         }
 216 
 217         if (OMPI_OSC_RDMA_TYPE_GET_ACC == request->type) {
 218             if (NULL == result) {
 219                 /* result buffer is not necessarily contiguous. use the opal datatype engine to
 220                  * copy the data over in this case */
 221                 struct iovec iov = {.iov_base = ptr, len};
 222                 uint32_t iov_count = 1;
 223                 size_t size = request->len;
 224 
 225                 opal_convertor_unpack (&request->convertor, &iov, &iov_count, &size);
 226                 opal_convertor_cleanup (&request->convertor);
 227             } else {
 228                 /* copy contiguous data to the result buffer */
 229                 ompi_datatype_sndrcv (ptr, len, MPI_BYTE, result, result_count, result_datatype);
 230             }
 231         }
 232 
 233         if (&ompi_mpi_op_replace.op == op) {
 234             return ompi_osc_rdma_put_contig (sync, peer, target_address, target_handle, (void *) source, len, request);
 235         }
 236 
 237         if (&ompi_mpi_op_no_op.op != op) {
 238             /* NTH: need to cast away const for the source buffer. the buffer will not be modified by this call */
 239             ompi_op_reduce (op, (void *) source, ptr, source_count, source_datatype);
 240 
 241             return ompi_osc_rdma_put_contig (sync, peer, target_address, target_handle, ptr, len, request);
 242         }
 243 
 244         if (request) {
 245             /* nothing more to do for this request */
 246             ompi_osc_rdma_request_complete (request, MPI_SUCCESS);
 247         }
 248 
 249         return OMPI_SUCCESS;
 250     }
 251 
 252     return ompi_osc_rdma_put_contig (sync, peer, target_address, target_handle, (void *) source, len, request);
 253 }
 254 
 255 static void ompi_osc_rdma_gacc_master_cleanup (ompi_osc_rdma_request_t *request)
 256 {
 257     ompi_osc_rdma_peer_accumulate_cleanup (request->module, request->peer, !ompi_osc_rdma_peer_is_exclusive (request->peer));
 258 }
 259 
 260 static inline int ompi_osc_rdma_gacc_master (ompi_osc_rdma_sync_t *sync, const void *source_addr, int source_count,
 261                                              ompi_datatype_t *source_datatype, void *result_addr, int result_count,
 262                                              ompi_datatype_t *result_datatype, ompi_osc_rdma_peer_t *peer, uint64_t target_address,
 263                                              mca_btl_base_registration_handle_t *target_handle, int target_count,
 264                                              ompi_datatype_t *target_datatype, ompi_op_t *op, ompi_osc_rdma_request_t *request)
 265 {
 266     ompi_osc_rdma_module_t *module = sync->module;
 267     struct iovec source_iovec[OMPI_OSC_RDMA_DECODE_MAX], target_iovec[OMPI_OSC_RDMA_DECODE_MAX];
 268     const size_t acc_limit = (mca_osc_rdma_component.buffer_size >> 3);
 269     uint32_t source_primitive_count, target_primitive_count;
 270     opal_convertor_t source_convertor, target_convertor;
 271     uint32_t source_iov_count, target_iov_count;
 272     uint32_t source_iov_index, target_iov_index;
 273     ompi_datatype_t *source_primitive, *target_primitive;
 274     /* needed for opal_convertor_raw but not used */
 275     size_t source_size, target_size;
 276     ompi_osc_rdma_request_t *subreq;
 277     size_t result_position;
 278     ptrdiff_t lb, extent;
 279     int ret, acc_len;
 280     bool done;
 281 
 282     if (!request) {
 283         OMPI_OSC_RDMA_REQUEST_ALLOC(module, peer, request);
 284         request->internal = true;
 285     }
 286 
 287     request->cleanup = ompi_osc_rdma_gacc_master_cleanup;
 288     request->type = result_datatype ? OMPI_OSC_RDMA_TYPE_GET_ACC : OMPI_OSC_RDMA_TYPE_ACC;
 289 
 290     (void) ompi_datatype_get_extent (target_datatype, &lb, &extent);
 291     target_address += lb;
 292 
 293     /* fast path for accumulate on built-in types */
 294     if (OPAL_LIKELY((!source_count || ompi_datatype_is_predefined (source_datatype)) &&
 295                     ompi_datatype_is_predefined (target_datatype) &&
 296                     (!result_count || ompi_datatype_is_predefined (result_datatype)) &&
 297                     (target_datatype->super.size * target_count <= acc_limit))) {
 298         if (source_datatype) {
 299             (void) ompi_datatype_get_extent (source_datatype, &lb, &extent);
 300             source_addr = (void *)((intptr_t) source_addr + lb);
 301         }
 302 
 303         if (result_datatype) {
 304             (void) ompi_datatype_get_extent (result_datatype, &lb, &extent);
 305             result_addr = (void *)((intptr_t) result_addr + lb);
 306         }
 307 
 308         ret = ompi_osc_rdma_gacc_contig (sync, source_addr, source_count, source_datatype, result_addr,
 309                                          result_count, result_datatype, peer, target_address,
 310                                          target_handle, target_count, target_datatype, op,
 311                                          request);
 312         if (OPAL_LIKELY(OMPI_SUCCESS == ret)) {
 313             return OMPI_SUCCESS;
 314         }
 315 
 316         if (source_datatype) {
 317             /* the convertors will handle the lb */
 318             (void) ompi_datatype_get_extent (source_datatype, &lb, &extent);
 319             source_addr = (void *)((intptr_t) source_addr - lb);
 320         }
 321 
 322         if (result_datatype) {
 323             (void) ompi_datatype_get_extent (result_datatype, &lb, &extent);
 324             result_addr = (void *)((intptr_t) result_addr - lb);
 325         }
 326     }
 327 
 328     OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "scheduling accumulate on non-contiguous datatype(s)");
 329 
 330     /* the convertor will handle lb from here */
 331     (void) ompi_datatype_get_extent (target_datatype, &lb, &extent);
 332     target_address -= lb;
 333 
 334     /* get the primitive datatype info */
 335     ret = ompi_osc_base_get_primitive_type_info (target_datatype, &target_primitive, &target_primitive_count);
 336     if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
 337         /* target datatype is not made up of a single basic datatype */
 338         return ret;
 339     }
 340 
 341     if (source_datatype) {
 342         ret = ompi_osc_base_get_primitive_type_info (source_datatype, &source_primitive, &source_primitive_count);
 343         if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
 344             /* target datatype is not made up of a single basic datatype */
 345             return ret;
 346         }
 347 
 348         if (OPAL_UNLIKELY(source_primitive != target_primitive)) {
 349             return MPI_ERR_TYPE;
 350         }
 351     }
 352 
 353     /* prepare convertors for the source and target. these convertors will be used to determine the
 354      * contiguous segments within the source and target. */
 355     /* the source may be NULL if using MPI_OP_NO_OP with MPI_Get_accumulate */
 356     if (source_datatype) {
 357         OBJ_CONSTRUCT(&source_convertor, opal_convertor_t);
 358         ret = opal_convertor_copy_and_prepare_for_send (ompi_mpi_local_convertor, &source_datatype->super, source_count, source_addr,
 359                                                        0, &source_convertor);
 360         if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
 361             return ret;
 362         }
 363     }
 364 
 365     /* target_datatype can never be NULL */
 366     OBJ_CONSTRUCT(&target_convertor, opal_convertor_t);
 367     ret = opal_convertor_copy_and_prepare_for_send (ompi_mpi_local_convertor, &target_datatype->super, target_count,
 368                                                     (void *) (intptr_t) target_address, 0, &target_convertor);
 369     if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
 370         return ret;
 371     }
 372 
 373     /* keep the request from completing until all the transfers have started */
 374     request->outstanding_requests = 1;
 375 
 376     target_iov_index = 0;
 377     target_iov_count = 0;
 378     result_position = 0;
 379     subreq = NULL;
 380 
 381     do {
 382         /* decode segments of the source data */
 383         source_iov_count = OMPI_OSC_RDMA_DECODE_MAX;
 384         source_iov_index = 0;
 385         /* opal_convertor_raw returns done when it has reached the end of the data */
 386         if (!source_datatype) {
 387             done = true;
 388             source_iovec[0].iov_len = (size_t) -1;
 389             source_iovec[0].iov_base = NULL;
 390             source_iov_count = 1;
 391         } else {
 392             done = opal_convertor_raw (&source_convertor, source_iovec, &source_iov_count, &source_size);
 393         }
 394 
 395         /* loop on the target segments until we have exhaused the decoded source data */
 396         while (source_iov_index != source_iov_count) {
 397             if (target_iov_index == target_iov_count) {
 398                 /* decode segments of the target buffer */
 399                 target_iov_count = OMPI_OSC_RDMA_DECODE_MAX;
 400                 target_iov_index = 0;
 401                 (void) opal_convertor_raw (&target_convertor, target_iovec, &target_iov_count, &target_size);
 402             }
 403 
 404             /* we already checked that the target was large enough. this should be impossible */
 405             assert (0 != target_iov_count);
 406 
 407             /* determine how much to put in this operation */
 408             acc_len = min(target_iovec[target_iov_index].iov_len, source_iovec[source_iov_index].iov_len);
 409             acc_len = min((size_t) acc_len, acc_limit);
 410 
 411             /* execute the get */
 412             if (!subreq) {
 413                 OMPI_OSC_RDMA_REQUEST_ALLOC(module, peer, subreq);
 414                 subreq->internal = true;
 415                 subreq->parent_request = request;
 416                 (void) OPAL_THREAD_ADD_FETCH32 (&request->outstanding_requests, 1);
 417             }
 418 
 419             if (result_datatype) {
 420                 /* prepare a convertor for this part of the result */
 421                 opal_convertor_copy_and_prepare_for_recv (ompi_mpi_local_convertor, &result_datatype->super, result_count,
 422                                                           result_addr, 0, &subreq->convertor);
 423                 opal_convertor_set_position (&subreq->convertor, &result_position);
 424                 subreq->type = OMPI_OSC_RDMA_TYPE_GET_ACC;
 425             } else {
 426                 subreq->type = OMPI_OSC_RDMA_TYPE_ACC;
 427             }
 428 
 429             ret = ompi_osc_rdma_gacc_contig (sync, source_iovec[source_iov_index].iov_base, acc_len / target_primitive->super.size,
 430                                              target_primitive, NULL, 0, NULL, peer,
 431                                              (uint64_t) (intptr_t) target_iovec[target_iov_index].iov_base, target_handle,
 432                                              acc_len / target_primitive->super.size, target_primitive, op, subreq);
 433             if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
 434                 if (OPAL_UNLIKELY(OMPI_ERR_OUT_OF_RESOURCE != ret)) {
 435                     OMPI_OSC_RDMA_REQUEST_RETURN(subreq);
 436                     (void) OPAL_THREAD_ADD_FETCH32 (&request->outstanding_requests, -1);
 437                     /* something bad happened. need to figure out how to handle these errors */
 438                     return ret;
 439                 }
 440 
 441                 /* progress and try again */
 442                 ompi_osc_rdma_progress (module);
 443                 continue;
 444             }
 445 
 446             subreq = NULL;
 447 
 448             /* adjust io vectors */
 449             target_iovec[target_iov_index].iov_len -= acc_len;
 450             source_iovec[source_iov_index].iov_len -= acc_len;
 451             target_iovec[target_iov_index].iov_base = (void *)((intptr_t) target_iovec[target_iov_index].iov_base + acc_len);
 452             source_iovec[source_iov_index].iov_base = (void *)((intptr_t) source_iovec[source_iov_index].iov_base + acc_len);
 453             result_position += acc_len;
 454 
 455             source_iov_index += !source_datatype || (0 == source_iovec[source_iov_index].iov_len);
 456             target_iov_index += (0 == target_iovec[target_iov_index].iov_len);
 457         }
 458     } while (!done);
 459 
 460     /* release our reference so the request can complete */
 461     ompi_osc_rdma_request_deref (request);
 462 
 463     if (source_datatype) {
 464         opal_convertor_cleanup (&source_convertor);
 465         OBJ_DESTRUCT(&source_convertor);
 466     }
 467 
 468     OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "finished scheduling rdma on non-contiguous datatype(s)");
 469 
 470     opal_convertor_cleanup (&target_convertor);
 471     OBJ_DESTRUCT(&target_convertor);
 472 
 473     return OMPI_SUCCESS;
 474 }
 475 
 476 static inline int ompi_osc_rdma_cas_atomic (ompi_osc_rdma_sync_t *sync, const void *source_addr, const void *compare_addr,
 477                                             void *result_addr, ompi_datatype_t *datatype, ompi_osc_rdma_peer_t *peer,
 478                                             uint64_t target_address, mca_btl_base_registration_handle_t *target_handle,
 479                                             bool lock_acquired)
 480 {
 481     ompi_osc_rdma_module_t *module = sync->module;
 482     int32_t atomic_flags = module->selected_btl->btl_atomic_flags;
 483     const size_t size = datatype->super.size;
 484     int64_t compare, source;
 485     int ret, flags;
 486 
 487     if (8 != size && !(4 == size && (MCA_BTL_ATOMIC_SUPPORTS_32BIT & atomic_flags))) {
 488         return OMPI_ERR_NOT_SUPPORTED;
 489     }
 490 
 491     compare = (8 == size) ? ((int64_t *) compare_addr)[0] : ((int32_t *) compare_addr)[0];
 492     source = (8 == size) ? ((int64_t *) source_addr)[0] : ((int32_t *) source_addr)[0];
 493     flags = (4 == size) ? MCA_BTL_ATOMIC_FLAG_32BIT : 0;
 494 
 495     OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "initiating compare-and-swap using %d-bit btl atomics. compare: 0x%"
 496                      PRIx64 ", origin: 0x%" PRIx64, (int) size * 8, *((int64_t *) compare_addr), *((int64_t *) source_addr));
 497 
 498     ret = ompi_osc_rdma_btl_cswap (module, peer->data_endpoint, target_address, target_handle, compare, source, flags,
 499                                    result_addr);
 500     if (OPAL_LIKELY(OMPI_SUCCESS == ret)) {
 501         ompi_osc_rdma_peer_accumulate_cleanup (module, peer, lock_acquired);
 502     }
 503 
 504     return ret;
 505 }
 506 
 507 static int ompi_osc_rdma_op_mapping[OMPI_OP_NUM_OF_TYPES + 1] = {
 508     [OMPI_OP_MAX] = MCA_BTL_ATOMIC_MAX,
 509     [OMPI_OP_MIN] = MCA_BTL_ATOMIC_MIN,
 510     [OMPI_OP_SUM] = MCA_BTL_ATOMIC_ADD,
 511     [OMPI_OP_BAND] = MCA_BTL_ATOMIC_AND,
 512     [OMPI_OP_BOR] = MCA_BTL_ATOMIC_OR,
 513     [OMPI_OP_BXOR] = MCA_BTL_ATOMIC_XOR,
 514     [OMPI_OP_LAND] = MCA_BTL_ATOMIC_LAND,
 515     [OMPI_OP_LOR] = MCA_BTL_ATOMIC_LOR,
 516     [OMPI_OP_LXOR] = MCA_BTL_ATOMIC_LXOR,
 517     [OMPI_OP_REPLACE] = MCA_BTL_ATOMIC_SWAP,
 518 };
 519 
 520 static int ompi_osc_rdma_fetch_and_op_atomic (ompi_osc_rdma_sync_t *sync, const void *origin_addr, void *result_addr, ompi_datatype_t *dt,
 521                                               ptrdiff_t extent, ompi_osc_rdma_peer_t *peer, uint64_t target_address,
 522                                               mca_btl_base_registration_handle_t *target_handle, ompi_op_t *op, ompi_osc_rdma_request_t *req,
 523                                               bool lock_acquired)
 524 {
 525     ompi_osc_rdma_module_t *module = sync->module;
 526     int32_t atomic_flags = module->selected_btl->btl_atomic_flags;
 527     int ret, btl_op, flags;
 528     int64_t origin;
 529 
 530     if ((8 != extent && !((MCA_BTL_ATOMIC_SUPPORTS_32BIT & atomic_flags) && 4 == extent)) ||
 531         (!(OMPI_DATATYPE_FLAG_DATA_INT & dt->super.flags) && !(MCA_BTL_ATOMIC_SUPPORTS_FLOAT & atomic_flags)) ||
 532         !ompi_op_is_intrinsic (op) || (0 == ompi_osc_rdma_op_mapping[op->op_type])) {
 533         return OMPI_ERR_NOT_SUPPORTED;
 534     }
 535 
 536     btl_op = ompi_osc_rdma_op_mapping[op->op_type];
 537     if (0 == btl_op) {
 538         return OMPI_ERR_NOT_SUPPORTED;
 539     }
 540 
 541     flags = (4 == extent) ? MCA_BTL_ATOMIC_FLAG_32BIT : 0;
 542     if (OMPI_DATATYPE_FLAG_DATA_FLOAT & dt->super.flags) {
 543         flags |= MCA_BTL_ATOMIC_FLAG_FLOAT;
 544     }
 545 
 546     OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "initiating fetch-and-op using %d-bit btl atomics. origin: 0x%" PRIx64,
 547                      (4 == extent) ? 32 : 64, *((int64_t *) origin_addr));
 548 
 549     origin = (8 == extent) ? ((int64_t *) origin_addr)[0] : ((int32_t *) origin_addr)[0];
 550 
 551     ret = ompi_osc_rdma_btl_fop (module, peer->data_endpoint, target_address, target_handle, btl_op, origin, flags,
 552                                  result_addr, true, NULL, NULL, NULL);
 553     if (OPAL_SUCCESS == ret) {
 554         /* done. release the lock */
 555         ompi_osc_rdma_peer_accumulate_cleanup (module, peer, lock_acquired);
 556 
 557         if (req) {
 558             ompi_osc_rdma_request_complete (req, MPI_SUCCESS);
 559         }
 560     }
 561 
 562     return ret;
 563 }
 564 
 565 static int ompi_osc_rdma_fetch_and_op_cas (ompi_osc_rdma_sync_t *sync, const void *origin_addr, void *result_addr, ompi_datatype_t *dt,
 566                                            ptrdiff_t extent, ompi_osc_rdma_peer_t *peer, uint64_t target_address,
 567                                            mca_btl_base_registration_handle_t *target_handle, ompi_op_t *op, ompi_osc_rdma_request_t *req,
 568                                            bool lock_acquired)
 569 {
 570     ompi_osc_rdma_module_t *module = sync->module;
 571     uint64_t address, offset, new_value, old_value;
 572     int ret;
 573 
 574     if (extent > 8) {
 575         return OMPI_ERR_NOT_SUPPORTED;
 576     }
 577 
 578     /* align the address. the user should not call with an unaligned address so don't need to range check here */
 579     address = target_address & ~7;
 580     offset = target_address & ~address;
 581 
 582     OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "initiating fetch-and-op using compare-and-swap");
 583 
 584     ret = ompi_osc_get_data_blocking (module, peer->data_endpoint, address, target_handle, &old_value, 8);
 585     if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
 586         return ret;
 587     }
 588 
 589     /* store the destination in the temporary buffer */
 590     do {
 591         new_value = old_value;
 592 
 593         if (&ompi_mpi_op_replace.op == op) {
 594             memcpy ((void *)((intptr_t) &new_value + offset), origin_addr, extent);
 595         } else if (&ompi_mpi_op_no_op.op != op) {
 596             ompi_op_reduce (op, (void *) origin_addr, (void*)((intptr_t) &new_value + offset), 1, dt);
 597         }
 598 
 599         ret = ompi_osc_rdma_btl_cswap (module, peer->data_endpoint, address, target_handle,
 600                                        old_value, new_value, 0, (int64_t*)&new_value);
 601         if (OPAL_SUCCESS != ret || new_value == old_value) {
 602             break;
 603         }
 604 
 605         old_value = new_value;
 606     } while (1);
 607 
 608     if (result_addr) {
 609         memcpy (result_addr, (void *)((intptr_t) &new_value + offset), extent);
 610     }
 611 
 612     if (OPAL_SUCCESS == ret) {
 613         /* done. release the lock */
 614         ompi_osc_rdma_peer_accumulate_cleanup (module, peer, lock_acquired);
 615 
 616         if (req) {
 617             ompi_osc_rdma_request_complete (req, MPI_SUCCESS);
 618         }
 619     }
 620 
 621     return ret;
 622 }
 623 
 624 static int ompi_osc_rdma_acc_single_atomic (ompi_osc_rdma_sync_t *sync, const void *origin_addr, ompi_datatype_t *dt, ptrdiff_t extent,
 625                                             ompi_osc_rdma_peer_t *peer, uint64_t target_address,  mca_btl_base_registration_handle_t *target_handle,
 626                                             ompi_op_t *op, ompi_osc_rdma_request_t *req, bool lock_acquired)
 627 {
 628     ompi_osc_rdma_module_t *module = sync->module;
 629     int32_t atomic_flags = module->selected_btl->btl_atomic_flags;
 630     int ret, btl_op, flags;
 631     int64_t origin;
 632 
 633     if (!(module->selected_btl->btl_flags & MCA_BTL_FLAGS_ATOMIC_OPS)) {
 634         /* btl put atomics not supported or disabled. fall back on fetch-and-op */
 635         return ompi_osc_rdma_fetch_and_op_atomic (sync, origin_addr, NULL, dt, extent, peer, target_address, target_handle,
 636                                                   op, req, lock_acquired);
 637     }
 638 
 639     if ((8 != extent && !((MCA_BTL_ATOMIC_SUPPORTS_32BIT & atomic_flags) && 4 == extent)) ||
 640         (!(OMPI_DATATYPE_FLAG_DATA_INT & dt->super.flags) && !(MCA_BTL_ATOMIC_SUPPORTS_FLOAT & atomic_flags)) ||
 641         !ompi_op_is_intrinsic (op) || (0 == ompi_osc_rdma_op_mapping[op->op_type])) {
 642         return OMPI_ERR_NOT_SUPPORTED;
 643     }
 644 
 645     origin = (8 == extent) ? ((uint64_t *) origin_addr)[0] : ((uint32_t *) origin_addr)[0];
 646 
 647     /* set the appropriate flags for this atomic */
 648     flags = (4 == extent) ? MCA_BTL_ATOMIC_FLAG_32BIT : 0;
 649     if (OMPI_DATATYPE_FLAG_DATA_FLOAT & dt->super.flags) {
 650         flags |= MCA_BTL_ATOMIC_FLAG_FLOAT;
 651     }
 652 
 653     btl_op = ompi_osc_rdma_op_mapping[op->op_type];
 654 
 655     OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "initiating accumulate using 64-bit btl atomics. origin: 0x%" PRIx64,
 656                      *((int64_t *) origin_addr));
 657 
 658     /* if we locked the peer its best to wait for completion before returning */
 659     ret = ompi_osc_rdma_btl_op (module, peer->data_endpoint, target_address, target_handle, btl_op, origin,
 660                                 flags, true, NULL, NULL, NULL);
 661     if (OPAL_SUCCESS == ret) {
 662         /* done. release the lock */
 663         ompi_osc_rdma_peer_accumulate_cleanup (module, peer, lock_acquired);
 664 
 665         if (req) {
 666             ompi_osc_rdma_request_complete (req, MPI_SUCCESS);
 667         }
 668     }
 669 
 670     return ret;
 671 }
 672 
 673 /**
 674  * ompi_osc_rdma_cas_get_complete:
 675  * Note: This function will not work as is in a heterogeneous environment.
 676  */
 677 static void ompi_osc_rdma_cas_put_complete (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
 678                                             void *local_address, mca_btl_base_registration_handle_t *local_handle,
 679                                             void *context, void *data, int status)
 680 {
 681     bool *complete = (bool *) context;
 682 
 683     *complete = true;
 684 }
 685 
 686 /**
 687  * @brief Support for compare-and-swap on arbitraty-sized datatypes
 688  *
 689  * This function is necessary to support compare-and-swap on types larger
 690  * than 64-bits. As of MPI-3.1 this can include MPI_INTEGER16 and possibly
 691  * MPI_LON_LONG_INT. The former is a 128-bit value and the later *may*
 692  * be depending on the platform, compiler, etc. This function currently
 693  * blocks until the operation is complete.
 694  */
 695 static inline int cas_rdma (ompi_osc_rdma_sync_t *sync, const void *source_addr, const void *compare_addr, void *result_addr,
 696                             ompi_datatype_t *datatype, ompi_osc_rdma_peer_t *peer, uint64_t target_address,
 697                             mca_btl_base_registration_handle_t *target_handle, bool lock_acquired)
 698 {
 699     ompi_osc_rdma_module_t *module = sync->module;
 700     unsigned long len = datatype->super.size;
 701     mca_btl_base_registration_handle_t *local_handle = NULL;
 702     ompi_osc_rdma_frag_t *frag = NULL;
 703     volatile bool complete = false;
 704     /* drop the const. this code will not attempt to change the value */
 705     char *ptr = (char *) source_addr;
 706     int ret;
 707 
 708     OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "initiating compare-and-swap using RMDA on %lu bytes to remote address %" PRIx64
 709                      ", sync %p", len, target_address, (void *) sync);
 710 
 711     OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "RDMA compare-and-swap initiating blocking btl get...");
 712     ret = ompi_osc_get_data_blocking (module, peer->data_endpoint, target_address, target_handle, result_addr, len);
 713     if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
 714         return ret;
 715     }
 716 
 717     if (0 != memcmp (result_addr, compare_addr, len)) {
 718         /* value does not match compare value, nothing more to do*/
 719         ompi_osc_rdma_peer_accumulate_cleanup (module, peer, lock_acquired);
 720         return OMPI_SUCCESS;
 721     }
 722 
 723     if (module->selected_btl->btl_register_mem && len > module->selected_btl->btl_put_local_registration_threshold) {
 724         do {
 725             ret = ompi_osc_rdma_frag_alloc (module, len, &frag, &ptr);
 726             if (OPAL_UNLIKELY(OMPI_SUCCESS == ret)) {
 727                 break;
 728             }
 729 
 730             ompi_osc_rdma_progress (module);
 731         } while (1);
 732 
 733         memcpy (ptr, source_addr, len);
 734         local_handle = frag->handle;
 735     }
 736 
 737     OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "RDMA compare-and-swap initiating blocking btl put...");
 738 
 739     do {
 740         ret = module->selected_btl->btl_put (module->selected_btl, peer->data_endpoint, ptr, target_address,
 741                                              local_handle, target_handle, len, 0, MCA_BTL_NO_ORDER,
 742                                              ompi_osc_rdma_cas_put_complete, (void *) &complete, NULL);
 743         if (OPAL_SUCCESS == ret || (OPAL_ERR_OUT_OF_RESOURCE != ret && OPAL_ERR_TEMP_OUT_OF_RESOURCE != ret)) {
 744             break;
 745         }
 746 
 747         /* spin a bit on progress */
 748         ompi_osc_rdma_progress (module);
 749     } while (1);
 750 
 751     if (OPAL_SUCCESS != ret) {
 752         /* something went horribly wrong */
 753         return ret;
 754     }
 755 
 756     while (!complete) {
 757         ompi_osc_rdma_progress (module);
 758     }
 759 
 760     OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "RDMA compare-and-swap compare-and-swap complete");
 761 
 762     if (frag) {
 763         ompi_osc_rdma_frag_complete (frag);
 764     }
 765 
 766     ompi_osc_rdma_peer_accumulate_cleanup (module, peer, lock_acquired);
 767 
 768     return ret;
 769 }
 770 
 771 
 772 int ompi_osc_rdma_compare_and_swap (const void *origin_addr, const void *compare_addr, void *result_addr,
 773                                     ompi_datatype_t *dt, int target_rank, ptrdiff_t target_disp,
 774                                     ompi_win_t *win)
 775 {
 776     ompi_osc_rdma_module_t *module = GET_MODULE(win);
 777     ompi_osc_rdma_peer_t *peer;
 778     mca_btl_base_registration_handle_t *target_handle;
 779     ompi_osc_rdma_sync_t *sync;
 780     uint64_t target_address;
 781     ptrdiff_t true_lb, true_extent;
 782     bool lock_acquired = false;
 783     int ret;
 784 
 785     OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "cswap: 0x%lx, 0x%lx, 0x%lx, %s, %d, %d, %s",
 786                      (unsigned long) origin_addr, (unsigned long) compare_addr, (unsigned long) result_addr,
 787                      dt->name, target_rank, (int) target_disp, win->w_name);
 788 
 789     sync = ompi_osc_rdma_module_sync_lookup (module, target_rank, &peer);
 790     if (OPAL_UNLIKELY(NULL == sync)) {
 791         return OMPI_ERR_RMA_SYNC;
 792     }
 793 
 794     ret = ompi_datatype_get_true_extent(dt, &true_lb, &true_extent);
 795     if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
 796         return ret;
 797     }
 798 
 799     ret = osc_rdma_get_remote_segment (module, peer, target_disp, true_lb+true_extent, &target_address, &target_handle);
 800     if (OPAL_UNLIKELY(OPAL_SUCCESS != ret)) {
 801         return ret;
 802     }
 803 
 804     /* to ensure order wait until the previous accumulate completes */
 805     while (!ompi_osc_rdma_peer_test_set_flag (peer, OMPI_OSC_RDMA_PEER_ACCUMULATING)) {
 806         ompi_osc_rdma_progress (module);
 807     }
 808 
 809     /* get an exclusive lock on the peer */
 810     if (!ompi_osc_rdma_peer_is_exclusive (peer) && !(module->acc_single_intrinsic || win->w_acc_ops <= OMPI_WIN_ACCUMULATE_OPS_SAME_OP)) {
 811         (void) ompi_osc_rdma_lock_acquire_exclusive (module, peer, offsetof (ompi_osc_rdma_state_t, accumulate_lock));
 812         lock_acquired = true;
 813     }
 814 
 815     /* either we have and exclusive lock (via MPI_Win_lock() or the accumulate lock) or the
 816      * user has indicated that they will only use the same op (or same op and no op) for
 817      * operations on overlapping memory ranges. that indicates it is safe to go ahead and
 818      * use network atomic operations. */
 819     ret = ompi_osc_rdma_cas_atomic (sync, origin_addr, compare_addr, result_addr, dt,
 820                                     peer, target_address, target_handle, lock_acquired);
 821     if (OMPI_SUCCESS == ret) {
 822         return OMPI_SUCCESS;
 823     }
 824 
 825     if (!(lock_acquired || ompi_osc_rdma_peer_is_exclusive (peer))) {
 826         (void) ompi_osc_rdma_lock_acquire_exclusive (module, peer, offsetof (ompi_osc_rdma_state_t, accumulate_lock));
 827         lock_acquired = true;
 828     }
 829 
 830     if (ompi_osc_rdma_peer_local_base (peer)) {
 831         ret = ompi_osc_rdma_cas_local (origin_addr, compare_addr, result_addr, dt,
 832                                        peer, target_address, target_handle, module,
 833                                        lock_acquired);
 834     } else {
 835         ret = cas_rdma (sync, origin_addr, compare_addr, result_addr, dt, peer, target_address,
 836                         target_handle, lock_acquired);
 837     }
 838 
 839     if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
 840         /* operation failed. the application will most likely abort but we still want to leave the window
 841          * in working state if possible. on successful completion the above calls with clear the lock
 842          * and accumulate state */
 843         ompi_osc_rdma_peer_accumulate_cleanup (module, peer, lock_acquired);
 844     }
 845 
 846     return ret;
 847 }
 848 
 849 
 850 static inline
 851 int ompi_osc_rdma_rget_accumulate_internal (ompi_osc_rdma_sync_t *sync, const void *origin_addr, int origin_count,
 852                                             ompi_datatype_t *origin_datatype, void *result_addr, int result_count,
 853                                             ompi_datatype_t *result_datatype, ompi_osc_rdma_peer_t *peer,
 854                                             int target_rank, MPI_Aint target_disp, int target_count,
 855                                             ompi_datatype_t *target_datatype, ompi_op_t *op,
 856                                             ompi_osc_rdma_request_t *request)
 857 {
 858     ompi_osc_rdma_module_t *module = sync->module;
 859     mca_btl_base_registration_handle_t *target_handle;
 860     uint64_t target_address;
 861     ptrdiff_t lb, origin_extent, target_span;
 862     bool lock_acquired = false;
 863     int ret;
 864 
 865     /* short-circuit case. note that origin_count may be 0 if op is MPI_NO_OP */
 866     if ((result_addr && 0 == result_count) || 0 == target_count) {
 867         if (request) {
 868             ompi_osc_rdma_request_complete (request, MPI_SUCCESS);
 869         }
 870 
 871         return OMPI_SUCCESS;
 872     }
 873 
 874     target_span = opal_datatype_span(&target_datatype->super, target_count, &lb);
 875 
 876     // a buffer defined by (buf, count, dt)
 877     // will have data starting at buf+offset and ending len bytes later:
 878     ret = osc_rdma_get_remote_segment (module, peer, target_disp, target_span+lb, &target_address, &target_handle);
 879     if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
 880         return ret;
 881     }
 882 
 883     (void) ompi_datatype_get_extent (origin_datatype, &lb, &origin_extent);
 884 
 885     /* to ensure order wait until the previous accumulate completes */
 886     while (!ompi_osc_rdma_peer_test_set_flag (peer, OMPI_OSC_RDMA_PEER_ACCUMULATING)) {
 887         ompi_osc_rdma_progress (module);
 888     }
 889 
 890     /* get an exclusive lock on the peer if needed */
 891     if (!ompi_osc_rdma_peer_is_exclusive (peer) && !module->acc_single_intrinsic) {
 892         lock_acquired = true;
 893         (void) ompi_osc_rdma_lock_acquire_exclusive (module, peer, offsetof (ompi_osc_rdma_state_t, accumulate_lock));
 894     }
 895 
 896     /* if the datatype is small enough (and the count is 1) then try to directly use the hardware to execute
 897      * the atomic operation. this should be safe in all cases as either 1) the user has assured us they will
 898      * never use atomics with count > 1, 2) we have the accumulate lock, or 3) we have an exclusive lock */
 899     if (origin_extent <= 8 && 1 == origin_count && !ompi_osc_rdma_peer_local_base (peer)) {
 900         if (module->acc_use_amo && ompi_datatype_is_predefined (origin_datatype)) {
 901             if (NULL == result_addr) {
 902                 ret = ompi_osc_rdma_acc_single_atomic (sync, origin_addr, origin_datatype, origin_extent, peer, target_address,
 903                                                        target_handle, op, request, lock_acquired);
 904             } else {
 905                 ret = ompi_osc_rdma_fetch_and_op_atomic (sync, origin_addr, result_addr, origin_datatype, origin_extent, peer, target_address,
 906                                                          target_handle, op, request, lock_acquired);
 907             }
 908 
 909             if (OMPI_SUCCESS == ret) {
 910                 return OMPI_SUCCESS;
 911             }
 912         }
 913 
 914         ret = ompi_osc_rdma_fetch_and_op_cas (sync, origin_addr, result_addr, origin_datatype, origin_extent, peer, target_address,
 915                                               target_handle, op, request, lock_acquired);
 916         if (OMPI_SUCCESS == ret) {
 917             return OMPI_SUCCESS;
 918         }
 919     }
 920 
 921     /* could not use network atomics. acquire the lock if needed and continue. */
 922     if (!lock_acquired && !ompi_osc_rdma_peer_is_exclusive (peer)) {
 923         lock_acquired = true;
 924         (void) ompi_osc_rdma_lock_acquire_exclusive (module, peer, offsetof (ompi_osc_rdma_state_t, accumulate_lock));
 925     }
 926 
 927     if (ompi_osc_rdma_peer_local_base (peer)) {
 928         /* local/self optimization */
 929         ret = ompi_osc_rdma_gacc_local (origin_addr, origin_count, origin_datatype, result_addr, result_count,
 930                                         result_datatype, peer, target_address, target_handle, target_count,
 931                                         target_datatype, op, module, request, lock_acquired);
 932     } else {
 933         /* do not need to pass the lock acquired flag to this function. the value of the flag can be obtained
 934          * just by calling ompi_osc_rdma_peer_is_exclusive() in this case. */
 935         ret = ompi_osc_rdma_gacc_master (sync, origin_addr, origin_count, origin_datatype, result_addr, result_count,
 936                                          result_datatype, peer, target_address, target_handle, target_count,
 937                                          target_datatype, op, request);
 938     }
 939 
 940     if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
 941         ompi_osc_rdma_peer_accumulate_cleanup (module, peer, lock_acquired);
 942     }
 943 
 944     return ret;
 945 }
 946 
 947 int ompi_osc_rdma_get_accumulate (const void *origin_addr, int origin_count, ompi_datatype_t *origin_datatype,
 948                                   void *result_addr, int result_count, ompi_datatype_t *result_datatype,
 949                                   int target_rank, MPI_Aint target_disp, int target_count, ompi_datatype_t *target_datatype,
 950                                   ompi_op_t *op, ompi_win_t *win)
 951 {
 952     ompi_osc_rdma_module_t *module = GET_MODULE(win);
 953     ompi_osc_rdma_peer_t *peer;
 954     ompi_osc_rdma_sync_t *sync;
 955 
 956     OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "get_acc: 0x%lx, %d, %s, 0x%lx, %d, %s, %d, 0x%lx, %d, %s, %s, %s",
 957                      (unsigned long) origin_addr, origin_count, origin_datatype->name,
 958                      (unsigned long) result_addr, result_count, result_datatype->name, target_rank,
 959                      (unsigned long) target_disp, target_count, target_datatype->name, op->o_name,
 960                      win->w_name);
 961 
 962     sync = ompi_osc_rdma_module_sync_lookup (module, target_rank, &peer);
 963     if (OPAL_UNLIKELY(NULL == sync)) {
 964         return OMPI_ERR_RMA_SYNC;
 965     }
 966 
 967     return ompi_osc_rdma_rget_accumulate_internal (sync, origin_addr, origin_count, origin_datatype,
 968                                                    result_addr, result_count, result_datatype,
 969                                                    peer, target_rank, target_disp, target_count,
 970                                                    target_datatype, op, NULL);
 971 }
 972 
 973 
 974 int ompi_osc_rdma_rget_accumulate (const void *origin_addr, int origin_count, ompi_datatype_t *origin_datatype,
 975                                    void *result_addr, int result_count, ompi_datatype_t *result_datatype,
 976                                    int target_rank, MPI_Aint target_disp, int target_count, ompi_datatype_t *target_datatype,
 977                                    ompi_op_t *op, ompi_win_t *win, ompi_request_t **request)
 978 {
 979     ompi_osc_rdma_module_t *module = GET_MODULE(win);
 980     ompi_osc_rdma_peer_t *peer;
 981     ompi_osc_rdma_request_t *rdma_request;
 982     ompi_osc_rdma_sync_t *sync;
 983     int ret;
 984 
 985     OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "rget_acc: 0x%lx, %d, %s, 0x%lx, %d, %s, %d, 0x%lx, %d, %s, %s, %s",
 986                      (unsigned long) origin_addr, origin_count, origin_datatype->name,
 987                      (unsigned long) result_addr, result_count, result_datatype->name, target_rank,
 988                      (unsigned long) target_disp, target_count, target_datatype->name, op->o_name,
 989                      win->w_name);
 990 
 991     sync = ompi_osc_rdma_module_sync_lookup (module, target_rank, &peer);
 992     if (OPAL_UNLIKELY(NULL == sync)) {
 993         return OMPI_ERR_RMA_SYNC;
 994     }
 995 
 996     OMPI_OSC_RDMA_REQUEST_ALLOC(module, peer, rdma_request);
 997 
 998     ret = ompi_osc_rdma_rget_accumulate_internal (sync, origin_addr, origin_count, origin_datatype, result_addr,
 999                                                   result_count, result_datatype, peer, target_rank, target_disp,
1000                                                   target_count, target_datatype, op, rdma_request);
1001     if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
1002         OMPI_OSC_RDMA_REQUEST_RETURN(rdma_request);
1003         return ret;
1004     }
1005 
1006     *request = &rdma_request->super;
1007 
1008     return OMPI_SUCCESS;
1009 }
1010 
1011 int ompi_osc_rdma_raccumulate (const void *origin_addr, int origin_count, ompi_datatype_t *origin_datatype, int target_rank,
1012                                ptrdiff_t target_disp, int target_count, ompi_datatype_t *target_datatype, ompi_op_t *op,
1013                                ompi_win_t *win, ompi_request_t **request)
1014 {
1015     ompi_osc_rdma_module_t *module = GET_MODULE(win);
1016     ompi_osc_rdma_peer_t *peer;
1017     ompi_osc_rdma_request_t *rdma_request;
1018     ompi_osc_rdma_sync_t *sync;
1019     int ret;
1020 
1021     OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "racc: 0x%lx, %d, %s, %d, 0x%lx, %d, %s, %s, %s",
1022                      (unsigned long) origin_addr, origin_count, origin_datatype->name, target_rank,
1023                      (unsigned long) target_disp, target_count, target_datatype->name, op->o_name, win->w_name);
1024 
1025     sync = ompi_osc_rdma_module_sync_lookup (module, target_rank, &peer);
1026     if (OPAL_UNLIKELY(NULL == sync)) {
1027         return OMPI_ERR_RMA_SYNC;
1028     }
1029 
1030     OMPI_OSC_RDMA_REQUEST_ALLOC(module, peer, rdma_request);
1031 
1032     ret = ompi_osc_rdma_rget_accumulate_internal (sync, origin_addr, origin_count, origin_datatype, NULL, 0,
1033                                                   NULL, peer, target_rank, target_disp, target_count, target_datatype,
1034                                                   op, rdma_request);
1035     if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
1036         OMPI_OSC_RDMA_REQUEST_RETURN(rdma_request);
1037         return ret;
1038     }
1039 
1040     *request = &rdma_request->super;
1041 
1042     return OMPI_SUCCESS;
1043 }
1044 
1045 int ompi_osc_rdma_accumulate (const void *origin_addr, int origin_count, ompi_datatype_t *origin_datatype, int target_rank,
1046                               ptrdiff_t target_disp, int target_count, ompi_datatype_t *target_datatype, ompi_op_t *op,
1047                               ompi_win_t *win)
1048 {
1049     ompi_osc_rdma_module_t *module = GET_MODULE(win);
1050     ompi_osc_rdma_peer_t *peer;
1051     ompi_osc_rdma_sync_t *sync;
1052 
1053     OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "acc: 0x%lx, %d, %s, %d, 0x%lx, %d, %s, %s, %s",
1054                      (unsigned long) origin_addr, origin_count, origin_datatype->name, target_rank,
1055                      (unsigned long) target_disp, target_count, target_datatype->name, op->o_name, win->w_name);
1056 
1057     sync = ompi_osc_rdma_module_sync_lookup (module, target_rank, &peer);
1058     if (OPAL_UNLIKELY(NULL == sync)) {
1059         return OMPI_ERR_RMA_SYNC;
1060     }
1061 
1062     return ompi_osc_rdma_rget_accumulate_internal (sync, origin_addr, origin_count, origin_datatype, NULL, 0,
1063                                                    NULL, peer, target_rank, target_disp, target_count, target_datatype,
1064                                                    op, NULL);
1065 }
1066 
1067 
1068 int ompi_osc_rdma_fetch_and_op (const void *origin_addr, void *result_addr, ompi_datatype_t *dt, int target_rank,
1069                                 ptrdiff_t target_disp, ompi_op_t *op, ompi_win_t *win)
1070 {
1071     ompi_osc_rdma_module_t *module = GET_MODULE(win);
1072     ompi_osc_rdma_peer_t *peer;
1073     ompi_osc_rdma_sync_t *sync;
1074 
1075     OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "fop: %p, %s, %d, %lu, %s, %s", result_addr, dt->name,
1076                      target_rank, (unsigned long) target_disp, op->o_name, win->w_name);
1077 
1078     sync = ompi_osc_rdma_module_sync_lookup (module, target_rank, &peer);
1079     if (OPAL_UNLIKELY(NULL == sync)) {
1080         return OMPI_ERR_RMA_SYNC;
1081     }
1082 
1083     return ompi_osc_rdma_rget_accumulate_internal (sync, origin_addr, 1, dt, result_addr, 1, dt, peer,
1084                                                    target_rank, target_disp, 1, dt, op, NULL);
1085 }
/* [<][>][^][v][top][bottom][index][help] */
root/ompi/mca/osc/rdma/osc_rdma_accumulate.c

DEFINITIONS
