root/ompi/mca/osc/rdma/osc_rdma_lock.h

INCLUDED FROM

DEFINITIONS

1. ompi_osc_rdma_trylock_local
2. ompi_osc_rdma_unlock_local
3. ompi_osc_rdma_btl_fop
4. ompi_osc_rdma_lock_btl_fop
5. ompi_osc_rdma_btl_op
6. ompi_osc_rdma_lock_btl_op
7. ompi_osc_rdma_btl_cswap
8. ompi_osc_rdma_lock_btl_cswap
9. ompi_osc_rdma_lock_release_shared
10. ompi_osc_rdma_lock_acquire_shared
11. ompi_osc_rdma_lock_try_acquire_exclusive
12. ompi_osc_rdma_lock_acquire_exclusive
13. ompi_osc_rdma_lock_release_exclusive

   1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
   2 /*
   3  * Copyright (c) 2014-2018 Los Alamos National Security, LLC.  All rights
   4  *                         reserved.
   5  * $COPYRIGHT$
   6  *
   7  * Additional copyrights may follow
   8  *
   9  * $HEADER$
  10  */
  11 
  12 #if !defined(OMPI_OSC_RDMA_LOCK_H)
  13 #define OMPI_OSC_RDMA_LOCK_H
  14 
  15 #include "osc_rdma_types.h"
  16 #include "osc_rdma_frag.h"
  17 
  18 static inline int ompi_osc_rdma_trylock_local (ompi_osc_rdma_atomic_lock_t *lock)
  19 {
  20     ompi_osc_rdma_lock_t _tmp_value = 0;
  21     return !ompi_osc_rdma_lock_compare_exchange (lock, &_tmp_value, OMPI_OSC_RDMA_LOCK_EXCLUSIVE);
  22 }
  23 
  24 static inline void ompi_osc_rdma_unlock_local (ompi_osc_rdma_atomic_lock_t *lock)
  25 {
  26     (void) ompi_osc_rdma_lock_add (lock, -OMPI_OSC_RDMA_LOCK_EXCLUSIVE);
  27 }
  28 
  29 /**
  30  * Dummy completion function for atomic operations
  31  */
  32 void ompi_osc_rdma_atomic_complete (mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
  33                                     void *local_address, mca_btl_base_registration_handle_t *local_handle,
  34                                     void *context, void *data, int status);
  35 
  36 __opal_attribute_always_inline__
  37 static inline int ompi_osc_rdma_btl_fop (ompi_osc_rdma_module_t *module, struct mca_btl_base_endpoint_t *endpoint,
  38                                          uint64_t address, mca_btl_base_registration_handle_t *address_handle, int op,
  39                                          int64_t operand, int flags, int64_t *result, const bool wait_for_completion,
  40                                          ompi_osc_rdma_pending_op_cb_fn_t cbfunc, void *cbdata, void *cbcontext)
  41 {
  42     ompi_osc_rdma_pending_op_t *pending_op;
  43     int ret = OPAL_ERROR;
  44 
  45     pending_op = OBJ_NEW(ompi_osc_rdma_pending_op_t);
  46     assert (NULL != pending_op);
  47 
  48     if (wait_for_completion) {
  49         OBJ_RETAIN(pending_op);
  50     } else {
  51         /* NTH: need to keep track of pending ops to avoid a potential teardown problem */
  52         pending_op->module = module;
  53         (void) opal_atomic_fetch_add_32 (&module->pending_ops, 1);
  54     }
  55 
  56     pending_op->op_result = (void *) result;
  57     pending_op->op_size = (MCA_BTL_ATOMIC_FLAG_32BIT & flags) ? 4 : 8;
  58     OBJ_RETAIN(pending_op);
  59     if (cbfunc) {
  60         pending_op->cbfunc = cbfunc;
  61         pending_op->cbdata = cbdata;
  62         pending_op->cbcontext = cbcontext;
  63     }
  64 
  65     /* spin until the btl has accepted the operation */
  66     do {
  67         if (NULL == pending_op->op_frag) {
  68             ret = ompi_osc_rdma_frag_alloc (module, 8, &pending_op->op_frag, (char **) &pending_op->op_buffer);
  69         }
  70 
  71         if (NULL != pending_op->op_frag) {
  72             ret = module->selected_btl->btl_atomic_fop (module->selected_btl, endpoint, pending_op->op_buffer,
  73                                                         (intptr_t) address, pending_op->op_frag->handle, address_handle,
  74                                                         op, operand, flags, MCA_BTL_NO_ORDER, ompi_osc_rdma_atomic_complete,
  75                                                         (void *) pending_op, NULL);
  76         }
  77 
  78         if (OPAL_LIKELY(!ompi_osc_rdma_oor(ret))) {
  79             break;
  80         }
  81         ompi_osc_rdma_progress (module);
  82     } while (1);
  83 
  84     if (OPAL_SUCCESS != ret) {
  85         if (OPAL_LIKELY(1 == ret)) {
  86             *result = ((int64_t *) pending_op->op_buffer)[0];
  87             ret = OMPI_SUCCESS;
  88             ompi_osc_rdma_atomic_complete (module->selected_btl, endpoint, pending_op->op_buffer,
  89                                            pending_op->op_frag->handle, (void *) pending_op, NULL, OPAL_SUCCESS);
  90         }
  91 
  92         /* need to release here because ompi_osc_rdma_atomic_complet was not called */
  93         OBJ_RELEASE(pending_op);
  94     } else if (wait_for_completion) {
  95         while (!pending_op->op_complete) {
  96             ompi_osc_rdma_progress (module);
  97         }
  98     }
  99 
 100     OBJ_RELEASE(pending_op);
 101 
 102     return ret;
 103 }
 104 
 105 __opal_attribute_always_inline__
 106 static inline int ompi_osc_rdma_lock_btl_fop (ompi_osc_rdma_module_t *module, ompi_osc_rdma_peer_t *peer, uint64_t address,
 107                                               int op, ompi_osc_rdma_lock_t operand, ompi_osc_rdma_lock_t *result,
 108                                               const bool wait_for_completion)
 109 {
 110     return ompi_osc_rdma_btl_fop (module, peer->state_endpoint, address, peer->state_handle, op, operand, 0, result,
 111                                   wait_for_completion, NULL, NULL, NULL);
 112 }
 113 
 114 __opal_attribute_always_inline__
 115 static inline int ompi_osc_rdma_btl_op (ompi_osc_rdma_module_t *module, struct mca_btl_base_endpoint_t *endpoint,
 116                                         uint64_t address, mca_btl_base_registration_handle_t *address_handle,
 117                                         int op, int64_t operand, int flags, const bool wait_for_completion,
 118                                         ompi_osc_rdma_pending_op_cb_fn_t cbfunc, void *cbdata, void *cbcontext)
 119 {
 120     ompi_osc_rdma_pending_op_t *pending_op;
 121     int ret;
 122 
 123     if (!(module->selected_btl->btl_flags & MCA_BTL_FLAGS_ATOMIC_OPS)) {
 124         return ompi_osc_rdma_btl_fop (module, endpoint, address, address_handle, op, operand, flags, NULL, wait_for_completion,
 125                                       cbfunc, cbdata, cbcontext);
 126     }
 127 
 128     pending_op = OBJ_NEW(ompi_osc_rdma_pending_op_t);
 129     assert (NULL != pending_op);
 130     OBJ_RETAIN(pending_op);
 131     if (cbfunc) {
 132         pending_op->cbfunc = cbfunc;
 133         pending_op->cbdata = cbdata;
 134         pending_op->cbcontext = cbcontext;
 135     }
 136 
 137     if (!wait_for_completion) {
 138         /* NTH: need to keep track of pending ops to avoid a potential teardown problem */
 139         pending_op->module = module;
 140         (void) opal_atomic_fetch_add_32 (&module->pending_ops, 1);
 141     }
 142 
 143     /* spin until the btl has accepted the operation */
 144     do {
 145         ret = module->selected_btl->btl_atomic_op (module->selected_btl, endpoint, (intptr_t) address, address_handle,
 146                                                    op, operand, flags, MCA_BTL_NO_ORDER, ompi_osc_rdma_atomic_complete,
 147                                                    (void *) pending_op, NULL);
 148 
 149         if (OPAL_LIKELY(!ompi_osc_rdma_oor(ret))) {
 150             break;
 151         }
 152         ompi_osc_rdma_progress (module);
 153     } while (1);
 154 
 155     if (OPAL_SUCCESS != ret) {
 156         /* need to release here because ompi_osc_rdma_atomic_complet was not called */
 157         OBJ_RELEASE(pending_op);
 158         if (OPAL_LIKELY(1 == ret)) {
 159             if (cbfunc) {
 160                 cbfunc (cbdata, cbcontext, OMPI_SUCCESS);
 161             }
 162             ret = OMPI_SUCCESS;
 163         }
 164     } else if (wait_for_completion) {
 165         while (!pending_op->op_complete) {
 166             ompi_osc_rdma_progress (module);
 167         }
 168     }
 169 
 170     OBJ_RELEASE(pending_op);
 171 
 172     return ret;
 173 }
 174 
 175 __opal_attribute_always_inline__
 176 static inline int ompi_osc_rdma_lock_btl_op (ompi_osc_rdma_module_t *module, ompi_osc_rdma_peer_t *peer, uint64_t address,
 177                                              int op, ompi_osc_rdma_lock_t operand, const bool wait_for_completion)
 178 {
 179     return ompi_osc_rdma_btl_op (module, peer->state_endpoint, address, peer->state_handle, op, operand, 0, wait_for_completion,
 180                                  NULL, NULL, NULL);
 181 }
 182 
 183 __opal_attribute_always_inline__
 184 static inline int ompi_osc_rdma_btl_cswap (ompi_osc_rdma_module_t *module, struct mca_btl_base_endpoint_t *endpoint,
 185                                            uint64_t address, mca_btl_base_registration_handle_t *address_handle,
 186                                            int64_t compare, int64_t value, int flags, int64_t *result)
 187 {
 188     ompi_osc_rdma_pending_op_t *pending_op;
 189     int ret;
 190 
 191     pending_op = OBJ_NEW(ompi_osc_rdma_pending_op_t);
 192     assert (NULL != pending_op);
 193 
 194     OBJ_RETAIN(pending_op);
 195 
 196     pending_op->op_result = (void *) result;
 197     pending_op->op_size = (MCA_BTL_ATOMIC_FLAG_32BIT & flags) ? 4 : 8;
 198 
 199     /* spin until the btl has accepted the operation */
 200     do {
 201         if (NULL == pending_op->op_frag) {
 202             ret = ompi_osc_rdma_frag_alloc (module, 8, &pending_op->op_frag, (char **) &pending_op->op_buffer);
 203         }
 204         if (NULL != pending_op->op_frag) {
 205             ret = module->selected_btl->btl_atomic_cswap (module->selected_btl, endpoint, pending_op->op_buffer,
 206                                                           address, pending_op->op_frag->handle, address_handle, compare,
 207                                                           value, flags, 0, ompi_osc_rdma_atomic_complete, (void *) pending_op,
 208                                                           NULL);
 209         }
 210 
 211         if (OPAL_LIKELY(!ompi_osc_rdma_oor(ret))) {
 212             break;
 213         }
 214         ompi_osc_rdma_progress (module);
 215     } while (1);
 216 
 217     if (OPAL_SUCCESS != ret) {
 218         if (OPAL_LIKELY(1 == ret)) {
 219             *result = ((int64_t *) pending_op->op_buffer)[0];
 220             ret = OMPI_SUCCESS;
 221         }
 222 
 223         /* need to release here because ompi_osc_rdma_atomic_complete was not called */
 224         OBJ_RELEASE(pending_op);
 225     } else {
 226         while (!pending_op->op_complete) {
 227             ompi_osc_rdma_progress (module);
 228         }
 229     }
 230 
 231     OBJ_RELEASE(pending_op);
 232 
 233     return ret;
 234 }
 235 
 236 __opal_attribute_always_inline__
 237 static inline int ompi_osc_rdma_lock_btl_cswap (ompi_osc_rdma_module_t *module, ompi_osc_rdma_peer_t *peer, uint64_t address,
 238                                                 ompi_osc_rdma_lock_t compare, ompi_osc_rdma_lock_t value, ompi_osc_rdma_lock_t *result)
 239 {
 240     return ompi_osc_rdma_btl_cswap (module, peer->state_endpoint, address, peer->state_handle, compare, value, 0, result);
 241 }
 242 
 243 /**
 244  * ompi_osc_rdma_lock_acquire_shared:
 245  *
 246  * @param[in] module    - osc/rdma module
 247  * @param[in] peer      - peer object
 248  * @param[in] value     - increment value
 249  * @param[in] offset    - offset of lock in remote peer's state segment
 250  *
 251  * @returns OMPI_SUCCESS on success and another ompi error code on failure
 252  *
 253  * This function increments a remote shared lock. The value provided in
 254  * {value} should be the negative of the one used for ompi_osc_rdma_lock_acquire_shared.
 255  * It is erroneous to release a shared lock not held by the calling process.
 256  */
 257 static inline int ompi_osc_rdma_lock_release_shared (ompi_osc_rdma_module_t *module, ompi_osc_rdma_peer_t *peer,
 258                                                      ompi_osc_rdma_lock_t value, ptrdiff_t offset)
 259 {
 260     uint64_t lock = (uint64_t) (intptr_t) peer->state + offset;
 261 
 262     OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_DEBUG, "releasing shared lock %" PRIx64 " on peer %d. value 0x%lx", lock,
 263                      peer->rank, (unsigned long) value);
 264 
 265     if (!ompi_osc_rdma_peer_local_state (peer)) {
 266         return ompi_osc_rdma_lock_btl_op (module, peer, lock, MCA_BTL_ATOMIC_ADD, value, false);
 267     }
 268 
 269     (void) ompi_osc_rdma_lock_add ((ompi_osc_rdma_atomic_lock_t *) lock, value);
 270 
 271     return OMPI_SUCCESS;
 272 }
 273 
 274 /**
 275  * ompi_osc_rdma_lock_acquire_shared:
 276  *
 277  * @param[in] module    - osc rdma module
 278  * @param[in] peer      - owner of lock
 279  * @param[in] value     - increment value
 280  * @param[in] offset    - offset of lock in remote peer's state segment
 281  * @param[in] check     - check value for success
 282  *
 283  * @returns OMPI_SUCCESS on success and another ompi error code on failure
 284  *
 285  * This function increments a remote shared lock and checks it against the
 286  * check value in {check}. If any of the bits in the prior counter value
 287  * match those in {check} the function decrements the value and tries again.
 288  */
 289 static inline int ompi_osc_rdma_lock_acquire_shared (ompi_osc_rdma_module_t *module, ompi_osc_rdma_peer_t *peer,
 290                                                      ompi_osc_rdma_lock_t value, ptrdiff_t offset,
 291                                                      ompi_osc_rdma_lock_t check)
 292 {
 293     uint64_t lock = (uint64_t) peer->state + offset;
 294     ompi_osc_rdma_lock_t lock_state;
 295     int ret;
 296 
 297     OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_DEBUG, "acquiring shared lock %" PRIx64 " on peer %d. value 0x%lx", lock,
 298                      peer->rank, (unsigned long) value);
 299 
 300     /* spin until the lock has been acquired */
 301     if (!ompi_osc_rdma_peer_local_state (peer)) {
 302         do {
 303             ret = ompi_osc_rdma_lock_btl_fop (module, peer, lock, MCA_BTL_ATOMIC_ADD, value, &lock_state, true);
 304             if (OPAL_UNLIKELY(OPAL_SUCCESS != ret)) {
 305                 OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_DEBUG, "failed to increment shared lock. opal error code %d", ret);
 306                 return ret;
 307             }
 308 
 309             OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_DEBUG, "shared lock incremented. old value 0x%lx", (unsigned long) lock_state);
 310 
 311             if (!(lock_state & check)) {
 312                 break;
 313             }
 314 
 315             OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_DEBUG, "another peer has exclusive access to lock");
 316 
 317             /* NTH: i think this is correct. backoff! */
 318             ompi_osc_rdma_lock_release_shared (module, peer, -value, offset);
 319             ompi_osc_rdma_progress (module);
 320         } while (1);
 321     } else {
 322         do {
 323             lock_state = ompi_osc_rdma_lock_add ((ompi_osc_rdma_atomic_lock_t *) lock, value);
 324             OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_DEBUG, "local shared lock incremented. old value 0x%lx",
 325                              (unsigned long) lock_state);
 326             if (!(lock_state & check)) {
 327                 break;
 328             }
 329 
 330             (void) ompi_osc_rdma_lock_add ((ompi_osc_rdma_atomic_lock_t *) lock, -value);
 331             ompi_osc_rdma_progress (module);
 332         } while (1);
 333     }
 334 
 335     OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_DEBUG, "shared lock acquired");
 336 
 337     return OMPI_SUCCESS;
 338 }
 339 
 340 /**
 341  * ompi_osc_rdma_lock_try_acquire_exclusive:
 342  *
 343  * @param[in] module   - osc/rdma module
 344  * @param[in] peer     - peer object
 345  * @param[in] offset   - offset of lock in peer's state structure
 346  *
 347  * @returns 0 on success, 1 on failure
 348  *
 349  * This function attempts to obtain an exclusive lock at {offset} in a peer's state.
 350  */
 351 static inline int ompi_osc_rdma_lock_try_acquire_exclusive (ompi_osc_rdma_module_t *module, ompi_osc_rdma_peer_t *peer,
 352                                                             ptrdiff_t offset)
 353 {
 354     uint64_t lock = (uint64_t) (uintptr_t) peer->state + offset;
 355     int ret;
 356 
 357     OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_DEBUG, "trying to acquire exclusive lock %" PRIx64 " on peer %d", lock,
 358                      peer->rank);
 359 
 360     if (!ompi_osc_rdma_peer_local_state (peer)) {
 361         /* set the temporary value so we can detect success. note that a lock should never be -1 */
 362         ompi_osc_rdma_lock_t lock_state = -1;
 363 
 364         ret = ompi_osc_rdma_lock_btl_cswap (module, peer, lock, 0, OMPI_OSC_RDMA_LOCK_EXCLUSIVE, &lock_state);
 365         if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
 366             return ret;
 367         }
 368 
 369 #if OPAL_ENABLE_DEBUG
 370         if (0 == lock_state) {
 371             OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_DEBUG, "exclusive lock acquired");
 372         } else {
 373             OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_DEBUG, "could not acquire exclusive lock. lock state 0x%" PRIx64,
 374                              (uint64_t) lock_state);
 375         }
 376 #endif
 377 
 378         return lock_state != 0;
 379     }
 380 
 381     return ompi_osc_rdma_trylock_local ((ompi_osc_rdma_atomic_lock_t *)(intptr_t) lock);
 382 }
 383 
 384 /**
 385  * ompi_osc_rdma_lock_acquire_exclusive:
 386  *
 387  * @param[in] module   - osc/rdma module
 388  * @param[in] peer     - peer object
 389  * @param[in] offset   - offset into the remote peer's state segment
 390  *
 391  * @returns OMPI_SUCCESS on success or another ompi error code on failure
 392  *
 393  * This function obtains an exclusive lock at {offset} in a peer's state.
 394  */
 395 static inline int ompi_osc_rdma_lock_acquire_exclusive (ompi_osc_rdma_module_t *module, ompi_osc_rdma_peer_t *peer,
 396                                                         ptrdiff_t offset)
 397 {
 398     int ret;
 399 
 400     while (1 == (ret = ompi_osc_rdma_lock_try_acquire_exclusive (module, peer, offset))) {
 401         ompi_osc_rdma_progress (module);
 402     }
 403 
 404     return ret;
 405 }
 406 
 407 /**
 408  * ompi_osc_rdma_lock_release_exclusive:
 409  *
 410  * @param[in] peer   - peer to unlock
 411  * @param[in] offset - offset into the remote peer's state segment
 412  *
 413  * @returns OMPI_SUCCESS on success or another ompi error code on failure
 414  *
 415  * This function unlocks the lock at {offset} in the remote peer's state
 416  * structure. It is illegal to call this function unless this process
 417  * holds the lock.
 418  */
 419 static inline int ompi_osc_rdma_lock_release_exclusive (ompi_osc_rdma_module_t *module, ompi_osc_rdma_peer_t *peer,
 420                                                         ptrdiff_t offset)
 421 {
 422     uint64_t lock = (uint64_t) (intptr_t) peer->state + offset;
 423     int ret = OMPI_SUCCESS;
 424 
 425     OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_DEBUG, "releasing exclusive lock %" PRIx64 " on peer %d\n", lock, peer->rank);
 426 
 427     if (!ompi_osc_rdma_peer_local_state (peer)) {
 428         ret = ompi_osc_rdma_lock_btl_op (module, peer, lock, MCA_BTL_ATOMIC_ADD, -OMPI_OSC_RDMA_LOCK_EXCLUSIVE,
 429                                          false);
 430         if (OMPI_SUCCESS != ret) {
 431             abort ();
 432         }
 433     } else {
 434         ompi_osc_rdma_unlock_local ((ompi_osc_rdma_atomic_lock_t *)(intptr_t) lock);
 435     }
 436 
 437     OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_DEBUG, "exclusive lock released");
 438 
 439     return ret;
 440 }
 441 
 442 #endif /* OMPI_OSC_RDMA_LOCK_H */