opal/include/opal/sys/atomic

/* [<][>][^][v][top][bottom][index][help] */
This source file includes following definitions.
opal_atomic_mb
opal_atomic_wmb
opal_atomic_rmb
opal_atomic_fetch_min_32
opal_atomic_fetch_max_32
opal_atomic_fetch_min_64
opal_atomic_fetch_max_64
opal_atomic_min_fetch_32
opal_atomic_max_fetch_32
opal_atomic_min_fetch_64
opal_atomic_max_fetch_64
opal_atomic_lock_init
opal_atomic_trylock
opal_atomic_lock
opal_atomic_unlock
opal_atomic_compare_exchange_strong_128
   1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
   2 /*
   3  * Copyright (c) 2018      Los Alamos National Security, LLC. All rights
   4  *                         reserved.
   5  * Copyright (c) 2018      Research Organization for Information Science
   6  *                         and Technology (RIST).  All rights reserved.
   7  * $COPYRIGHT$
   8  *
   9  * Additional copyrights may follow
  10  *
  11  * $HEADER$
  12  */
  13 
  14 /* This file provides shims between the opal atomics interface and the C11 atomics interface. It
  15  * is intended as the first step in moving to using C11 atomics across the entire codebase. Once
  16  * all officially supported compilers offer C11 atomic (GCC 4.9.0+, icc 2018+, pgi, xlc, etc) then
  17  * this shim will go away and the codebase will be updated to use C11's atomic support
  18  * directly.
  19  * This shim contains some functions already present in atomic_impl.h because we do not include
  20  * atomic_impl.h when using C11 atomics. It would require alot of #ifdefs to avoid duplicate
  21  * definitions to be worthwhile. */
  22 
  23 #if !defined(OPAL_ATOMIC_STDC_H)
  24 #define OPAL_ATOMIC_STDC_H
  25 
  26 #include <stdatomic.h>
  27 #include <stdint.h>
  28 #include "opal_stdint.h"
  29 
  30 #define OPAL_HAVE_ATOMIC_MEM_BARRIER 1
  31 
  32 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
  33 #define OPAL_HAVE_ATOMIC_SWAP_32 1
  34 
  35 #define OPAL_HAVE_ATOMIC_MATH_32 1
  36 #define OPAL_HAVE_ATOMIC_ADD_32 1
  37 #define OPAL_HAVE_ATOMIC_AND_32 1
  38 #define OPAL_HAVE_ATOMIC_OR_32 1
  39 #define OPAL_HAVE_ATOMIC_XOR_32 1
  40 #define OPAL_HAVE_ATOMIC_SUB_32 1
  41 
  42 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1
  43 #define OPAL_HAVE_ATOMIC_SWAP_64 1
  44 
  45 #define OPAL_HAVE_ATOMIC_MATH_64 1
  46 #define OPAL_HAVE_ATOMIC_ADD_64 1
  47 #define OPAL_HAVE_ATOMIC_AND_64 1
  48 #define OPAL_HAVE_ATOMIC_OR_64 1
  49 #define OPAL_HAVE_ATOMIC_XOR_64 1
  50 #define OPAL_HAVE_ATOMIC_SUB_64 1
  51 
  52 #define OPAL_HAVE_ATOMIC_LLSC_32 0
  53 #define OPAL_HAVE_ATOMIC_LLSC_64 0
  54 #define OPAL_HAVE_ATOMIC_LLSC_PTR 0
  55 
  56 #define OPAL_HAVE_ATOMIC_MIN_32 1
  57 #define OPAL_HAVE_ATOMIC_MAX_32 1
  58 
  59 #define OPAL_HAVE_ATOMIC_MIN_64 1
  60 #define OPAL_HAVE_ATOMIC_MAX_64 1
  61 
  62 #define OPAL_HAVE_ATOMIC_SPINLOCKS 1
  63 
  64 static inline void opal_atomic_mb (void)
  65 {
  66     atomic_thread_fence (memory_order_seq_cst);
  67 }
  68 
  69 static inline void opal_atomic_wmb (void)
  70 {
  71     atomic_thread_fence (memory_order_release);
  72 }
  73 
  74 static inline void opal_atomic_rmb (void)
  75 {
  76     atomic_thread_fence (memory_order_acquire);
  77 }
  78 
  79 #define opal_atomic_compare_exchange_strong_32(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_relaxed, memory_order_relaxed)
  80 #define opal_atomic_compare_exchange_strong_64(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_relaxed, memory_order_relaxed)
  81 #define opal_atomic_compare_exchange_strong_ptr(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_relaxed, memory_order_relaxed)
  82 #define opal_atomic_compare_exchange_strong_acq_32(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_acquire, memory_order_relaxed)
  83 #define opal_atomic_compare_exchange_strong_acq_64(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_acquire, memory_order_relaxed)
  84 #define opal_atomic_compare_exchange_strong_acq_ptr(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_acquire, memory_order_relaxed)
  85 
  86 #define opal_atomic_compare_exchange_strong_rel_32(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_release, memory_order_relaxed)
  87 #define opal_atomic_compare_exchange_strong_rel_64(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_release, memory_order_relaxed)
  88 #define opal_atomic_compare_exchange_strong_rel_ptr(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_release, memory_order_relaxed)
  89 
  90 #define opal_atomic_compare_exchange_strong(addr, oldval, newval) atomic_compare_exchange_strong_explicit (addr, oldval, newval, memory_order_relaxed, memory_order_relaxed)
  91 #define opal_atomic_compare_exchange_strong_acq(addr, oldval, newval)  atomic_compare_exchange_strong_explicit (addr, oldval, newval, memory_order_acquire, memory_order_relaxed)
  92 #define opal_atomic_compare_exchange_strong_rel(addr, oldval, newval)  atomic_compare_exchange_strong_explicit (addr, oldval, newval, memory_order_release, memory_order_relaxed)
  93 
  94 #define opal_atomic_swap_32(addr, value) atomic_exchange_explicit (addr, value, memory_order_relaxed)
  95 #define opal_atomic_swap_64(addr, value) atomic_exchange_explicit (addr, value, memory_order_relaxed)
  96 #define opal_atomic_swap_ptr(addr, value) atomic_exchange_explicit (addr, value, memory_order_relaxed)
  97 
  98 #define OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(op, bits, type, operator)      \
  99     static inline type opal_atomic_fetch_ ## op ##_## bits (opal_atomic_ ## type *addr, type value) \
 100     {                                                                   \
 101         return atomic_fetch_ ## op ## _explicit (addr, value, memory_order_relaxed); \
 102     }                                                                   \
 103                                                                         \
 104     static inline type opal_atomic_## op ## _fetch_ ## bits (opal_atomic_ ## type *addr, type value) \
 105     {                                                                   \
 106         return atomic_fetch_ ## op ## _explicit (addr, value, memory_order_relaxed) operator value; \
 107     }
 108 
 109 OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(add, 32, int32_t, +)
 110 OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(add, 64, int64_t, +)
 111 OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(add, size_t, size_t, +)
 112 
 113 OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(sub, 32, int32_t, -)
 114 OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(sub, 64, int64_t, -)
 115 OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(sub, size_t, size_t, -)
 116 
 117 OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(or, 32, int32_t, |)
 118 OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(or, 64, int64_t, |)
 119 
 120 OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(xor, 32, int32_t, ^)
 121 OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(xor, 64, int64_t, ^)
 122 
 123 OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(and, 32, int32_t, &)
 124 OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(and, 64, int64_t, &)
 125 
 126 #define opal_atomic_add(addr, value) (void) atomic_fetch_add_explicit (addr, value, memory_order_relaxed)
 127 
 128 static inline int32_t opal_atomic_fetch_min_32 (opal_atomic_int32_t *addr, int32_t value)
 129 {
 130     int32_t old = *addr;
 131     do {
 132         if (old <= value) {
 133             break;
 134         }
 135     } while (!opal_atomic_compare_exchange_strong_32 (addr, &old, value));
 136 
 137     return old;
 138 }
 139 
 140 static inline int32_t opal_atomic_fetch_max_32 (opal_atomic_int32_t *addr, int32_t value)
 141 {
 142     int32_t old = *addr;
 143     do {
 144         if (old >= value) {
 145             break;
 146         }
 147     } while (!opal_atomic_compare_exchange_strong_32 (addr, &old, value));
 148 
 149     return old;
 150 }
 151 
 152 static inline int64_t opal_atomic_fetch_min_64 (opal_atomic_int64_t *addr, int64_t value)
 153 {
 154     int64_t old = *addr;
 155     do {
 156         if (old <= value) {
 157             break;
 158         }
 159     } while (!opal_atomic_compare_exchange_strong_64 (addr, &old, value));
 160 
 161     return old;
 162 }
 163 
 164 static inline int64_t opal_atomic_fetch_max_64 (opal_atomic_int64_t *addr, int64_t value)
 165 {
 166     int64_t old = *addr;
 167     do {
 168         if (old >= value) {
 169             break;
 170         }
 171     } while (!opal_atomic_compare_exchange_strong_64 (addr, &old, value));
 172 
 173     return old;
 174 }
 175 
 176 static inline int32_t opal_atomic_min_fetch_32 (opal_atomic_int32_t *addr, int32_t value)
 177 {
 178     int32_t old = opal_atomic_fetch_min_32 (addr, value);
 179     return old <= value ? old : value;
 180 }
 181 
 182 static inline int32_t opal_atomic_max_fetch_32 (opal_atomic_int32_t *addr, int32_t value)
 183 {
 184     int32_t old = opal_atomic_fetch_max_32 (addr, value);
 185     return old >= value ? old : value;
 186 }
 187 
 188 static inline int64_t opal_atomic_min_fetch_64 (opal_atomic_int64_t *addr, int64_t value)
 189 {
 190     int64_t old = opal_atomic_fetch_min_64 (addr, value);
 191     return old <= value ? old : value;
 192 }
 193 
 194 static inline int64_t opal_atomic_max_fetch_64 (opal_atomic_int64_t *addr, int64_t value)
 195 {
 196     int64_t old = opal_atomic_fetch_max_64 (addr, value);
 197     return old >= value ? old : value;
 198 }
 199 
 200 #define OPAL_ATOMIC_LOCK_UNLOCKED false
 201 #define OPAL_ATOMIC_LOCK_LOCKED true
 202 
 203 #define OPAL_ATOMIC_LOCK_INIT ATOMIC_FLAG_INIT
 204 
 205 typedef atomic_flag opal_atomic_lock_t;
 206 
 207 /*
 208  * Lock initialization function. It set the lock to UNLOCKED.
 209  */
 210 static inline void opal_atomic_lock_init (opal_atomic_lock_t *lock, bool value)
 211 {
 212     atomic_flag_clear (lock);
 213 }
 214 
 215 
 216 static inline int opal_atomic_trylock (opal_atomic_lock_t *lock)
 217 {
 218     return (int) atomic_flag_test_and_set (lock);
 219 }
 220 
 221 
 222 static inline void opal_atomic_lock(opal_atomic_lock_t *lock)
 223 {
 224     while (opal_atomic_trylock (lock)) {
 225     }
 226 }
 227 
 228 
 229 static inline void opal_atomic_unlock (opal_atomic_lock_t *lock)
 230 {
 231     atomic_flag_clear (lock);
 232 }
 233 
 234 
 235 #if OPAL_HAVE_C11_CSWAP_INT128
 236 
 237 /* the C11 atomic compare-exchange is lock free so use it */
 238 #define opal_atomic_compare_exchange_strong_128 atomic_compare_exchange_strong
 239 
 240 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1
 241 
 242 #elif OPAL_HAVE_SYNC_BUILTIN_CSWAP_INT128
 243 
 244 /* fall back on the __sync builtin if available since it will emit the expected instruction on x86_64 (cmpxchng16b) */
 245 __opal_attribute_always_inline__
 246 static inline bool opal_atomic_compare_exchange_strong_128 (opal_atomic_int128_t *addr,
 247                                                             opal_int128_t *oldval, opal_int128_t newval)
 248 {
 249     opal_int128_t prev = __sync_val_compare_and_swap (addr, *oldval, newval);
 250     bool ret = prev == *oldval;
 251     *oldval = prev;
 252     return ret;
 253 }
 254 
 255 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1
 256 
 257 #else
 258 
 259 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 0
 260 
 261 #endif
 262 
 263 #endif /* !defined(OPAL_ATOMIC_STDC_H) */
/* [<][>][^][v][top][bottom][index][help] */
root/opal/include/opal/sys/atomic_stdc.h

INCLUDED FROM

DEFINITIONS
