This source file includes following definitions.
- opal_atomic_mb
- opal_atomic_rmb
- opal_atomic_wmb
- opal_atomic_compare_exchange_strong_acq_32
- opal_atomic_compare_exchange_strong_rel_32
- opal_atomic_compare_exchange_strong_32
- opal_atomic_swap_32
- opal_atomic_fetch_add_32
- opal_atomic_fetch_and_32
- opal_atomic_fetch_or_32
- opal_atomic_fetch_xor_32
- opal_atomic_fetch_sub_32
- opal_atomic_compare_exchange_strong_acq_64
- opal_atomic_compare_exchange_strong_rel_64
- opal_atomic_compare_exchange_strong_64
- opal_atomic_swap_64
- opal_atomic_fetch_add_64
- opal_atomic_fetch_and_64
- opal_atomic_fetch_or_64
- opal_atomic_fetch_xor_64
- opal_atomic_fetch_sub_64
- opal_atomic_compare_exchange_strong_128
- opal_atomic_compare_exchange_strong_128
- opal_atomic_lock_init
- opal_atomic_trylock
- opal_atomic_lock
- opal_atomic_unlock
   1 
   2 
   3 
   4 
   5 
   6 
   7 
   8 
   9 
  10 
  11 
  12 
  13 
  14 
  15 
  16 
  17 
  18 
  19 
  20 
  21 
  22 
  23 
  24 
  25 
  26 
  27 #ifndef OPAL_SYS_ARCH_ATOMIC_H
  28 #define OPAL_SYS_ARCH_ATOMIC_H 1
  29 
  30 
  31 
  32 
  33 
  34 
  35 #define OPAL_HAVE_ATOMIC_MEM_BARRIER 1
  36 
  37 #define OPAL_HAVE_ATOMIC_MATH_32 1
  38 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
  39 #define OPAL_HAVE_ATOMIC_ADD_32 1
  40 #define OPAL_HAVE_ATOMIC_AND_32 1
  41 #define OPAL_HAVE_ATOMIC_OR_32 1
  42 #define OPAL_HAVE_ATOMIC_XOR_32 1
  43 #define OPAL_HAVE_ATOMIC_SUB_32 1
  44 #define OPAL_HAVE_ATOMIC_SWAP_32 1
  45 #define OPAL_HAVE_ATOMIC_MATH_64 1
  46 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1
  47 #define OPAL_HAVE_ATOMIC_ADD_64 1
  48 #define OPAL_HAVE_ATOMIC_AND_64 1
  49 #define OPAL_HAVE_ATOMIC_OR_64 1
  50 #define OPAL_HAVE_ATOMIC_XOR_64 1
  51 #define OPAL_HAVE_ATOMIC_SUB_64 1
  52 #define OPAL_HAVE_ATOMIC_SWAP_64 1
  53 
  54 
  55 static inline void opal_atomic_mb(void)
  56 {
  57     __atomic_thread_fence (__ATOMIC_SEQ_CST);
  58 }
  59 
  60 static inline void opal_atomic_rmb(void)
  61 {
  62 #if OPAL_ASSEMBLY_ARCH == OPAL_X86_64
  63     
  64 
  65 
  66     __atomic_thread_fence (__ATOMIC_SEQ_CST);
  67 #else
  68     __atomic_thread_fence (__ATOMIC_ACQUIRE);
  69 #endif
  70 }
  71 
  72 static inline void opal_atomic_wmb(void)
  73 {
  74     __atomic_thread_fence (__ATOMIC_RELEASE);
  75 }
  76 
  77 #define MB() opal_atomic_mb()
  78 
  79 
  80 
  81 
  82 
  83 
  84 
  85 
  86 
  87 
  88  
  89 #if defined(__SUNPRO_C) || defined(__SUNPRO_CC)
  90 #pragma error_messages(off, E_ARG_INCOMPATIBLE_WITH_ARG_L)
  91 #endif
  92 
  93 static inline bool opal_atomic_compare_exchange_strong_acq_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
  94 {
  95     return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
  96 }
  97 
  98 
  99 static inline bool opal_atomic_compare_exchange_strong_rel_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
 100 {
 101     return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
 102 }
 103 
 104 static inline bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
 105 {
 106     return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
 107 }
 108 
 109 static inline int32_t opal_atomic_swap_32 (opal_atomic_int32_t *addr, int32_t newval)
 110 {
 111     int32_t oldval;
 112     __atomic_exchange (addr, &newval, &oldval, __ATOMIC_RELAXED);
 113     return oldval;
 114 }
 115 
 116 static inline int32_t opal_atomic_fetch_add_32(opal_atomic_int32_t *addr, int32_t delta)
 117 {
 118     return __atomic_fetch_add (addr, delta, __ATOMIC_RELAXED);
 119 }
 120 
 121 static inline int32_t opal_atomic_fetch_and_32(opal_atomic_int32_t *addr, int32_t value)
 122 {
 123     return __atomic_fetch_and (addr, value, __ATOMIC_RELAXED);
 124 }
 125 
 126 static inline int32_t opal_atomic_fetch_or_32(opal_atomic_int32_t *addr, int32_t value)
 127 {
 128     return __atomic_fetch_or (addr, value, __ATOMIC_RELAXED);
 129 }
 130 
 131 static inline int32_t opal_atomic_fetch_xor_32(opal_atomic_int32_t *addr, int32_t value)
 132 {
 133     return __atomic_fetch_xor (addr, value, __ATOMIC_RELAXED);
 134 }
 135 
 136 static inline int32_t opal_atomic_fetch_sub_32(opal_atomic_int32_t *addr, int32_t delta)
 137 {
 138     return __atomic_fetch_sub (addr, delta, __ATOMIC_RELAXED);
 139 }
 140 
 141 static inline bool opal_atomic_compare_exchange_strong_acq_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
 142 {
 143     return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
 144 }
 145 
 146 static inline bool opal_atomic_compare_exchange_strong_rel_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
 147 {
 148     return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
 149 }
 150 
 151 
 152 static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
 153 {
 154     return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
 155 }
 156 
 157 static inline int64_t opal_atomic_swap_64 (opal_atomic_int64_t *addr, int64_t newval)
 158 {
 159     int64_t oldval;
 160     __atomic_exchange (addr, &newval, &oldval, __ATOMIC_RELAXED);
 161     return oldval;
 162 }
 163 
 164 static inline int64_t opal_atomic_fetch_add_64(opal_atomic_int64_t *addr, int64_t delta)
 165 {
 166     return __atomic_fetch_add (addr, delta, __ATOMIC_RELAXED);
 167 }
 168 
 169 static inline int64_t opal_atomic_fetch_and_64(opal_atomic_int64_t *addr, int64_t value)
 170 {
 171     return __atomic_fetch_and (addr, value, __ATOMIC_RELAXED);
 172 }
 173 
 174 static inline int64_t opal_atomic_fetch_or_64(opal_atomic_int64_t *addr, int64_t value)
 175 {
 176     return __atomic_fetch_or (addr, value, __ATOMIC_RELAXED);
 177 }
 178 
 179 static inline int64_t opal_atomic_fetch_xor_64(opal_atomic_int64_t *addr, int64_t value)
 180 {
 181     return __atomic_fetch_xor (addr, value, __ATOMIC_RELAXED);
 182 }
 183 
 184 static inline int64_t opal_atomic_fetch_sub_64(opal_atomic_int64_t *addr, int64_t delta)
 185 {
 186     return __atomic_fetch_sub (addr, delta, __ATOMIC_RELAXED);
 187 }
 188 
 189 #if OPAL_HAVE_GCC_BUILTIN_CSWAP_INT128
 190 
 191 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1
 192 
 193 static inline bool opal_atomic_compare_exchange_strong_128 (opal_atomic_int128_t *addr,
 194                                                             opal_int128_t *oldval, opal_int128_t newval)
 195 {
 196     return __atomic_compare_exchange_n (addr, oldval, newval, false,
 197                                         __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
 198 }
 199 
 200 #elif defined(OPAL_HAVE_SYNC_BUILTIN_CSWAP_INT128) && OPAL_HAVE_SYNC_BUILTIN_CSWAP_INT128
 201 
 202 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1
 203 
 204 
 205 
 206 static inline bool opal_atomic_compare_exchange_strong_128 (opal_atomic_opal_int128_t *addr,
 207                                                             opal_int128_t *oldval, opal_int128_t newval)
 208 {
 209     opal_int128_t prev = __sync_val_compare_and_swap (addr, *oldval, newval);
 210     bool ret = prev == *oldval;
 211     *oldval = prev;
 212     return ret;
 213 }
 214 
 215 #endif
 216 
 217 #if defined(__HLE__)
 218 
 219 #include <immintrin.h>
 220 
 221 #define OPAL_HAVE_ATOMIC_SPINLOCKS 1
 222 
 223 static inline void opal_atomic_lock_init (opal_atomic_lock_t* lock, int32_t value)
 224 {
 225    lock->u.lock = value;
 226 }
 227 
 228 static inline int opal_atomic_trylock(opal_atomic_lock_t *lock)
 229 {
 230     int ret = __atomic_exchange_n (&lock->u.lock, OPAL_ATOMIC_LOCK_LOCKED,
 231                                    __ATOMIC_ACQUIRE | __ATOMIC_HLE_ACQUIRE);
 232     if (OPAL_ATOMIC_LOCK_LOCKED == ret) {
 233         
 234         _mm_pause ();
 235         return 1;
 236     }
 237 
 238     return 0;
 239 }
 240 
 241 static inline void opal_atomic_lock (opal_atomic_lock_t *lock)
 242 {
 243     while (OPAL_ATOMIC_LOCK_LOCKED == __atomic_exchange_n (&lock->u.lock, OPAL_ATOMIC_LOCK_LOCKED,
 244                                                       __ATOMIC_ACQUIRE | __ATOMIC_HLE_ACQUIRE)) {
 245         
 246         _mm_pause ();
 247     }
 248 }
 249 
 250 static inline void opal_atomic_unlock (opal_atomic_lock_t *lock)
 251 {
 252     __atomic_store_n (&lock->u.lock, OPAL_ATOMIC_LOCK_UNLOCKED,
 253                        __ATOMIC_RELEASE | __ATOMIC_HLE_RELEASE);
 254 }
 255 
 256 #endif
 257 
 258 #if defined(__SUNPRO_C) || defined(__SUNPRO_CC)
 259 #pragma error_messages(default, E_ARG_INCOMPATIBLE_WITH_ARG_L)
 260 #endif
 261 
 262 #endif