This source file includes following definitions.
- pmix_atomic_mb
- pmix_atomic_rmb
- pmix_atomic_wmb
- pmix_atomic_compare_exchange_strong_acq_32
- pmix_atomic_compare_exchange_strong_rel_32
- pmix_atomic_compare_exchange_strong_32
- pmix_atomic_swap_32
- pmix_atomic_fetch_add_32
- pmix_atomic_fetch_and_32
- pmix_atomic_fetch_or_32
- pmix_atomic_fetch_xor_32
- pmix_atomic_fetch_sub_32
- pmix_atomic_compare_exchange_strong_acq_64
- pmix_atomic_compare_exchange_strong_rel_64
- pmix_atomic_compare_exchange_strong_64
- pmix_atomic_swap_64
- pmix_atomic_fetch_add_64
- pmix_atomic_fetch_and_64
- pmix_atomic_fetch_or_64
- pmix_atomic_fetch_xor_64
- pmix_atomic_fetch_sub_64
- pmix_atomic_compare_exchange_strong_128
- pmix_atomic_compare_exchange_strong_128
- pmix_atomic_lock_init
- pmix_atomic_trylock
- pmix_atomic_lock
- pmix_atomic_unlock
   1 
   2 
   3 
   4 
   5 
   6 
   7 
   8 
   9 
  10 
  11 
  12 
  13 
  14 
  15 
  16 
  17 
  18 
  19 
  20 
  21 
  22 
  23 
  24 
  25 
  26 
  27 
  28 #ifndef PMIX_SYS_ARCH_ATOMIC_H
  29 #define PMIX_SYS_ARCH_ATOMIC_H 1
  30 
  31 
  32 
  33 
  34 
  35 
  36 #define PMIX_HAVE_ATOMIC_MEM_BARRIER 1
  37 
  38 #define PMIX_HAVE_ATOMIC_MATH_32 1
  39 #define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
  40 #define PMIX_HAVE_ATOMIC_ADD_32 1
  41 #define PMIX_HAVE_ATOMIC_AND_32 1
  42 #define PMIX_HAVE_ATOMIC_OR_32 1
  43 #define PMIX_HAVE_ATOMIC_XOR_32 1
  44 #define PMIX_HAVE_ATOMIC_SUB_32 1
  45 #define PMIX_HAVE_ATOMIC_SWAP_32 1
  46 #define PMIX_HAVE_ATOMIC_MATH_64 1
  47 #define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1
  48 #define PMIX_HAVE_ATOMIC_ADD_64 1
  49 #define PMIX_HAVE_ATOMIC_AND_64 1
  50 #define PMIX_HAVE_ATOMIC_OR_64 1
  51 #define PMIX_HAVE_ATOMIC_XOR_64 1
  52 #define PMIX_HAVE_ATOMIC_SUB_64 1
  53 #define PMIX_HAVE_ATOMIC_SWAP_64 1
  54 
  55 
  56 static inline void pmix_atomic_mb(void)
  57 {
  58     __atomic_thread_fence (__ATOMIC_SEQ_CST);
  59 }
  60 
  61 static inline void pmix_atomic_rmb(void)
  62 {
  63 #if PMIX_ASSEMBLY_ARCH == PMIX_X86_64
  64     
  65 
  66 
  67     __atomic_thread_fence (__ATOMIC_SEQ_CST);
  68 #else
  69     __atomic_thread_fence (__ATOMIC_ACQUIRE);
  70 #endif
  71 }
  72 
  73 static inline void pmix_atomic_wmb(void)
  74 {
  75     __atomic_thread_fence (__ATOMIC_RELEASE);
  76 }
  77 
  78 #define PMIXMB() pmix_atomic_mb()
  79 
  80 
  81 
  82 
  83 
  84 
  85 
  86 
  87 
  88 
  89  
  90 #if defined(__SUNPRO_C) || defined(__SUNPRO_CC)
  91 #pragma error_messages(off, E_ARG_INCOMPATIBLE_WITH_ARG_L)
  92 #endif
  93 
  94 static inline bool pmix_atomic_compare_exchange_strong_acq_32 (pmix_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
  95 {
  96     return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
  97 }
  98 
  99 
 100 static inline bool pmix_atomic_compare_exchange_strong_rel_32 (pmix_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
 101 {
 102     return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
 103 }
 104 
 105 static inline bool pmix_atomic_compare_exchange_strong_32 (pmix_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
 106 {
 107     return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
 108 }
 109 
 110 static inline int32_t pmix_atomic_swap_32 (pmix_atomic_int32_t *addr, int32_t newval)
 111 {
 112     int32_t oldval;
 113     __atomic_exchange (addr, &newval, &oldval, __ATOMIC_RELAXED);
 114     return oldval;
 115 }
 116 
 117 static inline int32_t pmix_atomic_fetch_add_32(pmix_atomic_int32_t *addr, int32_t delta)
 118 {
 119     return __atomic_fetch_add (addr, delta, __ATOMIC_RELAXED);
 120 }
 121 
 122 static inline int32_t pmix_atomic_fetch_and_32(pmix_atomic_int32_t *addr, int32_t value)
 123 {
 124     return __atomic_fetch_and (addr, value, __ATOMIC_RELAXED);
 125 }
 126 
 127 static inline int32_t pmix_atomic_fetch_or_32(pmix_atomic_int32_t *addr, int32_t value)
 128 {
 129     return __atomic_fetch_or (addr, value, __ATOMIC_RELAXED);
 130 }
 131 
 132 static inline int32_t pmix_atomic_fetch_xor_32(pmix_atomic_int32_t *addr, int32_t value)
 133 {
 134     return __atomic_fetch_xor (addr, value, __ATOMIC_RELAXED);
 135 }
 136 
 137 static inline int32_t pmix_atomic_fetch_sub_32(pmix_atomic_int32_t *addr, int32_t delta)
 138 {
 139     return __atomic_fetch_sub (addr, delta, __ATOMIC_RELAXED);
 140 }
 141 
 142 static inline bool pmix_atomic_compare_exchange_strong_acq_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
 143 {
 144     return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
 145 }
 146 
 147 static inline bool pmix_atomic_compare_exchange_strong_rel_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
 148 {
 149     return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
 150 }
 151 
 152 
 153 static inline bool pmix_atomic_compare_exchange_strong_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
 154 {
 155     return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
 156 }
 157 
 158 static inline int64_t pmix_atomic_swap_64 (pmix_atomic_int64_t *addr, int64_t newval)
 159 {
 160     int64_t oldval;
 161     __atomic_exchange (addr, &newval, &oldval, __ATOMIC_RELAXED);
 162     return oldval;
 163 }
 164 
 165 static inline int64_t pmix_atomic_fetch_add_64(pmix_atomic_int64_t *addr, int64_t delta)
 166 {
 167     return __atomic_fetch_add (addr, delta, __ATOMIC_RELAXED);
 168 }
 169 
 170 static inline int64_t pmix_atomic_fetch_and_64(pmix_atomic_int64_t *addr, int64_t value)
 171 {
 172     return __atomic_fetch_and (addr, value, __ATOMIC_RELAXED);
 173 }
 174 
 175 static inline int64_t pmix_atomic_fetch_or_64(pmix_atomic_int64_t *addr, int64_t value)
 176 {
 177     return __atomic_fetch_or (addr, value, __ATOMIC_RELAXED);
 178 }
 179 
 180 static inline int64_t pmix_atomic_fetch_xor_64(pmix_atomic_int64_t *addr, int64_t value)
 181 {
 182     return __atomic_fetch_xor (addr, value, __ATOMIC_RELAXED);
 183 }
 184 
 185 static inline int64_t pmix_atomic_fetch_sub_64(pmix_atomic_int64_t *addr, int64_t delta)
 186 {
 187     return __atomic_fetch_sub (addr, delta, __ATOMIC_RELAXED);
 188 }
 189 
 190 #if PMIX_HAVE_GCC_BUILTIN_CSWAP_INT128
 191 
 192 #define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1
 193 
 194 static inline bool pmix_atomic_compare_exchange_strong_128 (pmix_atomic_int128_t *addr,
 195                                                             pmix_int128_t *oldval, pmix_int128_t newval)
 196 {
 197     return __atomic_compare_exchange_n (addr, oldval, newval, false,
 198                                         __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
 199 }
 200 
 201 #elif defined(PMIX_HAVE_SYNC_BUILTIN_CSWAP_INT128) && PMIX_HAVE_SYNC_BUILTIN_CSWAP_INT128
 202 
 203 #define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1
 204 
 205 
 206 
 207 static inline bool pmix_atomic_compare_exchange_strong_128 (pmix_atomic_pmix_int128_t *addr,
 208                                                             pmix_int128_t *oldval, pmix_int128_t newval)
 209 {
 210     pmix_int128_t prev = __sync_val_compare_and_swap (addr, *oldval, newval);
 211     bool ret = prev == *oldval;
 212     *oldval = prev;
 213     return ret;
 214 }
 215 
 216 #endif
 217 
 218 #if defined(__HLE__)
 219 
 220 #include <immintrin.h>
 221 
 222 #define PMIX_HAVE_ATOMIC_SPINLOCKS 1
 223 
 224 static inline void pmix_atomic_lock_init (pmix_atomic_lock_t* lock, int32_t value)
 225 {
 226    lock->u.lock = value;
 227 }
 228 
 229 static inline int pmix_atomic_trylock(pmix_atomic_lock_t *lock)
 230 {
 231     int ret = __atomic_exchange_n (&lock->u.lock, PMIX_ATOMIC_LOCK_LOCKED,
 232                                    __ATOMIC_ACQUIRE | __ATOMIC_HLE_ACQUIRE);
 233     if (PMIX_ATOMIC_LOCK_LOCKED == ret) {
 234         
 235         _mm_pause ();
 236         return 1;
 237     }
 238 
 239     return 0;
 240 }
 241 
 242 static inline void pmix_atomic_lock (pmix_atomic_lock_t *lock)
 243 {
 244     while (PMIX_ATOMIC_LOCK_LOCKED == __atomic_exchange_n (&lock->u.lock, PMIX_ATOMIC_LOCK_LOCKED,
 245                                                       __ATOMIC_ACQUIRE | __ATOMIC_HLE_ACQUIRE)) {
 246         
 247         _mm_pause ();
 248     }
 249 }
 250 
 251 static inline void pmix_atomic_unlock (pmix_atomic_lock_t *lock)
 252 {
 253     __atomic_store_n (&lock->u.lock, PMIX_ATOMIC_LOCK_UNLOCKED,
 254                        __ATOMIC_RELEASE | __ATOMIC_HLE_RELEASE);
 255 }
 256 
 257 #endif
 258 
 259 #if defined(__SUNPRO_C) || defined(__SUNPRO_CC)
 260 #pragma error_messages(default, E_ARG_INCOMPATIBLE_WITH_ARG_L)
 261 #endif
 262 
 263 #endif