root/opal/include/opal/sys/gcc_builtin/atomic.h

/* [<][>][^][v][top][bottom][index][help] */

INCLUDED FROM

DEFINITIONS

This source file includes following definitions.
  1. opal_atomic_mb
  2. opal_atomic_rmb
  3. opal_atomic_wmb
  4. opal_atomic_compare_exchange_strong_acq_32
  5. opal_atomic_compare_exchange_strong_rel_32
  6. opal_atomic_compare_exchange_strong_32
  7. opal_atomic_swap_32
  8. opal_atomic_fetch_add_32
  9. opal_atomic_fetch_and_32
  10. opal_atomic_fetch_or_32
  11. opal_atomic_fetch_xor_32
  12. opal_atomic_fetch_sub_32
  13. opal_atomic_compare_exchange_strong_acq_64
  14. opal_atomic_compare_exchange_strong_rel_64
  15. opal_atomic_compare_exchange_strong_64
  16. opal_atomic_swap_64
  17. opal_atomic_fetch_add_64
  18. opal_atomic_fetch_and_64
  19. opal_atomic_fetch_or_64
  20. opal_atomic_fetch_xor_64
  21. opal_atomic_fetch_sub_64
  22. opal_atomic_compare_exchange_strong_128
  23. opal_atomic_compare_exchange_strong_128
  24. opal_atomic_lock_init
  25. opal_atomic_trylock
  26. opal_atomic_lock
  27. opal_atomic_unlock
   1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
   2 /*
   3  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
   4  *                         University Research and Technology
   5  *                         Corporation.  All rights reserved.
   6  * Copyright (c) 2004-2013 The University of Tennessee and The University
   7  *                         of Tennessee Research Foundation.  All rights
   8  *                         reserved.
   9  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
  10  *                         University of Stuttgart.  All rights reserved.
  11  * Copyright (c) 2004-2005 The Regents of the University of California.
  12  *                         All rights reserved.
  13  * Copyright (c) 2011      Sandia National Laboratories. All rights reserved.
  14  * Copyright (c) 2014-2018 Los Alamos National Security, LLC. All rights
  15  *                         reserved.
  16  * Copyright (c) 2016-2017 Research Organization for Information Science
  17  *                         and Technology (RIST). All rights reserved.
  18  * Copyright (c) 2018      Triad National Security, LLC. All rights
  19  *                         reserved.
  20  * $COPYRIGHT$
  21  *
  22  * Additional copyrights may follow
  23  *
  24  * $HEADER$
  25  */
  26 
  27 #ifndef OPAL_SYS_ARCH_ATOMIC_H
  28 #define OPAL_SYS_ARCH_ATOMIC_H 1
  29 
  30 /**********************************************************************
  31  *
  32  * Memory Barriers
  33  *
  34  *********************************************************************/
  35 #define OPAL_HAVE_ATOMIC_MEM_BARRIER 1
  36 
  37 #define OPAL_HAVE_ATOMIC_MATH_32 1
  38 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
  39 #define OPAL_HAVE_ATOMIC_ADD_32 1
  40 #define OPAL_HAVE_ATOMIC_AND_32 1
  41 #define OPAL_HAVE_ATOMIC_OR_32 1
  42 #define OPAL_HAVE_ATOMIC_XOR_32 1
  43 #define OPAL_HAVE_ATOMIC_SUB_32 1
  44 #define OPAL_HAVE_ATOMIC_SWAP_32 1
  45 #define OPAL_HAVE_ATOMIC_MATH_64 1
  46 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1
  47 #define OPAL_HAVE_ATOMIC_ADD_64 1
  48 #define OPAL_HAVE_ATOMIC_AND_64 1
  49 #define OPAL_HAVE_ATOMIC_OR_64 1
  50 #define OPAL_HAVE_ATOMIC_XOR_64 1
  51 #define OPAL_HAVE_ATOMIC_SUB_64 1
  52 #define OPAL_HAVE_ATOMIC_SWAP_64 1
  53 
  54 
  55 static inline void opal_atomic_mb(void)
  56 {
  57     __atomic_thread_fence (__ATOMIC_SEQ_CST);
  58 }
  59 
  60 static inline void opal_atomic_rmb(void)
  61 {
  62 #if OPAL_ASSEMBLY_ARCH == OPAL_X86_64
  63     /* work around a bug in older gcc versions where ACQUIRE seems to get
  64      * treated as a no-op instead of being equivalent to
  65      * __asm__ __volatile__("": : :"memory") */
  66     __atomic_thread_fence (__ATOMIC_SEQ_CST);
  67 #else
  68     __atomic_thread_fence (__ATOMIC_ACQUIRE);
  69 #endif
  70 }
  71 
  72 static inline void opal_atomic_wmb(void)
  73 {
  74     __atomic_thread_fence (__ATOMIC_RELEASE);
  75 }
  76 
  77 #define MB() opal_atomic_mb()
  78 
  79 /**********************************************************************
  80  *
  81  * Atomic math operations
  82  *
  83  *********************************************************************/
  84 
  85 /*
  86  * Suppress numerous (spurious ?) warnings from Oracle Studio compilers
  87  * see https://community.oracle.com/thread/3968347
  88  */ 
  89 #if defined(__SUNPRO_C) || defined(__SUNPRO_CC)
  90 #pragma error_messages(off, E_ARG_INCOMPATIBLE_WITH_ARG_L)
  91 #endif
  92 
  93 static inline bool opal_atomic_compare_exchange_strong_acq_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
  94 {
  95     return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
  96 }
  97 
  98 
  99 static inline bool opal_atomic_compare_exchange_strong_rel_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
 100 {
 101     return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
 102 }
 103 
 104 static inline bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
 105 {
 106     return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
 107 }
 108 
 109 static inline int32_t opal_atomic_swap_32 (opal_atomic_int32_t *addr, int32_t newval)
 110 {
 111     int32_t oldval;
 112     __atomic_exchange (addr, &newval, &oldval, __ATOMIC_RELAXED);
 113     return oldval;
 114 }
 115 
 116 static inline int32_t opal_atomic_fetch_add_32(opal_atomic_int32_t *addr, int32_t delta)
 117 {
 118     return __atomic_fetch_add (addr, delta, __ATOMIC_RELAXED);
 119 }
 120 
 121 static inline int32_t opal_atomic_fetch_and_32(opal_atomic_int32_t *addr, int32_t value)
 122 {
 123     return __atomic_fetch_and (addr, value, __ATOMIC_RELAXED);
 124 }
 125 
 126 static inline int32_t opal_atomic_fetch_or_32(opal_atomic_int32_t *addr, int32_t value)
 127 {
 128     return __atomic_fetch_or (addr, value, __ATOMIC_RELAXED);
 129 }
 130 
 131 static inline int32_t opal_atomic_fetch_xor_32(opal_atomic_int32_t *addr, int32_t value)
 132 {
 133     return __atomic_fetch_xor (addr, value, __ATOMIC_RELAXED);
 134 }
 135 
 136 static inline int32_t opal_atomic_fetch_sub_32(opal_atomic_int32_t *addr, int32_t delta)
 137 {
 138     return __atomic_fetch_sub (addr, delta, __ATOMIC_RELAXED);
 139 }
 140 
 141 static inline bool opal_atomic_compare_exchange_strong_acq_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
 142 {
 143     return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
 144 }
 145 
 146 static inline bool opal_atomic_compare_exchange_strong_rel_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
 147 {
 148     return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
 149 }
 150 
 151 
 152 static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
 153 {
 154     return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
 155 }
 156 
 157 static inline int64_t opal_atomic_swap_64 (opal_atomic_int64_t *addr, int64_t newval)
 158 {
 159     int64_t oldval;
 160     __atomic_exchange (addr, &newval, &oldval, __ATOMIC_RELAXED);
 161     return oldval;
 162 }
 163 
 164 static inline int64_t opal_atomic_fetch_add_64(opal_atomic_int64_t *addr, int64_t delta)
 165 {
 166     return __atomic_fetch_add (addr, delta, __ATOMIC_RELAXED);
 167 }
 168 
 169 static inline int64_t opal_atomic_fetch_and_64(opal_atomic_int64_t *addr, int64_t value)
 170 {
 171     return __atomic_fetch_and (addr, value, __ATOMIC_RELAXED);
 172 }
 173 
 174 static inline int64_t opal_atomic_fetch_or_64(opal_atomic_int64_t *addr, int64_t value)
 175 {
 176     return __atomic_fetch_or (addr, value, __ATOMIC_RELAXED);
 177 }
 178 
 179 static inline int64_t opal_atomic_fetch_xor_64(opal_atomic_int64_t *addr, int64_t value)
 180 {
 181     return __atomic_fetch_xor (addr, value, __ATOMIC_RELAXED);
 182 }
 183 
 184 static inline int64_t opal_atomic_fetch_sub_64(opal_atomic_int64_t *addr, int64_t delta)
 185 {
 186     return __atomic_fetch_sub (addr, delta, __ATOMIC_RELAXED);
 187 }
 188 
 189 #if OPAL_HAVE_GCC_BUILTIN_CSWAP_INT128
 190 
 191 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1
 192 
 193 static inline bool opal_atomic_compare_exchange_strong_128 (opal_atomic_int128_t *addr,
 194                                                             opal_int128_t *oldval, opal_int128_t newval)
 195 {
 196     return __atomic_compare_exchange_n (addr, oldval, newval, false,
 197                                         __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
 198 }
 199 
 200 #elif defined(OPAL_HAVE_SYNC_BUILTIN_CSWAP_INT128) && OPAL_HAVE_SYNC_BUILTIN_CSWAP_INT128
 201 
 202 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1
 203 
 204 /* __atomic version is not lock-free so use legacy __sync version */
 205 
 206 static inline bool opal_atomic_compare_exchange_strong_128 (opal_atomic_opal_int128_t *addr,
 207                                                             opal_int128_t *oldval, opal_int128_t newval)
 208 {
 209     opal_int128_t prev = __sync_val_compare_and_swap (addr, *oldval, newval);
 210     bool ret = prev == *oldval;
 211     *oldval = prev;
 212     return ret;
 213 }
 214 
 215 #endif
 216 
 217 #if defined(__HLE__)
 218 
 219 #include <immintrin.h>
 220 
 221 #define OPAL_HAVE_ATOMIC_SPINLOCKS 1
 222 
 223 static inline void opal_atomic_lock_init (opal_atomic_lock_t* lock, int32_t value)
 224 {
 225    lock->u.lock = value;
 226 }
 227 
 228 static inline int opal_atomic_trylock(opal_atomic_lock_t *lock)
 229 {
 230     int ret = __atomic_exchange_n (&lock->u.lock, OPAL_ATOMIC_LOCK_LOCKED,
 231                                    __ATOMIC_ACQUIRE | __ATOMIC_HLE_ACQUIRE);
 232     if (OPAL_ATOMIC_LOCK_LOCKED == ret) {
 233         /* abort the transaction */
 234         _mm_pause ();
 235         return 1;
 236     }
 237 
 238     return 0;
 239 }
 240 
 241 static inline void opal_atomic_lock (opal_atomic_lock_t *lock)
 242 {
 243     while (OPAL_ATOMIC_LOCK_LOCKED == __atomic_exchange_n (&lock->u.lock, OPAL_ATOMIC_LOCK_LOCKED,
 244                                                       __ATOMIC_ACQUIRE | __ATOMIC_HLE_ACQUIRE)) {
 245         /* abort the transaction */
 246         _mm_pause ();
 247     }
 248 }
 249 
 250 static inline void opal_atomic_unlock (opal_atomic_lock_t *lock)
 251 {
 252     __atomic_store_n (&lock->u.lock, OPAL_ATOMIC_LOCK_UNLOCKED,
 253                        __ATOMIC_RELEASE | __ATOMIC_HLE_RELEASE);
 254 }
 255 
 256 #endif
 257 
 258 #if defined(__SUNPRO_C) || defined(__SUNPRO_CC)
 259 #pragma error_messages(default, E_ARG_INCOMPATIBLE_WITH_ARG_L)
 260 #endif
 261 
 262 #endif /* ! OPAL_SYS_ARCH_ATOMIC_H */
/* [<][>][^][v][top][bottom][index][help] */