root/opal/include/opal/sys/x86_64/atomic.h

INCLUDED FROM

DEFINITIONS

1. opal_atomic_mb
2. opal_atomic_rmb
3. opal_atomic_wmb
4. opal_atomic_isync
5. opal_atomic_compare_exchange_strong_32
6. opal_atomic_compare_exchange_strong_64
7. opal_atomic_compare_exchange_strong_128
8. opal_atomic_swap_32
9. opal_atomic_swap_64
10. opal_atomic_fetch_add_32
11. opal_atomic_fetch_add_64
12. opal_atomic_fetch_sub_32
13. opal_atomic_fetch_sub_64

   1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
   2 /*
   3  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
   4  *                         University Research and Technology
   5  *                         Corporation.  All rights reserved.
   6  * Copyright (c) 2004-2010 The University of Tennessee and The University
   7  *                         of Tennessee Research Foundation.  All rights
   8  *                         reserved.
   9  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
  10  *                         University of Stuttgart.  All rights reserved.
  11  * Copyright (c) 2004-2005 The Regents of the University of California.
  12  *                         All rights reserved.
  13  * Copyright (c) 2007      Sun Microsystems, Inc.  All rights reserverd.
  14  * Copyright (c) 2012-2018 Los Alamos National Security, LLC. All rights
  15  *                         reserved.
  16  * Copyright (c) 2016-2017 Research Organization for Information Science
  17  *                         and Technology (RIST). All rights reserved.
  18  * $COPYRIGHT$
  19  *
  20  * Additional copyrights may follow
  21  *
  22  * $HEADER$
  23  */
  24 #ifndef OPAL_SYS_ARCH_ATOMIC_H
  25 #define OPAL_SYS_ARCH_ATOMIC_H 1
  26 
  27 /*
  28  * On x86_64, we use cmpxchg.
  29  */
  30 
  31 
  32 #define SMPLOCK "lock; "
  33 #define MB() __asm__ __volatile__("": : :"memory")
  34 
  35 
  36 /**********************************************************************
  37  *
  38  * Define constants for AMD64 / x86_64 / EM64T / ...
  39  *
  40  *********************************************************************/
  41 #define OPAL_HAVE_ATOMIC_MEM_BARRIER 1
  42 
  43 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
  44 
  45 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1
  46 
  47 /**********************************************************************
  48  *
  49  * Memory Barriers
  50  *
  51  *********************************************************************/
  52 #if OPAL_GCC_INLINE_ASSEMBLY
  53 
  54 static inline void opal_atomic_mb(void)
  55 {
  56     MB();
  57 }
  58 
  59 
  60 static inline void opal_atomic_rmb(void)
  61 {
  62     MB();
  63 }
  64 
  65 
  66 static inline void opal_atomic_wmb(void)
  67 {
  68     MB();
  69 }
  70 
  71 static inline void opal_atomic_isync(void)
  72 {
  73 }
  74 
  75 #endif /* OPAL_GCC_INLINE_ASSEMBLY */
  76 
  77 
  78 /**********************************************************************
  79  *
  80  * Atomic math operations
  81  *
  82  *********************************************************************/
  83 #if OPAL_GCC_INLINE_ASSEMBLY
  84 
  85 static inline bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
  86 {
  87    unsigned char ret;
  88    __asm__ __volatile__ (
  89                        SMPLOCK "cmpxchgl %3,%2   \n\t"
  90                                "sete     %0      \n\t"
  91                        : "=qm" (ret), "+a" (*oldval), "+m" (*addr)
  92                        : "q"(newval)
  93                        : "memory", "cc");
  94 
  95    return (bool) ret;
  96 }
  97 
  98 #endif /* OPAL_GCC_INLINE_ASSEMBLY */
  99 
 100 #define opal_atomic_compare_exchange_strong_acq_32 opal_atomic_compare_exchange_strong_32
 101 #define opal_atomic_compare_exchange_strong_rel_32 opal_atomic_compare_exchange_strong_32
 102 
 103 #if OPAL_GCC_INLINE_ASSEMBLY
 104 
 105 static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
 106 {
 107    unsigned char ret;
 108    __asm__ __volatile__ (
 109                        SMPLOCK "cmpxchgq %3,%2   \n\t"
 110                                "sete     %0      \n\t"
 111                        : "=qm" (ret), "+a" (*oldval), "+m" (*((opal_atomic_long_t *)addr))
 112                        : "q"(newval)
 113                        : "memory", "cc"
 114                        );
 115 
 116    return (bool) ret;
 117 }
 118 
 119 #endif /* OPAL_GCC_INLINE_ASSEMBLY */
 120 
 121 #define opal_atomic_compare_exchange_strong_acq_64 opal_atomic_compare_exchange_strong_64
 122 #define opal_atomic_compare_exchange_strong_rel_64 opal_atomic_compare_exchange_strong_64
 123 
 124 #if OPAL_GCC_INLINE_ASSEMBLY && OPAL_HAVE_CMPXCHG16B && HAVE_OPAL_INT128_T
 125 
 126 static inline bool opal_atomic_compare_exchange_strong_128 (opal_atomic_int128_t *addr, opal_int128_t *oldval, opal_int128_t newval)
 127 {
 128     unsigned char ret;
 129 
 130     /* cmpxchg16b compares the value at the address with eax:edx (low:high). if the values are
 131      * the same the contents of ebx:ecx are stores at the address. in all cases the value stored
 132      * at the address is returned in eax:edx. */
 133     __asm__ __volatile__ (SMPLOCK "cmpxchg16b (%%rsi)   \n\t"
 134                                   "sete     %0      \n\t"
 135                           : "=qm" (ret), "+a" (((int64_t *)oldval)[0]), "+d" (((int64_t *)oldval)[1])
 136                           : "S" (addr), "b" (((int64_t *)&newval)[0]), "c" (((int64_t *)&newval)[1])
 137                           : "memory", "cc");
 138 
 139     return (bool) ret;
 140 }
 141 
 142 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1
 143 
 144 #endif /* OPAL_GCC_INLINE_ASSEMBLY */
 145 
 146 
 147 #if OPAL_GCC_INLINE_ASSEMBLY
 148 
 149 #define OPAL_HAVE_ATOMIC_SWAP_32 1
 150 
 151 #define OPAL_HAVE_ATOMIC_SWAP_64 1
 152 
 153 static inline int32_t opal_atomic_swap_32( opal_atomic_int32_t *addr,
 154                                            int32_t newval)
 155 {
 156     int32_t oldval;
 157 
 158     __asm__ __volatile__("xchg %1, %0" :
 159                          "=r" (oldval), "+m" (*addr) :
 160                          "0" (newval) :
 161                          "memory");
 162     return oldval;
 163 }
 164 
 165 #endif /* OPAL_GCC_INLINE_ASSEMBLY */
 166 
 167 #if OPAL_GCC_INLINE_ASSEMBLY
 168 
 169 static inline int64_t opal_atomic_swap_64( opal_atomic_int64_t *addr,
 170                                            int64_t newval)
 171 {
 172     int64_t oldval;
 173 
 174     __asm__ __volatile__("xchgq %1, %0" :
 175                          "=r" (oldval), "+m" (*addr) :
 176                          "0" (newval) :
 177                          "memory");
 178     return oldval;
 179 }
 180 
 181 #endif /* OPAL_GCC_INLINE_ASSEMBLY */
 182 
 183 
 184 
 185 #if OPAL_GCC_INLINE_ASSEMBLY
 186 
 187 #define OPAL_HAVE_ATOMIC_MATH_32 1
 188 #define OPAL_HAVE_ATOMIC_MATH_64 1
 189 
 190 #define OPAL_HAVE_ATOMIC_ADD_32 1
 191 
 192 /**
 193  * atomic_add - add integer to atomic variable
 194  * @i: integer value to add
 195  * @v: pointer of type int
 196  *
 197  * Atomically adds @i to @v.
 198  */
 199 static inline int32_t opal_atomic_fetch_add_32(opal_atomic_int32_t* v, int i)
 200 {
 201     int ret = i;
 202    __asm__ __volatile__(
 203                         SMPLOCK "xaddl %1,%0"
 204                         :"+m" (*v), "+r" (ret)
 205                         :
 206                         :"memory", "cc"
 207                         );
 208    return ret;
 209 }
 210 
 211 #define OPAL_HAVE_ATOMIC_ADD_64 1
 212 
 213 /**
 214  * atomic_add - add integer to atomic variable
 215  * @i: integer value to add
 216  * @v: pointer of type int
 217  *
 218  * Atomically adds @i to @v.
 219  */
 220 static inline int64_t opal_atomic_fetch_add_64(opal_atomic_int64_t* v, int64_t i)
 221 {
 222     int64_t ret = i;
 223    __asm__ __volatile__(
 224                         SMPLOCK "xaddq %1,%0"
 225                         :"+m" (*v), "+r" (ret)
 226                         :
 227                         :"memory", "cc"
 228                         );
 229    return ret;
 230 }
 231 
 232 #define OPAL_HAVE_ATOMIC_SUB_32 1
 233 
 234 /**
 235  * atomic_sub - subtract the atomic variable
 236  * @i: integer value to subtract
 237  * @v: pointer of type int
 238  *
 239  * Atomically subtracts @i from @v.
 240  */
 241 static inline int32_t opal_atomic_fetch_sub_32(opal_atomic_int32_t* v, int i)
 242 {
 243     int ret = -i;
 244    __asm__ __volatile__(
 245                         SMPLOCK "xaddl %1,%0"
 246                         :"+m" (*v), "+r" (ret)
 247                         :
 248                         :"memory", "cc"
 249                         );
 250    return ret;
 251 }
 252 
 253 #define OPAL_HAVE_ATOMIC_SUB_64 1
 254 
 255 /**
 256  * atomic_sub - subtract the atomic variable
 257  * @i: integer value to subtract
 258  * @v: pointer of type int
 259  *
 260  * Atomically subtracts @i from @v.
 261  */
 262 static inline int64_t opal_atomic_fetch_sub_64(opal_atomic_int64_t* v, int64_t i)
 263 {
 264     int64_t ret = -i;
 265    __asm__ __volatile__(
 266                         SMPLOCK "xaddq %1,%0"
 267                         :"+m" (*v), "+r" (ret)
 268                         :
 269                         :"memory", "cc"
 270                         );
 271    return ret;
 272 }
 273 
 274 #endif /* OPAL_GCC_INLINE_ASSEMBLY */
 275 
 276 #endif /* ! OPAL_SYS_ARCH_ATOMIC_H */