root/opal/include/opal/sys/arm/atomic.h

INCLUDED FROM

DEFINITIONS

1. opal_atomic_mb
2. opal_atomic_rmb
3. opal_atomic_wmb
4. opal_atomic_isync
5. opal_atomic_compare_exchange_strong_32
6. opal_atomic_compare_exchange_strong_acq_32
7. opal_atomic_compare_exchange_strong_rel_32
8. opal_atomic_compare_exchange_strong_64
9. opal_atomic_compare_exchange_strong_acq_64
10. opal_atomic_compare_exchange_strong_rel_64
11. opal_atomic_fetch_add_32
12. opal_atomic_fetch_sub_32

   1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
   2 /*
   3  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
   4  *                         University Research and Technology
   5  *                         Corporation.  All rights reserved.
   6  * Copyright (c) 2004-2005 The University of Tennessee and The University
   7  *                         of Tennessee Research Foundation.  All rights
   8  *                         reserved.
   9  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
  10  *                         University of Stuttgart.  All rights reserved.
  11  * Copyright (c) 2004-2005 The Regents of the University of California.
  12  *                         All rights reserved.
  13  * Copyright (c) 2010      IBM Corporation.  All rights reserved.
  14  * Copyright (c) 2010      ARM ltd.  All rights reserved.
  15  * Copyright (c) 2017-2018 Los Alamos National Security, LLC. All rights
  16  *                         reserved.
  17  * $COPYRIGHT$
  18  *
  19  * Additional copyrights may follow
  20  *
  21  * $HEADER$
  22  */
  23 
  24 /*
  25  * ARMv5 and earlier lack robust atomic operations and therefore this file uses
  26  * Linux kernel support where needed.  The kernel also provides memory barriers
  27  * and this file uses them for ARMv5 and earlier processors, which lack the
  28  * memory barrier instruction.  These kernel functions are available on kernel
  29  * versions 2.6.15 and greater; using them will result in undefined behavior on
  30  * older kernels.
  31  * See Documentation/arm/kernel_user_helpers.txt in the kernel tree for details
  32  */
  33 
  34 #ifndef OPAL_SYS_ARCH_ATOMIC_H
  35 #define OPAL_SYS_ARCH_ATOMIC_H 1
  36 
  37 #if (OPAL_ASM_ARM_VERSION >= 7)
  38 
  39 #define OPAL_HAVE_ATOMIC_MEM_BARRIER 1
  40 /* use the DMB instruction if available... */
  41 
  42 #define MB()  __asm__ __volatile__ ("dmb" : : : "memory")
  43 #define RMB() __asm__ __volatile__ ("dmb" : : : "memory")
  44 #define WMB() __asm__ __volatile__ ("dmb" : : : "memory")
  45 
  46 #elif (OPAL_ASM_ARM_VERSION == 6)
  47 
  48 #define OPAL_HAVE_ATOMIC_MEM_BARRIER 1
  49 /* ...or the v6-specific equivalent... */
  50 
  51 #define MB()  __asm__ __volatile__ ("mcr p15, 0, r0, c7, c10, 5" : : : "memory")
  52 #define RMB() MB()
  53 #define WMB() MB()
  54 
  55 #else
  56 
  57 #define OPAL_HAVE_ATOMIC_MEM_BARRIER 1
  58 /* ...otherwise use the Linux kernel-provided barrier */
  59 
  60 #define MB() (*((void (*)(void))(0xffff0fa0)))()
  61 #define RMB() MB()
  62 #define WMB() MB()
  63 
  64 #endif
  65 
  66 /**********************************************************************
  67  *
  68  * Memory Barriers
  69  *
  70  *********************************************************************/
  71 
  72 #if (OPAL_HAVE_ATOMIC_MEM_BARRIER == 1)
  73 
  74 static inline
  75 void opal_atomic_mb(void)
  76 {
  77     MB();
  78 }
  79 
  80 
  81 static inline
  82 void opal_atomic_rmb(void)
  83 {
  84     RMB();
  85 }
  86 
  87 
  88 static inline
  89 void opal_atomic_wmb(void)
  90 {
  91     WMB();
  92 }
  93 
  94 static inline
  95 void opal_atomic_isync(void)
  96 {
  97 }
  98 
  99 #endif
 100 
 101 
 102 /**********************************************************************
 103  *
 104  * Atomic math operations
 105  *
 106  *********************************************************************/
 107 
 108 #if (OPAL_GCC_INLINE_ASSEMBLY && (OPAL_ASM_ARM_VERSION >= 6))
 109 
 110 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
 111 #define OPAL_HAVE_ATOMIC_MATH_32 1
 112 static inline bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
 113 {
 114   int32_t prev, tmp;
 115   bool ret;
 116 
 117    __asm__ __volatile__ (
 118                          "1:  ldrex   %0, [%2]        \n"
 119                          "    cmp     %0, %3          \n"
 120                          "    bne     2f              \n"
 121                          "    strex   %1, %4, [%2]    \n"
 122                          "    cmp     %1, #0          \n"
 123                          "    bne     1b              \n"
 124                          "2:                          \n"
 125 
 126                          : "=&r" (prev), "=&r" (tmp)
 127                          : "r" (addr), "r" (*oldval), "r" (newval)
 128                          : "cc", "memory");
 129 
 130    ret = (prev == *oldval);
 131    *oldval = prev;
 132    return ret;
 133 }
 134 
 135 /* these two functions aren't inlined in the non-gcc case because then
 136    there would be two function calls (since neither cmpset_32 nor
 137    atomic_?mb can be inlined).  Instead, we "inline" them by hand in
 138    the assembly, meaning there is one function call overhead instead
 139    of two */
 140 static inline bool opal_atomic_compare_exchange_strong_acq_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
 141 {
 142     bool rc;
 143 
 144     rc = opal_atomic_compare_exchange_strong_32 (addr, oldval, newval);
 145     opal_atomic_rmb();
 146 
 147     return rc;
 148 }
 149 
 150 
 151 static inline bool opal_atomic_compare_exchange_strong_rel_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
 152 {
 153     opal_atomic_wmb();
 154     return opal_atomic_compare_exchange_strong_32 (addr, oldval, newval);
 155 }
 156 
 157 #if (OPAL_ASM_SUPPORT_64BIT == 1)
 158 
 159 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1
 160 static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
 161 {
 162     int64_t prev;
 163     int tmp;
 164     bool ret;
 165 
 166     __asm__ __volatile__ (
 167                           "1:  ldrexd  %0, %H0, [%2]           \n"
 168                           "    cmp     %0, %3                  \n"
 169                           "    it      eq                      \n"
 170                           "    cmpeq   %H0, %H3                \n"
 171                           "    bne     2f                      \n"
 172                           "    strexd  %1, %4, %H4, [%2]       \n"
 173                           "    cmp     %1, #0                  \n"
 174                           "    bne     1b                      \n"
 175                           "2:                                    \n"
 176 
 177                           : "=&r" (prev), "=&r" (tmp)
 178                           : "r" (addr), "r" (*oldval), "r" (newval)
 179                           : "cc", "memory");
 180 
 181     ret = (prev == *oldval);
 182     *oldval = prev;
 183     return ret;
 184 }
 185 
 186 /* these two functions aren't inlined in the non-gcc case because then
 187    there would be two function calls (since neither cmpset_64 nor
 188    atomic_?mb can be inlined).  Instead, we "inline" them by hand in
 189    the assembly, meaning there is one function call overhead instead
 190    of two */
 191 static inline bool opal_atomic_compare_exchange_strong_acq_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
 192 {
 193     bool rc;
 194 
 195     rc = opal_atomic_compare_exchange_strong_64 (addr, oldval, newval);
 196     opal_atomic_rmb();
 197 
 198     return rc;
 199 }
 200 
 201 
 202 static inline bool opal_atomic_compare_exchange_strong_rel_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
 203 {
 204     opal_atomic_wmb();
 205     return opal_atomic_compare_exchange_strong_64 (addr, oldval, newval);
 206 }
 207 
 208 #endif
 209 
 210 
 211 #define OPAL_HAVE_ATOMIC_ADD_32 1
 212 static inline int32_t opal_atomic_fetch_add_32(opal_atomic_int32_t* v, int inc)
 213 {
 214     int32_t t, old;
 215     int tmp;
 216 
 217     __asm__ __volatile__(
 218                          "1:  ldrex   %1, [%3]        \n"
 219                          "    add     %0, %1, %4      \n"
 220                          "    strex   %2, %0, [%3]    \n"
 221                          "    cmp     %2, #0          \n"
 222                          "    bne     1b              \n"
 223 
 224                          : "=&r" (t), "=&r" (old), "=&r" (tmp)
 225                          : "r" (v), "r" (inc)
 226                          : "cc", "memory");
 227 
 228 
 229     return old;
 230 }
 231 
 232 #define OPAL_HAVE_ATOMIC_SUB_32 1
 233 static inline int32_t opal_atomic_fetch_sub_32(opal_atomic_int32_t* v, int dec)
 234 {
 235     int32_t t, old;
 236     int tmp;
 237 
 238     __asm__ __volatile__(
 239                          "1:  ldrex   %1, [%3]        \n"
 240                          "    sub     %0, %1, %4      \n"
 241                          "    strex   %2, %0, [%3]    \n"
 242                          "    cmp     %2, #0          \n"
 243                          "    bne     1b              \n"
 244 
 245                          : "=&r" (t), "=&r" (old), "=&r" (tmp)
 246                          : "r" (v), "r" (dec)
 247                          : "cc", "memory");
 248 
 249     return t;
 250 }
 251 
 252 #endif
 253 
 254 #endif /* ! OPAL_SYS_ARCH_ATOMIC_H */