root/opal/include/opal/sys/powerpc/atomic.h

INCLUDED FROM

DEFINITIONS

1. opal_atomic_mb
2. opal_atomic_rmb
3. opal_atomic_wmb
4. opal_atomic_isync
5. opal_atomic_compare_exchange_strong_32
6. opal_atomic_compare_exchange_strong_acq_32
7. opal_atomic_compare_exchange_strong_rel_32
8. opal_atomic_swap_32
9. OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64
10. opal_atomic_swap_64
11. opal_atomic_compare_exchange_strong_64
12. opal_atomic_compare_exchange_strong_acq_64
13. opal_atomic_compare_exchange_strong_rel_64

   1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
   2 /*
   3  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
   4  *                         University Research and Technology
   5  *                         Corporation.  All rights reserved.
   6  * Copyright (c) 2004-2005 The University of Tennessee and The University
   7  *                         of Tennessee Research Foundation.  All rights
   8  *                         reserved.
   9  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
  10  *                         University of Stuttgart.  All rights reserved.
  11  * Copyright (c) 2004-2005 The Regents of the University of California.
  12  *                         All rights reserved.
  13  * Copyright (c) 2010-2017 IBM Corporation.  All rights reserved.
  14  * Copyright (c) 2015-2018 Los Alamos National Security, LLC. All rights
  15  *                         reserved.
  16  * $COPYRIGHT$
  17  *
  18  * Additional copyrights may follow
  19  *
  20  * $HEADER$
  21  */
  22 
  23 #ifndef OPAL_SYS_ARCH_ATOMIC_H
  24 #define OPAL_SYS_ARCH_ATOMIC_H 1
  25 
  26 /*
  27  * On powerpc ...
  28  */
  29 
  30 #define MB()  __asm__ __volatile__ ("sync" : : : "memory")
  31 #define RMB() __asm__ __volatile__ ("lwsync" : : : "memory")
  32 #define WMB() __asm__ __volatile__ ("lwsync" : : : "memory")
  33 #define ISYNC() __asm__ __volatile__ ("isync" : : : "memory")
  34 
  35 
  36 /**********************************************************************
  37  *
  38  * Define constants for PowerPC 32
  39  *
  40  *********************************************************************/
  41 #define OPAL_HAVE_ATOMIC_MEM_BARRIER 1
  42 
  43 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
  44 #define OPAL_HAVE_ATOMIC_SWAP_32 1
  45 #define OPAL_HAVE_ATOMIC_LLSC_32 1
  46 
  47 #define OPAL_HAVE_ATOMIC_MATH_32 1
  48 #define OPAL_HAVE_ATOMIC_ADD_32 1
  49 #define OPAL_HAVE_ATOMIC_AND_32 1
  50 #define OPAL_HAVE_ATOMIC_OR_32 1
  51 #define OPAL_HAVE_ATOMIC_XOR_32 1
  52 #define OPAL_HAVE_ATOMIC_SUB_32 1
  53 
  54 
  55 #if (OPAL_ASSEMBLY_ARCH == OPAL_POWERPC64) || OPAL_ASM_SUPPORT_64BIT
  56 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1
  57 #define OPAL_HAVE_ATOMIC_SWAP_64 1
  58 #define OPAL_HAVE_ATOMIC_LLSC_64 1
  59 #define OPAL_HAVE_ATOMIC_MATH_64 1
  60 #define OPAL_HAVE_ATOMIC_ADD_64 1
  61 #define OPAL_HAVE_ATOMIC_AND_64 1
  62 #define OPAL_HAVE_ATOMIC_OR_64 1
  63 #define OPAL_HAVE_ATOMIC_XOR_64 1
  64 #define OPAL_HAVE_ATOMIC_SUB_64 1
  65 #endif
  66 
  67 
  68 /**********************************************************************
  69  *
  70  * Memory Barriers
  71  *
  72  *********************************************************************/
  73 #if OPAL_GCC_INLINE_ASSEMBLY
  74 
  75 static inline
  76 void opal_atomic_mb(void)
  77 {
  78     MB();
  79 }
  80 
  81 
  82 static inline
  83 void opal_atomic_rmb(void)
  84 {
  85     RMB();
  86 }
  87 
  88 
  89 static inline
  90 void opal_atomic_wmb(void)
  91 {
  92     WMB();
  93 }
  94 
  95 static inline
  96 void opal_atomic_isync(void)
  97 {
  98     ISYNC();
  99 }
 100 
 101 #endif /* end OPAL_GCC_INLINE_ASSEMBLY */
 102 
 103 /**********************************************************************
 104  *
 105  * Atomic math operations
 106  *
 107  *********************************************************************/
 108 #if OPAL_GCC_INLINE_ASSEMBLY
 109 
 110 #ifdef __xlC__
 111 /* work-around bizzare xlc bug in which it sign-extends
 112    a pointer to a 32-bit signed integer */
 113 #define OPAL_ASM_ADDR(a) ((uintptr_t)a)
 114 #else
 115 #define OPAL_ASM_ADDR(a) (a)
 116 #endif
 117 
 118 #if defined(__PGI)
 119 /* work-around for bug in PGI 16.5-16.7 where the compiler fails to
 120  * correctly emit load instructions for 64-bit operands. without this
 121  * it will emit lwz instead of ld to load the 64-bit operand. */
 122 #define OPAL_ASM_VALUE64(x) (void *)(intptr_t) (x)
 123 #else
 124 #define OPAL_ASM_VALUE64(x) x
 125 #endif
 126 
 127 static inline bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
 128 {
 129     int32_t prev;
 130     bool ret;
 131 
 132     __asm__ __volatile__ (
 133                           "1: lwarx   %0, 0, %2  \n\t"
 134                           "   cmpw    0, %0, %3  \n\t"
 135                           "   bne-    2f         \n\t"
 136                           "   stwcx.  %4, 0, %2  \n\t"
 137                           "   bne-    1b         \n\t"
 138                           "2:"
 139                           : "=&r" (prev), "=m" (*addr)
 140                           : "r" OPAL_ASM_ADDR(addr), "r" (*oldval), "r" (newval), "m" (*addr)
 141                           : "cc", "memory");
 142 
 143     ret = (prev == *oldval);
 144     *oldval = prev;
 145     return ret;
 146 }
 147 
 148 /* NTH: the LL/SC support is done through macros due to issues with non-optimized builds. The reason
 149  * is that even with an always_inline attribute the compiler may still emit instructions to store then
 150  * load the arguments to/from the stack. This sequence may cause the ll reservation to be cancelled. */
 151 #define opal_atomic_ll_32(addr, ret)                                    \
 152     do {                                                                \
 153         opal_atomic_int32_t *_addr = (addr);                               \
 154         int32_t _ret;                                                   \
 155         __asm__ __volatile__ ("lwarx   %0, 0, %1  \n\t"                 \
 156                               : "=&r" (_ret)                            \
 157                               : "r" (_addr)                             \
 158                               );                                        \
 159         ret = (typeof(ret)) _ret;                                       \
 160     } while (0)
 161 
 162 #define opal_atomic_sc_32(addr, value, ret)                             \
 163     do {                                                                \
 164         opal_atomic_int32_t *_addr = (addr);                               \
 165         int32_t _ret, _foo, _newval = (int32_t) value;                  \
 166                                                                         \
 167         __asm__ __volatile__ ("   stwcx.  %4, 0, %3  \n\t"              \
 168                               "   li      %0,0       \n\t"              \
 169                               "   bne-    1f         \n\t"              \
 170                               "   ori     %0,%0,1    \n\t"              \
 171                               "1:"                                      \
 172                               : "=r" (_ret), "=m" (*_addr), "=r" (_foo) \
 173                               : "r" (_addr), "r" (_newval)              \
 174                               : "cc", "memory");                        \
 175         ret = _ret;                                                     \
 176     } while (0)
 177 
 178 /* these two functions aren't inlined in the non-gcc case because then
 179    there would be two function calls (since neither cmpset_32 nor
 180    atomic_?mb can be inlined).  Instead, we "inline" them by hand in
 181    the assembly, meaning there is one function call overhead instead
 182    of two */
 183 static inline bool opal_atomic_compare_exchange_strong_acq_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
 184 {
 185     bool rc;
 186 
 187     rc = opal_atomic_compare_exchange_strong_32 (addr, oldval, newval);
 188     opal_atomic_rmb();
 189 
 190     return rc;
 191 }
 192 
 193 
 194 static inline bool opal_atomic_compare_exchange_strong_rel_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
 195 {
 196     opal_atomic_wmb();
 197     return opal_atomic_compare_exchange_strong_32 (addr, oldval, newval);
 198 }
 199 
 200 static inline int32_t opal_atomic_swap_32(opal_atomic_int32_t *addr, int32_t newval)
 201 {
 202     int32_t ret;
 203 
 204     __asm__ __volatile__ ("1: lwarx   %0, 0, %2  \n\t"
 205                           "   stwcx.  %3, 0, %2  \n\t"
 206                           "   bne-    1b         \n\t"
 207                           : "=&r" (ret), "=m" (*addr)
 208                           : "r" (addr), "r" (newval)
 209                           : "cc", "memory");
 210 
 211    return ret;
 212 }
 213 
 214 #endif /* OPAL_GCC_INLINE_ASSEMBLY */
 215 
 216 
 217 #if (OPAL_ASSEMBLY_ARCH == OPAL_POWERPC64)
 218 
 219 #if  OPAL_GCC_INLINE_ASSEMBLY
 220 
 221 #define OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(type, instr)               \
 222 static inline int64_t opal_atomic_fetch_ ## type ## _64(opal_atomic_int64_t* v, int64_t val) \
 223 {                                                                       \
 224     int64_t t, old;                                                     \
 225                                                                         \
 226     __asm__ __volatile__(                                               \
 227                          "1:   ldarx   %1, 0, %4    \n\t"               \
 228                          "     " #instr "     %0, %3, %1   \n\t"        \
 229                          "     stdcx.  %0, 0, %4    \n\t"               \
 230                          "     bne-    1b           \n\t"               \
 231                          : "=&r" (t), "=&r" (old), "=m" (*v)            \
 232                          : "r" (OPAL_ASM_VALUE64(val)), "r" OPAL_ASM_ADDR(v), "m" (*v) \
 233                          : "cc");                                       \
 234                                                                         \
 235     return old;                                                         \
 236 }
 237 
 238 OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(add, add)
 239 OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(and, and)
 240 OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(or, or)
 241 OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(xor, xor)
 242 OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(sub, subf)
 243 
 244 static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
 245 {
 246     int64_t prev;
 247     bool ret;
 248 
 249     __asm__ __volatile__ (
 250                           "1: ldarx   %0, 0, %2  \n\t"
 251                           "   cmpd    0, %0, %3  \n\t"
 252                           "   bne-    2f         \n\t"
 253                           "   stdcx.  %4, 0, %2  \n\t"
 254                           "   bne-    1b         \n\t"
 255                           "2:"
 256                           : "=&r" (prev), "=m" (*addr)
 257                           : "r" (addr), "r" (OPAL_ASM_VALUE64(*oldval)), "r" (OPAL_ASM_VALUE64(newval)), "m" (*addr)
 258                           : "cc", "memory");
 259 
 260     ret = (prev == *oldval);
 261     *oldval = prev;
 262     return ret;
 263 }
 264 
 265 #define opal_atomic_ll_64(addr, ret)                                    \
 266     do {                                                                \
 267         opal_atomic_int64_t *_addr = (addr);                               \
 268         int64_t _ret;                                                   \
 269         __asm__ __volatile__ ("ldarx   %0, 0, %1  \n\t"                 \
 270                               : "=&r" (_ret)                            \
 271                               : "r" (_addr)                             \
 272                               );                                        \
 273         ret = (typeof(ret)) _ret;                                       \
 274     } while (0)
 275 
 276 #define opal_atomic_sc_64(addr, value, ret)                             \
 277     do {                                                                \
 278         opal_atomic_int64_t *_addr = (addr);                               \
 279         int64_t _newval = (int64_t) value;                              \
 280         int32_t _ret;                                                   \
 281                                                                         \
 282         __asm__ __volatile__ ("   stdcx.  %2, 0, %1  \n\t"              \
 283                               "   li      %0,0       \n\t"              \
 284                               "   bne-    1f         \n\t"              \
 285                               "   ori     %0,%0,1    \n\t"              \
 286                               "1:"                                      \
 287                               : "=r" (_ret)                             \
 288                               : "r" (_addr), "r" (OPAL_ASM_VALUE64(_newval)) \
 289                               : "cc", "memory");                        \
 290         ret = _ret;                                                     \
 291     } while (0)
 292 
 293 static inline int64_t opal_atomic_swap_64(opal_atomic_int64_t *addr, int64_t newval)
 294 {
 295    int64_t ret;
 296 
 297    __asm__ __volatile__ ("1: ldarx   %0, 0, %2  \n\t"
 298                          "   stdcx.  %3, 0, %2  \n\t"
 299                          "   bne-    1b         \n\t"
 300                          : "=&r" (ret), "=m" (*addr)
 301                          : "r" (addr), "r" (OPAL_ASM_VALUE64(newval))
 302                          : "cc", "memory");
 303 
 304    return ret;
 305 }
 306 
 307 #endif /* OPAL_GCC_INLINE_ASSEMBLY */
 308 
 309 #elif (OPAL_ASSEMBLY_ARCH == OPAL_POWERPC32) && OPAL_ASM_SUPPORT_64BIT
 310 
 311 #ifndef ll_low /* GLIBC provides these somewhere, so protect */
 312 #define ll_low(x)       *(((unsigned int*)&(x))+0)
 313 #define ll_high(x)      *(((unsigned int*)&(x))+1)
 314 #endif
 315 
 316 #if  OPAL_GCC_INLINE_ASSEMBLY
 317 
 318 static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
 319 {
 320     int64_t prev;
 321     int ret;
 322 
 323     /*
 324      * We force oldval and newval into memory because PPC doesn't
 325      * appear to have a way to do a move register with offset.  Since
 326      * this is 32-bit code, a 64 bit integer will be loaded into two
 327      * registers (assuming no inlining, addr will be in r3, oldval
 328      * will be in r4 and r5, and newval will be r6 and r7.  We need
 329      * to load the whole thing into one register.  So we have the
 330      * compiler push the values into memory and load the double word
 331      * into registers.  We use r4,r5 so that the main block of code
 332      * is very similar to the pure 64 bit version.
 333      */
 334    __asm__ __volatile__ (
 335                          "ld r4,%3         \n\t"
 336                          "ld r5,%4        \n\t"
 337                          "1: ldarx   %1, 0, %2  \n\t"
 338                          "   cmpd    0, %1, r4  \n\t"
 339                          "   bne-    2f         \n\t"
 340                          "   stdcx.  r5, 0, %2  \n\t"
 341                          "   bne-    1b         \n\t"
 342                          "2:                    \n\t"
 343                          "xor r5,r4,%1          \n\t"
 344                          "subfic r9,r5,0        \n\t"
 345                          "adde %0,r9,r5         \n\t"
 346                          : "=&r" (ret), "+r" (prev)
 347                          : "r"OPAL_ASM_ADDR(addr),
 348                            "m"(*oldval), "m"(newval)
 349                          : "r4", "r5", "r9", "cc", "memory");
 350    *oldval = prev;
 351    return (bool) ret;
 352 }
 353 
 354 #endif /* OPAL_GCC_INLINE_ASSEMBLY */
 355 
 356 #endif /* OPAL_ASM_SUPPORT_64BIT */
 357 
 358 #if OPAL_GCC_INLINE_ASSEMBLY
 359 
 360 /* these two functions aren't inlined in the non-gcc case because then
 361    there would be two function calls (since neither cmpset_64 nor
 362    atomic_?mb can be inlined).  Instead, we "inline" them by hand in
 363    the assembly, meaning there is one function call overhead instead
 364    of two */
 365 static inline bool opal_atomic_compare_exchange_strong_acq_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
 366 {
 367     bool rc;
 368 
 369     rc = opal_atomic_compare_exchange_strong_64 (addr, oldval, newval);
 370     opal_atomic_rmb();
 371 
 372     return rc;
 373 }
 374 
 375 
 376 static inline bool opal_atomic_compare_exchange_strong_rel_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
 377 {
 378     opal_atomic_wmb();
 379     return opal_atomic_compare_exchange_strong_64 (addr, oldval, newval);
 380 }
 381 
 382 
 383 #define OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_32(type, instr)               \
 384 static inline int32_t opal_atomic_fetch_ ## type ## _32(opal_atomic_int32_t* v, int val) \
 385 {                                                                       \
 386     int32_t t, old;                                                     \
 387                                                                         \
 388     __asm__ __volatile__(                                               \
 389                          "1:   lwarx   %1, 0, %4    \n\t"               \
 390                          "     " #instr "     %0, %3, %1   \n\t"        \
 391                          "     stwcx.  %0, 0, %4    \n\t"               \
 392                          "     bne-    1b           \n\t"               \
 393                          : "=&r" (t), "=&r" (old), "=m" (*v)            \
 394                          : "r" (val), "r" OPAL_ASM_ADDR(v), "m" (*v)    \
 395                          : "cc");                                       \
 396                                                                         \
 397     return old;                                                         \
 398 }
 399 
 400 OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_32(add, add)
 401 OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_32(and, and)
 402 OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_32(or, or)
 403 OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_32(xor, xor)
 404 OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_32(sub, subf)
 405 
 406 #endif /* OPAL_GCC_INLINE_ASSEMBLY */
 407 
 408 #endif /* ! OPAL_SYS_ARCH_ATOMIC_H */