root/opal/include/opal/sys/atomic.h

/* [<][>][^][v][top][bottom][index][help] */

INCLUDED FROM

DEFINITIONS

This source file includes following definitions.
  1. opal_atomic_add_fetch_size_t
  2. opal_atomic_fetch_add_size_t
  3. opal_atomic_sub_fetch_size_t
  4. opal_atomic_fetch_sub_size_t
   1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
   2 /*
   3  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
   4  *                         University Research and Technology
   5  *                         Corporation.  All rights reserved.
   6  * Copyright (c) 2004-2006 The University of Tennessee and The University
   7  *                         of Tennessee Research Foundation.  All rights
   8  *                         reserved.
   9  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
  10  *                         University of Stuttgart.  All rights reserved.
  11  * Copyright (c) 2004-2005 The Regents of the University of California.
  12  *                         All rights reserved.
  13  * Copyright (c) 2007      Sun Microsystems, Inc.  All rights reserved.
  14  * Copyright (c) 2011      Sandia National Laboratories. All rights reserved.
  15  * Copyright (c) 2011-2017 Los Alamos National Security, LLC. All rights
  16  *                         reserved.
  17  * Copyright (c) 2017      Research Organization for Information Science
  18  *                         and Technology (RIST). All rights reserved.
  19  * $COPYRIGHT$
  20  *
  21  * Additional copyrights may follow
  22  *
  23  * $HEADER$
  24  */
  25 
  26 /** @file
  27  *
  28  * Atomic operations.
  29  *
  30  * This API is patterned after the FreeBSD kernel atomic interface
  31  * (which is influenced by Intel's ia64 architecture).  The
  32  * FreeBSD interface is documented at
  33  *
  34  * http://www.freebsd.org/cgi/man.cgi?query=atomic&sektion=9
  35  *
  36  * Only the necessary subset of functions are implemented here.
  37  *
  38  * The following #defines will be true / false based on
  39  * assembly support:
  40  *
  41  *  - \c OPAL_HAVE_ATOMIC_MEM_BARRIER atomic memory barriers
  42  *  - \c OPAL_HAVE_ATOMIC_SPINLOCKS atomic spinlocks
  43  *  - \c OPAL_HAVE_ATOMIC_MATH_32 if 32 bit add/sub/compare-exchange can be done "atomicly"
  44  *  - \c OPAL_HAVE_ATOMIC_MATH_64 if 64 bit add/sub/compare-exchange can be done "atomicly"
  45  *
  46  * Note that for the Atomic math, atomic add/sub may be implemented as
  47  * C code using opal_atomic_compare_exchange.  The appearance of atomic
  48  * operation will be upheld in these cases.
  49  */
  50 
  51 #ifndef OPAL_SYS_ATOMIC_H
  52 #define OPAL_SYS_ATOMIC_H 1
  53 
  54 #include "opal_config.h"
  55 
  56 #include <stdbool.h>
  57 
  58 #include "opal/sys/architecture.h"
  59 #include "opal_stdatomic.h"
  60 
  61 #if OPAL_ASSEMBLY_BUILTIN == OPAL_BUILTIN_C11
  62 
  63 #include "atomic_stdc.h"
  64 
  65 #else /* !OPAL_C_HAVE__ATOMIC */
  66 
  67 /* do some quick #define cleanup in cases where we are doing
  68    testing... */
  69 #ifdef OPAL_DISABLE_INLINE_ASM
  70 #undef OPAL_C_GCC_INLINE_ASSEMBLY
  71 #define OPAL_C_GCC_INLINE_ASSEMBLY 0
  72 #endif
  73 
  74 /* define OPAL_{GCC,DEC,XLC}_INLINE_ASSEMBLY based on the
  75    OPAL_C_{GCC,DEC,XLC}_INLINE_ASSEMBLY defines and whether we
  76    are in C or C++ */
  77 #if defined(c_plusplus) || defined(__cplusplus)
  78 /* We no longer support inline assembly for C++ as OPAL is a C-only interface */
  79 #define OPAL_GCC_INLINE_ASSEMBLY 0
  80 #else
  81 #define OPAL_GCC_INLINE_ASSEMBLY OPAL_C_GCC_INLINE_ASSEMBLY
  82 #endif
  83 
  84 
  85 BEGIN_C_DECLS
  86 /**********************************************************************
  87  *
  88  * Data structures for atomic ops
  89  *
  90  *********************************************************************/
  91 /**
  92  * Volatile lock object (with optional padding).
  93  *
  94  * \note The internals of the lock are included here, but should be
  95  * considered private.  The implementation currently in use may choose
  96  * to use an int or unsigned char as the lock value - the user is not
  97  * informed either way.
  98  */
  99 struct opal_atomic_lock_t {
 100     union {
 101         opal_atomic_int32_t lock;     /**< The lock address (an integer) */
 102         volatile unsigned char sparc_lock; /**< The lock address on sparc */
 103         char padding[sizeof(int)]; /**< Array for optional padding */
 104     } u;
 105 };
 106 typedef struct opal_atomic_lock_t opal_atomic_lock_t;
 107 
 108 /**********************************************************************
 109  *
 110  * Set or unset these macros in the architecture-specific atomic.h
 111  * files if we need to specify them as inline or non-inline
 112  *
 113  *********************************************************************/
 114 #if !OPAL_GCC_INLINE_ASSEMBLY
 115 #define OPAL_HAVE_INLINE_ATOMIC_MEM_BARRIER 0
 116 #define OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32 0
 117 #define OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64 0
 118 #define OPAL_HAVE_INLINE_ATOMIC_ADD_32 0
 119 #define OPAL_HAVE_INLINE_ATOMIC_AND_32 0
 120 #define OPAL_HAVE_INLINE_ATOMIC_OR_32 0
 121 #define OPAL_HAVE_INLINE_ATOMIC_XOR_32 0
 122 #define OPAL_HAVE_INLINE_ATOMIC_SUB_32 0
 123 #define OPAL_HAVE_INLINE_ATOMIC_ADD_64 0
 124 #define OPAL_HAVE_INLINE_ATOMIC_AND_64 0
 125 #define OPAL_HAVE_INLINE_ATOMIC_OR_64 0
 126 #define OPAL_HAVE_INLINE_ATOMIC_XOR_64 0
 127 #define OPAL_HAVE_INLINE_ATOMIC_SUB_64 0
 128 #define OPAL_HAVE_INLINE_ATOMIC_SWAP_32 0
 129 #define OPAL_HAVE_INLINE_ATOMIC_SWAP_64 0
 130 #else
 131 #define OPAL_HAVE_INLINE_ATOMIC_MEM_BARRIER 1
 132 #define OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32 1
 133 #define OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64 1
 134 #define OPAL_HAVE_INLINE_ATOMIC_ADD_32 1
 135 #define OPAL_HAVE_INLINE_ATOMIC_AND_32 1
 136 #define OPAL_HAVE_INLINE_ATOMIC_OR_32 1
 137 #define OPAL_HAVE_INLINE_ATOMIC_XOR_32 1
 138 #define OPAL_HAVE_INLINE_ATOMIC_SUB_32 1
 139 #define OPAL_HAVE_INLINE_ATOMIC_ADD_64 1
 140 #define OPAL_HAVE_INLINE_ATOMIC_AND_64 1
 141 #define OPAL_HAVE_INLINE_ATOMIC_OR_64 1
 142 #define OPAL_HAVE_INLINE_ATOMIC_XOR_64 1
 143 #define OPAL_HAVE_INLINE_ATOMIC_SUB_64 1
 144 #define OPAL_HAVE_INLINE_ATOMIC_SWAP_32 1
 145 #define OPAL_HAVE_INLINE_ATOMIC_SWAP_64 1
 146 #endif
 147 
 148 /**
 149  * Enumeration of lock states
 150  */
 151 enum {
 152     OPAL_ATOMIC_LOCK_UNLOCKED = 0,
 153     OPAL_ATOMIC_LOCK_LOCKED = 1
 154 };
 155 
 156 #define OPAL_ATOMIC_LOCK_INIT {.u = {.lock = OPAL_ATOMIC_LOCK_UNLOCKED}}
 157 
 158 /**********************************************************************
 159  *
 160  * Load the appropriate architecture files and set some reasonable
 161  * default values for our support
 162  *
 163  *********************************************************************/
 164 #if defined(DOXYGEN)
 165 /* don't include system-level gorp when generating doxygen files */
 166 #elif OPAL_ASSEMBLY_BUILTIN == OPAL_BUILTIN_SYNC
 167 #include "opal/sys/sync_builtin/atomic.h"
 168 #elif OPAL_ASSEMBLY_BUILTIN == OPAL_BUILTIN_GCC
 169 #include "opal/sys/gcc_builtin/atomic.h"
 170 #elif OPAL_ASSEMBLY_ARCH == OPAL_X86_64
 171 #include "opal/sys/x86_64/atomic.h"
 172 #elif OPAL_ASSEMBLY_ARCH == OPAL_ARM
 173 #include "opal/sys/arm/atomic.h"
 174 #elif OPAL_ASSEMBLY_ARCH == OPAL_ARM64
 175 #include "opal/sys/arm64/atomic.h"
 176 #elif OPAL_ASSEMBLY_ARCH == OPAL_IA32
 177 #include "opal/sys/ia32/atomic.h"
 178 #elif OPAL_ASSEMBLY_ARCH == OPAL_IA64
 179 #include "opal/sys/ia64/atomic.h"
 180 #elif OPAL_ASSEMBLY_ARCH == OPAL_MIPS
 181 #include "opal/sys/mips/atomic.h"
 182 #elif OPAL_ASSEMBLY_ARCH == OPAL_POWERPC32
 183 #include "opal/sys/powerpc/atomic.h"
 184 #elif OPAL_ASSEMBLY_ARCH == OPAL_POWERPC64
 185 #include "opal/sys/powerpc/atomic.h"
 186 #elif OPAL_ASSEMBLY_ARCH == OPAL_SPARC
 187 #include "opal/sys/sparc/atomic.h"
 188 #elif OPAL_ASSEMBLY_ARCH == OPAL_SPARCV9_32
 189 #include "opal/sys/sparcv9/atomic.h"
 190 #elif OPAL_ASSEMBLY_ARCH == OPAL_SPARCV9_64
 191 #include "opal/sys/sparcv9/atomic.h"
 192 #endif
 193 
 194 #ifndef DOXYGEN
 195 /* compare and set operations can't really be emulated from software,
 196    so if these defines aren't already set, they should be set to 0
 197    now */
 198 #ifndef OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32
 199 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 0
 200 #endif
 201 #ifndef OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
 202 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 0
 203 #endif
 204 #ifndef OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128
 205 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 0
 206 #endif
 207 #ifndef OPAL_HAVE_ATOMIC_LLSC_32
 208 #define OPAL_HAVE_ATOMIC_LLSC_32 0
 209 #endif
 210 #ifndef OPAL_HAVE_ATOMIC_LLSC_64
 211 #define OPAL_HAVE_ATOMIC_LLSC_64 0
 212 #endif
 213 #endif /* DOXYGEN */
 214 
 215 /**********************************************************************
 216  *
 217  * Memory Barriers - defined here if running doxygen or have barriers
 218  *                   but can't inline
 219  *
 220  *********************************************************************/
 221 #if !defined(OPAL_HAVE_ATOMIC_MEM_BARRIER) && !defined(DOXYGEN)
 222 /* no way to emulate in C code */
 223 #define OPAL_HAVE_ATOMIC_MEM_BARRIER 0
 224 #endif
 225 
 226 #if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_MEM_BARRIER
 227 /**
 228  * Memory barrier
 229  *
 230  * Will use system-specific features to instruct the processor and
 231  * memory controller that all writes and reads that have been posted
 232  * before the call to \c opal_atomic_mb() must appear to have
 233  * completed before the next read or write.
 234  *
 235  * \note This can have some expensive side effects, including flushing
 236  * the pipeline, preventing the cpu from reordering instructions, and
 237  * generally grinding the memory controller's performance.  Use only
 238  * if you need *both* read and write barriers.
 239  */
 240 
 241 #if OPAL_HAVE_INLINE_ATOMIC_MEM_BARRIER
 242 static inline
 243 #endif
 244 void opal_atomic_mb(void);
 245 
 246 /**
 247  * Read memory barrier
 248  *
 249  * Use system-specific features to instruct the processor and memory
 250  * conrtoller that all reads that have been posted before the call to
 251  * \c opal_atomic_rmb() must appear to have been completed before the
 252  * next read.  Nothing is said about the ordering of writes when using
 253  * \c opal_atomic_rmb().
 254  */
 255 
 256 #if OPAL_HAVE_INLINE_ATOMIC_MEM_BARRIER
 257 static inline
 258 #endif
 259 void opal_atomic_rmb(void);
 260 
 261 /**
 262  * Write memory barrier.
 263  *
 264  * Use system-specific features to instruct the processor and memory
 265  * conrtoller that all writes that have been posted before the call to
 266  * \c opal_atomic_wmb() must appear to have been completed before the
 267  * next write.  Nothing is said about the ordering of reads when using
 268  * \c opal_atomic_wmb().
 269  */
 270 
 271 #if OPAL_HAVE_INLINE_ATOMIC_MEM_BARRIER
 272 static inline
 273 #endif
 274 void opal_atomic_wmb(void);
 275 
 276 #endif /* defined(DOXYGEN) || OPAL_HAVE_ATOMIC_MEM_BARRIER */
 277 
 278 
 279 /**********************************************************************
 280  *
 281  * Atomic spinlocks - always inlined, if have atomic compare-and-swap
 282  *
 283  *********************************************************************/
 284 
 285 #if !defined(OPAL_HAVE_ATOMIC_SPINLOCKS) && !defined(DOXYGEN)
 286 /* 0 is more like "pending" - we'll fix up at the end after all
 287    the static inline functions are declared */
 288 #define OPAL_HAVE_ATOMIC_SPINLOCKS 0
 289 #endif
 290 
 291 #if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_SPINLOCKS || (OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64)
 292 
 293 /**
 294  * Initialize a lock to value
 295  *
 296  * @param lock         Address of the lock
 297  * @param value        Initial value to set lock to
 298  */
 299 #if OPAL_HAVE_ATOMIC_SPINLOCKS == 0
 300 static inline
 301 #endif
 302 void opal_atomic_lock_init(opal_atomic_lock_t* lock, int32_t value);
 303 
 304 
 305 /**
 306  * Try to acquire a lock.
 307  *
 308  * @param lock          Address of the lock.
 309  * @return              0 if the lock was acquired, 1 otherwise.
 310  */
 311 #if OPAL_HAVE_ATOMIC_SPINLOCKS == 0
 312 static inline
 313 #endif
 314 int opal_atomic_trylock(opal_atomic_lock_t *lock);
 315 
 316 
 317 /**
 318  * Acquire a lock by spinning.
 319  *
 320  * @param lock          Address of the lock.
 321  */
 322 #if OPAL_HAVE_ATOMIC_SPINLOCKS == 0
 323 static inline
 324 #endif
 325 void opal_atomic_lock(opal_atomic_lock_t *lock);
 326 
 327 
 328 /**
 329  * Release a lock.
 330  *
 331  * @param lock          Address of the lock.
 332  */
 333 #if OPAL_HAVE_ATOMIC_SPINLOCKS == 0
 334 static inline
 335 #endif
 336 void opal_atomic_unlock(opal_atomic_lock_t *lock);
 337 
 338 
 339 #if OPAL_HAVE_ATOMIC_SPINLOCKS == 0
 340 #undef OPAL_HAVE_ATOMIC_SPINLOCKS
 341 #define OPAL_HAVE_ATOMIC_SPINLOCKS (OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64)
 342 #define OPAL_NEED_INLINE_ATOMIC_SPINLOCKS 1
 343 #endif
 344 
 345 #endif /* OPAL_HAVE_ATOMIC_SPINLOCKS */
 346 
 347 
 348 /**********************************************************************
 349  *
 350  * Atomic math operations
 351  *
 352  *********************************************************************/
 353 #if !defined(OPAL_HAVE_ATOMIC_CMPSET_32) && !defined(DOXYGEN)
 354 #define OPAL_HAVE_ATOMIC_CMPSET_32 0
 355 #endif
 356 #if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_CMPSET_32
 357 
 358 #if OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32
 359 static inline
 360 #endif
 361 bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval,
 362                                              int32_t newval);
 363 
 364 #if OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32
 365 static inline
 366 #endif
 367 bool opal_atomic_compare_exchange_strong_acq_32 (opal_atomic_int32_t *addr, int32_t *oldval,
 368                                                  int32_t newval);
 369 
 370 #if OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32
 371 static inline
 372 #endif
 373 bool opal_atomic_compare_exchange_strong_rel_32 (opal_atomic_int32_t *addr, int32_t *oldval,
 374                                                  int32_t newval);
 375 #endif
 376 
 377 
 378 #if !defined(OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64) && !defined(DOXYGEN)
 379 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 0
 380 #endif
 381 #if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
 382 
 383 #if OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64
 384 static inline
 385 #endif
 386 bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval,
 387                                              int64_t newval);
 388 
 389 #if OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64
 390 static inline
 391 #endif
 392 bool opal_atomic_compare_exchange_strong_acq_64 (opal_atomic_int64_t *addr, int64_t *oldval,
 393                                                  int64_t newval);
 394 
 395 #if OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64
 396 static inline
 397 #endif
 398 bool opal_atomic_compare_exchange_strong_rel_64 (opal_atomic_int64_t *addr, int64_t *oldval,
 399                                                  int64_t newval);
 400 
 401 #endif
 402 
 403 #if !defined(OPAL_HAVE_ATOMIC_MATH_32) && !defined(DOXYGEN)
 404   /* define to 0 for these tests.  WIll fix up later. */
 405   #define OPAL_HAVE_ATOMIC_MATH_32 0
 406 #endif
 407 
 408 #if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32
 409 
 410 static inline int32_t opal_atomic_add_fetch_32(opal_atomic_int32_t *addr, int delta);
 411 static inline int32_t opal_atomic_fetch_add_32(opal_atomic_int32_t *addr, int delta);
 412 static inline int32_t opal_atomic_and_fetch_32(opal_atomic_int32_t *addr, int32_t value);
 413 static inline int32_t opal_atomic_fetch_and_32(opal_atomic_int32_t *addr, int32_t value);
 414 static inline int32_t opal_atomic_or_fetch_32(opal_atomic_int32_t *addr, int32_t value);
 415 static inline int32_t opal_atomic_fetch_or_32(opal_atomic_int32_t *addr, int32_t value);
 416 static inline int32_t opal_atomic_xor_fetch_32(opal_atomic_int32_t *addr, int32_t value);
 417 static inline int32_t opal_atomic_fetch_xor_32(opal_atomic_int32_t *addr, int32_t value);
 418 static inline int32_t opal_atomic_sub_fetch_32(opal_atomic_int32_t *addr, int delta);
 419 static inline int32_t opal_atomic_fetch_sub_32(opal_atomic_int32_t *addr, int delta);
 420 static inline int32_t opal_atomic_min_fetch_32 (opal_atomic_int32_t *addr, int32_t value);
 421 static inline int32_t opal_atomic_fetch_min_32 (opal_atomic_int32_t *addr, int32_t value);
 422 static inline int32_t opal_atomic_max_fetch_32 (opal_atomic_int32_t *addr, int32_t value);
 423 static inline int32_t opal_atomic_fetch_max_32 (opal_atomic_int32_t *addr, int32_t value);
 424 
 425 #endif /* OPAL_HAVE_ATOMIC_MATH_32 */
 426 
 427 #if ! OPAL_HAVE_ATOMIC_MATH_32
 428 /* fix up the value of opal_have_atomic_math_32 to allow for C versions */
 429 #undef OPAL_HAVE_ATOMIC_MATH_32
 430 #define OPAL_HAVE_ATOMIC_MATH_32 OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32
 431 #endif
 432 
 433 #ifndef OPAL_HAVE_ATOMIC_MATH_64
 434 /* define to 0 for these tests.  WIll fix up later. */
 435 #define OPAL_HAVE_ATOMIC_MATH_64 0
 436 #endif
 437 
 438 #if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_MATH_64 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
 439 
 440 static inline int64_t opal_atomic_add_fetch_64(opal_atomic_int64_t *addr, int64_t delta);
 441 static inline int64_t opal_atomic_fetch_add_64(opal_atomic_int64_t *addr, int64_t delta);
 442 static inline int64_t opal_atomic_and_fetch_64(opal_atomic_int64_t *addr, int64_t value);
 443 static inline int64_t opal_atomic_fetch_and_64(opal_atomic_int64_t *addr, int64_t value);
 444 static inline int64_t opal_atomic_or_fetch_64(opal_atomic_int64_t *addr, int64_t value);
 445 static inline int64_t opal_atomic_fetch_or_64(opal_atomic_int64_t *addr, int64_t value);
 446 static inline int64_t opal_atomic_fetch_xor_64(opal_atomic_int64_t *addr, int64_t value);
 447 static inline int64_t opal_atomic_sub_fetch_64(opal_atomic_int64_t *addr, int64_t delta);
 448 static inline int64_t opal_atomic_fetch_sub_64(opal_atomic_int64_t *addr, int64_t delta);
 449 static inline int64_t opal_atomic_min_fetch_64 (opal_atomic_int64_t *addr, int64_t value);
 450 static inline int64_t opal_atomic_fetch_min_64 (opal_atomic_int64_t *addr, int64_t value);
 451 static inline int64_t opal_atomic_max_fetch_64 (opal_atomic_int64_t *addr, int64_t value);
 452 static inline int64_t opal_atomic_fetch_max_64 (opal_atomic_int64_t *addr, int64_t value);
 453 
 454 #endif /* OPAL_HAVE_ATOMIC_MATH_64 */
 455 
 456 #if ! OPAL_HAVE_ATOMIC_MATH_64
 457 /* fix up the value of opal_have_atomic_math_64 to allow for C versions */
 458 #undef OPAL_HAVE_ATOMIC_MATH_64
 459 #define OPAL_HAVE_ATOMIC_MATH_64 OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
 460 #endif
 461 
 462 /* provide a size_t add/subtract.  When in debug mode, make it an
 463  * inline function so that we don't have any casts in the
 464  *  interface and can catch type errors.  When not in debug mode,
 465  * just make it a macro, so that there's no performance penalty
 466  */
 467 #if defined(DOXYGEN) || OPAL_ENABLE_DEBUG
 468 static inline size_t
 469 opal_atomic_add_fetch_size_t(opal_atomic_size_t *addr, size_t delta)
 470 {
 471 #if SIZEOF_SIZE_T == 4
 472     return (size_t) opal_atomic_add_fetch_32((int32_t*) addr, delta);
 473 #elif SIZEOF_SIZE_T == 8
 474     return (size_t) opal_atomic_add_fetch_64((int64_t*) addr, delta);
 475 #else
 476 #error "Unknown size_t size"
 477 #endif
 478 }
 479 
 480 static inline size_t
 481 opal_atomic_fetch_add_size_t(opal_atomic_size_t *addr, size_t delta)
 482 {
 483 #if SIZEOF_SIZE_T == 4
 484     return (size_t) opal_atomic_fetch_add_32((int32_t*) addr, delta);
 485 #elif SIZEOF_SIZE_T == 8
 486     return (size_t) opal_atomic_fetch_add_64((int64_t*) addr, delta);
 487 #else
 488 #error "Unknown size_t size"
 489 #endif
 490 }
 491 
 492 static inline size_t
 493 opal_atomic_sub_fetch_size_t(opal_atomic_size_t *addr, size_t delta)
 494 {
 495 #if SIZEOF_SIZE_T == 4
 496     return (size_t) opal_atomic_sub_fetch_32((int32_t*) addr, delta);
 497 #elif SIZEOF_SIZE_T == 8
 498     return (size_t) opal_atomic_sub_fetch_64((int64_t*) addr, delta);
 499 #else
 500 #error "Unknown size_t size"
 501 #endif
 502 }
 503 
 504 static inline size_t
 505 opal_atomic_fetch_sub_size_t(opal_atomic_size_t *addr, size_t delta)
 506 {
 507 #if SIZEOF_SIZE_T == 4
 508     return (size_t) opal_atomic_fetch_sub_32((int32_t*) addr, delta);
 509 #elif SIZEOF_SIZE_T == 8
 510     return (size_t) opal_atomic_fetch_sub_64((int64_t*) addr, delta);
 511 #else
 512 #error "Unknown size_t size"
 513 #endif
 514 }
 515 
 516 #else
 517 #if SIZEOF_SIZE_T == 4
 518 #define opal_atomic_add_fetch_size_t(addr, delta) ((size_t) opal_atomic_add_fetch_32((opal_atomic_int32_t *) addr, delta))
 519 #define opal_atomic_fetch_add_size_t(addr, delta) ((size_t) opal_atomic_fetch_add_32((opal_atomic_int32_t *) addr, delta))
 520 #define opal_atomic_sub_fetch_size_t(addr, delta) ((size_t) opal_atomic_sub_fetch_32((opal_atomic_int32_t *) addr, delta))
 521 #define opal_atomic_fetch_sub_size_t(addr, delta) ((size_t) opal_atomic_fetch_sub_32((opal_atomic_int32_t *) addr, delta))
 522 #elif SIZEOF_SIZE_T == 8
 523 #define opal_atomic_add_fetch_size_t(addr, delta) ((size_t) opal_atomic_add_fetch_64((opal_atomic_int64_t *) addr, delta))
 524 #define opal_atomic_fetch_add_size_t(addr, delta) ((size_t) opal_atomic_fetch_add_64((opal_atomic_int64_t *) addr, delta))
 525 #define opal_atomic_sub_fetch_size_t(addr, delta) ((size_t) opal_atomic_sub_fetch_64((opal_atomic_int64_t *) addr, delta))
 526 #define opal_atomic_fetch_sub_size_t(addr, delta) ((size_t) opal_atomic_fetch_sub_64((opal_atomic_int64_t *) addr, delta))
 527 #else
 528 #error "Unknown size_t size"
 529 #endif
 530 #endif
 531 
 532 #if defined(DOXYGEN) || (OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64)
 533 /* these are always done with inline functions, so always mark as
 534    static inline */
 535 
 536 static inline bool opal_atomic_compare_exchange_strong_xx (opal_atomic_intptr_t *addr, intptr_t *oldval,
 537                                                            int64_t newval, size_t length);
 538 static inline bool opal_atomic_compare_exchange_strong_acq_xx (opal_atomic_intptr_t *addr, intptr_t *oldval,
 539                                                                int64_t newval, size_t length);
 540 static inline bool opal_atomic_compare_exchange_strong_rel_xx (opal_atomic_intptr_t *addr, intptr_t *oldval,
 541                                                                int64_t newval, size_t length);
 542 
 543 
 544 static inline bool opal_atomic_compare_exchange_strong_ptr (opal_atomic_intptr_t* addr, intptr_t *oldval,
 545                                                             intptr_t newval);
 546 static inline bool opal_atomic_compare_exchange_strong_acq_ptr (opal_atomic_intptr_t* addr, intptr_t *oldval,
 547                                                                 intptr_t newval);
 548 static inline bool opal_atomic_compare_exchange_strong_rel_ptr (opal_atomic_intptr_t* addr, intptr_t *oldval,
 549                                                                 intptr_t newval);
 550 
 551 /**
 552  * Atomic compare and set of generic type with relaxed semantics. This
 553  * macro detect at compile time the type of the first argument and
 554  * choose the correct function to be called.
 555  *
 556  * \note This macro should only be used for integer types.
 557  *
 558  * @param addr          Address of <TYPE>.
 559  * @param oldval        Comparison value address of <TYPE>.
 560  * @param newval        New value to set if comparision is true <TYPE>.
 561  *
 562  * See opal_atomic_compare_exchange_* for pseudo-code.
 563  */
 564 #define opal_atomic_compare_exchange_strong( ADDR, OLDVAL, NEWVAL )                  \
 565     opal_atomic_compare_exchange_strong_xx( (opal_atomic_intptr_t*)(ADDR), (intptr_t *)(OLDVAL), \
 566                                             (intptr_t)(NEWVAL), sizeof(*(ADDR)) )
 567 
 568 /**
 569  * Atomic compare and set of generic type with acquire semantics. This
 570  * macro detect at compile time the type of the first argument and
 571  * choose the correct function to be called.
 572  *
 573  * \note This macro should only be used for integer types.
 574  *
 575  * @param addr          Address of <TYPE>.
 576  * @param oldval        Comparison value address of <TYPE>.
 577  * @param newval        New value to set if comparision is true <TYPE>.
 578  *
 579  * See opal_atomic_compare_exchange_acq_* for pseudo-code.
 580  */
 581 #define opal_atomic_compare_exchange_strong_acq( ADDR, OLDVAL, NEWVAL )                  \
 582     opal_atomic_compare_exchange_strong_acq_xx( (opal_atomic_intptr_t*)(ADDR), (intptr_t *)(OLDVAL), \
 583                                                 (intptr_t)(NEWVAL), sizeof(*(ADDR)) )
 584 
 585 /**
 586  * Atomic compare and set of generic type with release semantics. This
 587  * macro detect at compile time the type of the first argument and
 588  * choose the correct function to be called.
 589  *
 590  * \note This macro should only be used for integer types.
 591  *
 592  * @param addr          Address of <TYPE>.
 593  * @param oldval        Comparison value address of <TYPE>.
 594  * @param newval        New value to set if comparision is true <TYPE>.
 595  *
 596  * See opal_atomic_compare_exchange_rel_* for pseudo-code.
 597  */
 598 #define opal_atomic_compare_exchange_strong_rel( ADDR, OLDVAL, NEWVAL ) \
 599     opal_atomic_compare_exchange_strong_rel_xx( (opal_atomic_intptr_t*)(ADDR), (intptr_t *)(OLDVAL), \
 600                                                 (intptr_t)(NEWVAL), sizeof(*(ADDR)) )
 601 
 602 
 603 #endif /* (OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64) */
 604 
 605 #if defined(DOXYGEN) || (OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_MATH_64)
 606 
 607 static inline void opal_atomic_add_xx(opal_atomic_intptr_t* addr,
 608                                       int32_t value, size_t length);
 609 static inline void opal_atomic_sub_xx(opal_atomic_intptr_t* addr,
 610                                       int32_t value, size_t length);
 611 
 612 static inline intptr_t opal_atomic_add_fetch_ptr( opal_atomic_intptr_t* addr, void* delta );
 613 static inline intptr_t opal_atomic_fetch_add_ptr( opal_atomic_intptr_t* addr, void* delta );
 614 static inline intptr_t opal_atomic_sub_fetch_ptr( opal_atomic_intptr_t* addr, void* delta );
 615 static inline intptr_t opal_atomic_fetch_sub_ptr( opal_atomic_intptr_t* addr, void* delta );
 616 
 617 /**
 618  * Atomically increment the content depending on the type. This
 619  * macro detect at compile time the type of the first argument
 620  * and choose the correct function to be called.
 621  *
 622  * \note This macro should only be used for integer types.
 623  *
 624  * @param addr          Address of <TYPE>
 625  * @param delta         Value to add (converted to <TYPE>).
 626  */
 627 #define opal_atomic_add( ADDR, VALUE )                                  \
 628    opal_atomic_add_xx( (opal_atomic_intptr_t*)(ADDR), (int32_t)(VALUE), \
 629                        sizeof(*(ADDR)) )
 630 
 631 /**
 632  * Atomically decrement the content depending on the type. This
 633  * macro detect at compile time the type of the first argument
 634  * and choose the correct function to be called.
 635  *
 636  * \note This macro should only be used for integer types.
 637  *
 638  * @param addr          Address of <TYPE>
 639  * @param delta         Value to substract (converted to <TYPE>).
 640  */
 641 #define opal_atomic_sub( ADDR, VALUE )                                  \
 642    opal_atomic_sub_xx( (opal_atomic_intptr_t*)(ADDR), (int32_t)(VALUE),        \
 643                       sizeof(*(ADDR)) )
 644 
 645 #endif /* OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_MATH_64 */
 646 
 647 
 648 /*
 649  * Include inline implementations of everything not defined directly
 650  * in assembly
 651  */
 652 #include "opal/sys/atomic_impl.h"
 653 
 654 #endif /* !OPAL_C_HAVE__ATOMIC */
 655 
 656 END_C_DECLS
 657 
 658 #endif /* OPAL_SYS_ATOMIC_H */
/* [<][>][^][v][top][bottom][index][help] */