root/opal/mca/pmix/pmix4x/pmix/src/atomics/sys/atomic.h

/* [<][>][^][v][top][bottom][index][help] */

INCLUDED FROM

DEFINITIONS

This source file includes following definitions.
  1. pmix_atomic_add_fetch_size_t
  2. pmix_atomic_fetch_add_size_t
  3. pmix_atomic_sub_fetch_size_t
  4. pmix_atomic_fetch_sub_size_t
   1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
   2 /*
   3  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
   4  *                         University Research and Technology
   5  *                         Corporation.  All rights reserved.
   6  * Copyright (c) 2004-2006 The University of Tennessee and The University
   7  *                         of Tennessee Research Foundation.  All rights
   8  *                         reserved.
   9  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
  10  *                         University of Stuttgart.  All rights reserved.
  11  * Copyright (c) 2004-2005 The Regents of the University of California.
  12  *                         All rights reserved.
  13  * Copyright (c) 2007      Sun Microsystems, Inc.  All rights reserved.
  14  * Copyright (c) 2011      Sandia National Laboratories. All rights reserved.
  15  * Copyright (c) 2011-2017 Los Alamos National Security, LLC. All rights
  16  *                         reserved.
  17  * Copyright (c) 2017      Research Organization for Information Science
  18  *                         and Technology (RIST). All rights reserved.
  19  * Copyright (c) 2018      Intel, Inc.  All rights reserved.
  20  * $COPYRIGHT$
  21  *
  22  * Additional copyrights may follow
  23  *
  24  * $HEADER$
  25  */
  26 
  27 /** @file
  28  *
  29  * Atomic operations.
  30  *
  31  * This API is patterned after the FreeBSD kernel atomic interface
  32  * (which is influenced by Intel's ia64 architecture).  The
  33  * FreeBSD interface is documented at
  34  *
  35  * http://www.freebsd.org/cgi/man.cgi?query=atomic&sektion=9
  36  *
  37  * Only the necessary subset of functions are implemented here.
  38  *
  39  * The following #defines will be true / false based on
  40  * assembly support:
  41  *
  42  *  - \c PMIX_HAVE_ATOMIC_MEM_BARRIER atomic memory barriers
  43  *  - \c PMIX_HAVE_ATOMIC_SPINLOCKS atomic spinlocks
  44  *  - \c PMIX_HAVE_ATOMIC_MATH_32 if 32 bit add/sub/compare-exchange can be done "atomicly"
  45  *  - \c PMIX_HAVE_ATOMIC_MATH_64 if 64 bit add/sub/compare-exchange can be done "atomicly"
  46  *
  47  * Note that for the Atomic math, atomic add/sub may be implemented as
  48  * C code using pmix_atomic_compare_exchange.  The appearance of atomic
  49  * operation will be upheld in these cases.
  50  */
  51 
  52 #ifndef PMIX_SYS_ATOMIC_H
  53 #define PMIX_SYS_ATOMIC_H 1
  54 
  55 #include "pmix_config.h"
  56 
  57 #include <stdbool.h>
  58 
  59 #include "src/atomics/sys/architecture.h"
  60 #include "src/include/pmix_stdatomic.h"
  61 
  62 #if PMIX_ASSEMBLY_BUILTIN == PMIX_BUILTIN_C11
  63 
  64 #include "atomic_stdc.h"
  65 
  66 #else /* !PMIX_C_HAVE__ATOMIC */
  67 
  68 /* do some quick #define cleanup in cases where we are doing
  69    testing... */
  70 #ifdef PMIX_DISABLE_INLINE_ASM
  71 #undef PMIX_C_GCC_INLINE_ASSEMBLY
  72 #define PMIX_C_GCC_INLINE_ASSEMBLY 0
  73 #endif
  74 
  75 /* define PMIX_{GCC,DEC,XLC}_INLINE_ASSEMBLY based on the
  76    PMIX_C_{GCC,DEC,XLC}_INLINE_ASSEMBLY defines and whether we
  77    are in C or C++ */
  78 #if defined(c_plusplus) || defined(__cplusplus)
  79 /* We no longer support inline assembly for C++ as PMIX is a C-only interface */
  80 #define PMIX_GCC_INLINE_ASSEMBLY 0
  81 #else
  82 #define PMIX_GCC_INLINE_ASSEMBLY PMIX_C_GCC_INLINE_ASSEMBLY
  83 #endif
  84 
  85 
  86 BEGIN_C_DECLS
  87 /**********************************************************************
  88  *
  89  * Data structures for atomic ops
  90  *
  91  *********************************************************************/
  92 /**
  93  * Volatile lock object (with optional padding).
  94  *
  95  * \note The internals of the lock are included here, but should be
  96  * considered private.  The implementation currently in use may choose
  97  * to use an int or unsigned char as the lock value - the user is not
  98  * informed either way.
  99  */
 100 struct pmix_atomic_lock_t {
 101     union {
 102         pmix_atomic_int32_t lock;     /**< The lock address (an integer) */
 103         volatile unsigned char sparc_lock; /**< The lock address on sparc */
 104         char padding[sizeof(int)]; /**< Array for optional padding */
 105     } u;
 106 };
 107 typedef struct pmix_atomic_lock_t pmix_atomic_lock_t;
 108 
 109 /**********************************************************************
 110  *
 111  * Set or unset these macros in the architecture-specific atomic.h
 112  * files if we need to specify them as inline or non-inline
 113  *
 114  *********************************************************************/
 115 #if !PMIX_GCC_INLINE_ASSEMBLY
 116 #define PMIX_HAVE_INLINE_ATOMIC_MEM_BARRIER 0
 117 #define PMIX_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32 0
 118 #define PMIX_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64 0
 119 #define PMIX_HAVE_INLINE_ATOMIC_ADD_32 0
 120 #define PMIX_HAVE_INLINE_ATOMIC_AND_32 0
 121 #define PMIX_HAVE_INLINE_ATOMIC_OR_32 0
 122 #define PMIX_HAVE_INLINE_ATOMIC_XOR_32 0
 123 #define PMIX_HAVE_INLINE_ATOMIC_SUB_32 0
 124 #define PMIX_HAVE_INLINE_ATOMIC_ADD_64 0
 125 #define PMIX_HAVE_INLINE_ATOMIC_AND_64 0
 126 #define PMIX_HAVE_INLINE_ATOMIC_OR_64 0
 127 #define PMIX_HAVE_INLINE_ATOMIC_XOR_64 0
 128 #define PMIX_HAVE_INLINE_ATOMIC_SUB_64 0
 129 #define PMIX_HAVE_INLINE_ATOMIC_SWAP_32 0
 130 #define PMIX_HAVE_INLINE_ATOMIC_SWAP_64 0
 131 #else
 132 #define PMIX_HAVE_INLINE_ATOMIC_MEM_BARRIER 1
 133 #define PMIX_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32 1
 134 #define PMIX_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64 1
 135 #define PMIX_HAVE_INLINE_ATOMIC_ADD_32 1
 136 #define PMIX_HAVE_INLINE_ATOMIC_AND_32 1
 137 #define PMIX_HAVE_INLINE_ATOMIC_OR_32 1
 138 #define PMIX_HAVE_INLINE_ATOMIC_XOR_32 1
 139 #define PMIX_HAVE_INLINE_ATOMIC_SUB_32 1
 140 #define PMIX_HAVE_INLINE_ATOMIC_ADD_64 1
 141 #define PMIX_HAVE_INLINE_ATOMIC_AND_64 1
 142 #define PMIX_HAVE_INLINE_ATOMIC_OR_64 1
 143 #define PMIX_HAVE_INLINE_ATOMIC_XOR_64 1
 144 #define PMIX_HAVE_INLINE_ATOMIC_SUB_64 1
 145 #define PMIX_HAVE_INLINE_ATOMIC_SWAP_32 1
 146 #define PMIX_HAVE_INLINE_ATOMIC_SWAP_64 1
 147 #endif
 148 
 149 /**
 150  * Enumeration of lock states
 151  */
 152 enum {
 153     PMIX_ATOMIC_LOCK_UNLOCKED = 0,
 154     PMIX_ATOMIC_LOCK_LOCKED = 1
 155 };
 156 
 157 #define PMIX_ATOMIC_LOCK_INIT {.u = {.lock = PMIX_ATOMIC_LOCK_UNLOCKED}}
 158 
 159 /**********************************************************************
 160  *
 161  * Load the appropriate architecture files and set some reasonable
 162  * default values for our support
 163  *
 164  *********************************************************************/
 165 #if defined(DOXYGEN)
 166 /* don't include system-level gorp when generating doxygen files */
 167 #elif PMIX_ASSEMBLY_BUILTIN == PMIX_BUILTIN_SYNC
 168 #include "src/atomics/sys/sync_builtin/atomic.h"
 169 #elif PMIX_ASSEMBLY_BUILTIN == PMIX_BUILTIN_GCC
 170 #include "src/atomics/sys/gcc_builtin/atomic.h"
 171 #elif PMIX_ASSEMBLY_ARCH == PMIX_X86_64
 172 #include "src/atomics/sys/x86_64/atomic.h"
 173 #elif PMIX_ASSEMBLY_ARCH == PMIX_ARM
 174 #include "src/atomics/sys/arm/atomic.h"
 175 #elif PMIX_ASSEMBLY_ARCH == PMIX_ARM64
 176 #include "src/atomics/sys/arm64/atomic.h"
 177 #elif PMIX_ASSEMBLY_ARCH == PMIX_IA32
 178 #include "src/atomics/sys/ia32/atomic.h"
 179 #elif PMIX_ASSEMBLY_ARCH == PMIX_IA64
 180 #include "src/atomics/sys/ia64/atomic.h"
 181 #elif PMIX_ASSEMBLY_ARCH == PMIX_MIPS
 182 #include "src/atomics/sys/mips/atomic.h"
 183 #elif PMIX_ASSEMBLY_ARCH == PMIX_POWERPC32
 184 #include "src/atomics/sys/powerpc/atomic.h"
 185 #elif PMIX_ASSEMBLY_ARCH == PMIX_POWERPC64
 186 #include "src/atomics/sys/powerpc/atomic.h"
 187 #elif PMIX_ASSEMBLY_ARCH == PMIX_SPARC
 188 #include "src/atomics/sys/sparc/atomic.h"
 189 #elif PMIX_ASSEMBLY_ARCH == PMIX_SPARCV9_32
 190 #include "src/atomics/sys/sparcv9/atomic.h"
 191 #elif PMIX_ASSEMBLY_ARCH == PMIX_SPARCV9_64
 192 #include "src/atomics/sys/sparcv9/atomic.h"
 193 #endif
 194 
 195 #ifndef DOXYGEN
 196 /* compare and set operations can't really be emulated from software,
 197    so if these defines aren't already set, they should be set to 0
 198    now */
 199 #ifndef PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32
 200 #define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 0
 201 #endif
 202 #ifndef PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64
 203 #define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64 0
 204 #endif
 205 #ifndef PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_128
 206 #define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_128 0
 207 #endif
 208 #ifndef PMIX_HAVE_ATOMIC_LLSC_32
 209 #define PMIX_HAVE_ATOMIC_LLSC_32 0
 210 #endif
 211 #ifndef PMIX_HAVE_ATOMIC_LLSC_64
 212 #define PMIX_HAVE_ATOMIC_LLSC_64 0
 213 #endif
 214 #endif /* DOXYGEN */
 215 
 216 /**********************************************************************
 217  *
 218  * Memory Barriers - defined here if running doxygen or have barriers
 219  *                   but can't inline
 220  *
 221  *********************************************************************/
 222 #if !defined(PMIX_HAVE_ATOMIC_MEM_BARRIER) && !defined(DOXYGEN)
 223 /* no way to emulate in C code */
 224 #define PMIX_HAVE_ATOMIC_MEM_BARRIER 0
 225 #endif
 226 
 227 #if defined(DOXYGEN) || PMIX_HAVE_ATOMIC_MEM_BARRIER
 228 /**
 229  * Memory barrier
 230  *
 231  * Will use system-specific features to instruct the processor and
 232  * memory controller that all writes and reads that have been posted
 233  * before the call to \c pmix_atomic_mb() must appear to have
 234  * completed before the next read or write.
 235  *
 236  * \note This can have some expensive side effects, including flushing
 237  * the pipeline, preventing the cpu from reordering instructions, and
 238  * generally grinding the memory controller's performance.  Use only
 239  * if you need *both* read and write barriers.
 240  */
 241 
 242 #if PMIX_HAVE_INLINE_ATOMIC_MEM_BARRIER
 243 static inline
 244 #endif
 245 void pmix_atomic_mb(void);
 246 
 247 /**
 248  * Read memory barrier
 249  *
 250  * Use system-specific features to instruct the processor and memory
 251  * conrtoller that all reads that have been posted before the call to
 252  * \c pmix_atomic_rmb() must appear to have been completed before the
 253  * next read.  Nothing is said about the ordering of writes when using
 254  * \c pmix_atomic_rmb().
 255  */
 256 
 257 #if PMIX_HAVE_INLINE_ATOMIC_MEM_BARRIER
 258 static inline
 259 #endif
 260 void pmix_atomic_rmb(void);
 261 
 262 /**
 263  * Write memory barrier.
 264  *
 265  * Use system-specific features to instruct the processor and memory
 266  * conrtoller that all writes that have been posted before the call to
 267  * \c pmix_atomic_wmb() must appear to have been completed before the
 268  * next write.  Nothing is said about the ordering of reads when using
 269  * \c pmix_atomic_wmb().
 270  */
 271 
 272 #if PMIX_HAVE_INLINE_ATOMIC_MEM_BARRIER
 273 static inline
 274 #endif
 275 void pmix_atomic_wmb(void);
 276 
 277 #endif /* defined(DOXYGEN) || PMIX_HAVE_ATOMIC_MEM_BARRIER */
 278 
 279 
 280 /**********************************************************************
 281  *
 282  * Atomic spinlocks - always inlined, if have atomic compare-and-swap
 283  *
 284  *********************************************************************/
 285 
 286 #if !defined(PMIX_HAVE_ATOMIC_SPINLOCKS) && !defined(DOXYGEN)
 287 /* 0 is more like "pending" - we'll fix up at the end after all
 288    the static inline functions are declared */
 289 #define PMIX_HAVE_ATOMIC_SPINLOCKS 0
 290 #endif
 291 
 292 #if defined(DOXYGEN) || PMIX_HAVE_ATOMIC_SPINLOCKS || (PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 || PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64)
 293 
 294 /**
 295  * Initialize a lock to value
 296  *
 297  * @param lock         Address of the lock
 298  * @param value        Initial value to set lock to
 299  */
 300 #if PMIX_HAVE_ATOMIC_SPINLOCKS == 0
 301 static inline
 302 #endif
 303 void pmix_atomic_lock_init(pmix_atomic_lock_t* lock, int32_t value);
 304 
 305 
 306 /**
 307  * Try to acquire a lock.
 308  *
 309  * @param lock          Address of the lock.
 310  * @return              0 if the lock was acquired, 1 otherwise.
 311  */
 312 #if PMIX_HAVE_ATOMIC_SPINLOCKS == 0
 313 static inline
 314 #endif
 315 int pmix_atomic_trylock(pmix_atomic_lock_t *lock);
 316 
 317 
 318 /**
 319  * Acquire a lock by spinning.
 320  *
 321  * @param lock          Address of the lock.
 322  */
 323 #if PMIX_HAVE_ATOMIC_SPINLOCKS == 0
 324 static inline
 325 #endif
 326 void pmix_atomic_lock(pmix_atomic_lock_t *lock);
 327 
 328 
 329 /**
 330  * Release a lock.
 331  *
 332  * @param lock          Address of the lock.
 333  */
 334 #if PMIX_HAVE_ATOMIC_SPINLOCKS == 0
 335 static inline
 336 #endif
 337 void pmix_atomic_unlock(pmix_atomic_lock_t *lock);
 338 
 339 
 340 #if PMIX_HAVE_ATOMIC_SPINLOCKS == 0
 341 #undef PMIX_HAVE_ATOMIC_SPINLOCKS
 342 #define PMIX_HAVE_ATOMIC_SPINLOCKS (PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 || PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64)
 343 #define PMIX_NEED_INLINE_ATOMIC_SPINLOCKS 1
 344 #endif
 345 
 346 #endif /* PMIX_HAVE_ATOMIC_SPINLOCKS */
 347 
 348 
 349 /**********************************************************************
 350  *
 351  * Atomic math operations
 352  *
 353  *********************************************************************/
 354 #if !defined(PMIX_HAVE_ATOMIC_CMPSET_32) && !defined(DOXYGEN)
 355 #define PMIX_HAVE_ATOMIC_CMPSET_32 0
 356 #endif
 357 #if defined(DOXYGEN) || PMIX_HAVE_ATOMIC_CMPSET_32
 358 
 359 #if PMIX_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32
 360 static inline
 361 #endif
 362 bool pmix_atomic_compare_exchange_strong_32 (pmix_atomic_int32_t *addr, int32_t *oldval,
 363                                              int32_t newval);
 364 
 365 #if PMIX_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32
 366 static inline
 367 #endif
 368 bool pmix_atomic_compare_exchange_strong_acq_32 (pmix_atomic_int32_t *addr, int32_t *oldval,
 369                                                  int32_t newval);
 370 
 371 #if PMIX_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32
 372 static inline
 373 #endif
 374 bool pmix_atomic_compare_exchange_strong_rel_32 (pmix_atomic_int32_t *addr, int32_t *oldval,
 375                                                  int32_t newval);
 376 #endif
 377 
 378 
 379 #if !defined(PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64) && !defined(DOXYGEN)
 380 #define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64 0
 381 #endif
 382 #if defined(DOXYGEN) || PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64
 383 
 384 #if PMIX_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64
 385 static inline
 386 #endif
 387 bool pmix_atomic_compare_exchange_strong_64 (pmix_atomic_int64_t *addr, int64_t *oldval,
 388                                              int64_t newval);
 389 
 390 #if PMIX_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64
 391 static inline
 392 #endif
 393 bool pmix_atomic_compare_exchange_strong_acq_64 (pmix_atomic_int64_t *addr, int64_t *oldval,
 394                                                  int64_t newval);
 395 
 396 #if PMIX_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64
 397 static inline
 398 #endif
 399 bool pmix_atomic_compare_exchange_strong_rel_64 (pmix_atomic_int64_t *addr, int64_t *oldval,
 400                                                  int64_t newval);
 401 
 402 #endif
 403 
 404 #if !defined(PMIX_HAVE_ATOMIC_MATH_32) && !defined(DOXYGEN)
 405   /* define to 0 for these tests.  WIll fix up later. */
 406   #define PMIX_HAVE_ATOMIC_MATH_32 0
 407 #endif
 408 
 409 #if defined(DOXYGEN) || PMIX_HAVE_ATOMIC_MATH_32 || PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32
 410 
 411 static inline int32_t pmix_atomic_add_fetch_32(pmix_atomic_int32_t *addr, int delta);
 412 static inline int32_t pmix_atomic_fetch_add_32(pmix_atomic_int32_t *addr, int delta);
 413 static inline int32_t pmix_atomic_and_fetch_32(pmix_atomic_int32_t *addr, int32_t value);
 414 static inline int32_t pmix_atomic_fetch_and_32(pmix_atomic_int32_t *addr, int32_t value);
 415 static inline int32_t pmix_atomic_or_fetch_32(pmix_atomic_int32_t *addr, int32_t value);
 416 static inline int32_t pmix_atomic_fetch_or_32(pmix_atomic_int32_t *addr, int32_t value);
 417 static inline int32_t pmix_atomic_xor_fetch_32(pmix_atomic_int32_t *addr, int32_t value);
 418 static inline int32_t pmix_atomic_fetch_xor_32(pmix_atomic_int32_t *addr, int32_t value);
 419 static inline int32_t pmix_atomic_sub_fetch_32(pmix_atomic_int32_t *addr, int delta);
 420 static inline int32_t pmix_atomic_fetch_sub_32(pmix_atomic_int32_t *addr, int delta);
 421 static inline int32_t pmix_atomic_min_fetch_32 (pmix_atomic_int32_t *addr, int32_t value);
 422 static inline int32_t pmix_atomic_fetch_min_32 (pmix_atomic_int32_t *addr, int32_t value);
 423 static inline int32_t pmix_atomic_max_fetch_32 (pmix_atomic_int32_t *addr, int32_t value);
 424 static inline int32_t pmix_atomic_fetch_max_32 (pmix_atomic_int32_t *addr, int32_t value);
 425 
 426 #endif /* PMIX_HAVE_ATOMIC_MATH_32 */
 427 
 428 #if ! PMIX_HAVE_ATOMIC_MATH_32
 429 /* fix up the value of pmix_have_atomic_math_32 to allow for C versions */
 430 #undef PMIX_HAVE_ATOMIC_MATH_32
 431 #define PMIX_HAVE_ATOMIC_MATH_32 PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32
 432 #endif
 433 
 434 #ifndef PMIX_HAVE_ATOMIC_MATH_64
 435 /* define to 0 for these tests.  WIll fix up later. */
 436 #define PMIX_HAVE_ATOMIC_MATH_64 0
 437 #endif
 438 
 439 #if defined(DOXYGEN) || PMIX_HAVE_ATOMIC_MATH_64 || PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64
 440 
 441 static inline int64_t pmix_atomic_add_fetch_64(pmix_atomic_int64_t *addr, int64_t delta);
 442 static inline int64_t pmix_atomic_fetch_add_64(pmix_atomic_int64_t *addr, int64_t delta);
 443 static inline int64_t pmix_atomic_and_fetch_64(pmix_atomic_int64_t *addr, int64_t value);
 444 static inline int64_t pmix_atomic_fetch_and_64(pmix_atomic_int64_t *addr, int64_t value);
 445 static inline int64_t pmix_atomic_or_fetch_64(pmix_atomic_int64_t *addr, int64_t value);
 446 static inline int64_t pmix_atomic_fetch_or_64(pmix_atomic_int64_t *addr, int64_t value);
 447 static inline int64_t pmix_atomic_fetch_xor_64(pmix_atomic_int64_t *addr, int64_t value);
 448 static inline int64_t pmix_atomic_sub_fetch_64(pmix_atomic_int64_t *addr, int64_t delta);
 449 static inline int64_t pmix_atomic_fetch_sub_64(pmix_atomic_int64_t *addr, int64_t delta);
 450 static inline int64_t pmix_atomic_min_fetch_64 (pmix_atomic_int64_t *addr, int64_t value);
 451 static inline int64_t pmix_atomic_fetch_min_64 (pmix_atomic_int64_t *addr, int64_t value);
 452 static inline int64_t pmix_atomic_max_fetch_64 (pmix_atomic_int64_t *addr, int64_t value);
 453 static inline int64_t pmix_atomic_fetch_max_64 (pmix_atomic_int64_t *addr, int64_t value);
 454 
 455 #endif /* PMIX_HAVE_ATOMIC_MATH_64 */
 456 
 457 #if ! PMIX_HAVE_ATOMIC_MATH_64
 458 /* fix up the value of pmix_have_atomic_math_64 to allow for C versions */
 459 #undef PMIX_HAVE_ATOMIC_MATH_64
 460 #define PMIX_HAVE_ATOMIC_MATH_64 PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64
 461 #endif
 462 
 463 /* provide a size_t add/subtract.  When in debug mode, make it an
 464  * inline function so that we don't have any casts in the
 465  *  interface and can catch type errors.  When not in debug mode,
 466  * just make it a macro, so that there's no performance penalty
 467  */
 468 #if defined(DOXYGEN) || PMIX_ENABLE_DEBUG
 469 static inline size_t
 470 pmix_atomic_add_fetch_size_t(pmix_atomic_size_t *addr, size_t delta)
 471 {
 472 #if SIZEOF_SIZE_T == 4
 473     return (size_t) pmix_atomic_add_fetch_32((int32_t*) addr, delta);
 474 #elif SIZEOF_SIZE_T == 8
 475     return (size_t) pmix_atomic_add_fetch_64((int64_t*) addr, delta);
 476 #else
 477 #error "Unknown size_t size"
 478 #endif
 479 }
 480 
 481 static inline size_t
 482 pmix_atomic_fetch_add_size_t(pmix_atomic_size_t *addr, size_t delta)
 483 {
 484 #if SIZEOF_SIZE_T == 4
 485     return (size_t) pmix_atomic_fetch_add_32((int32_t*) addr, delta);
 486 #elif SIZEOF_SIZE_T == 8
 487     return (size_t) pmix_atomic_fetch_add_64((int64_t*) addr, delta);
 488 #else
 489 #error "Unknown size_t size"
 490 #endif
 491 }
 492 
 493 static inline size_t
 494 pmix_atomic_sub_fetch_size_t(pmix_atomic_size_t *addr, size_t delta)
 495 {
 496 #if SIZEOF_SIZE_T == 4
 497     return (size_t) pmix_atomic_sub_fetch_32((int32_t*) addr, delta);
 498 #elif SIZEOF_SIZE_T == 8
 499     return (size_t) pmix_atomic_sub_fetch_64((int64_t*) addr, delta);
 500 #else
 501 #error "Unknown size_t size"
 502 #endif
 503 }
 504 
 505 static inline size_t
 506 pmix_atomic_fetch_sub_size_t(pmix_atomic_size_t *addr, size_t delta)
 507 {
 508 #if SIZEOF_SIZE_T == 4
 509     return (size_t) pmix_atomic_fetch_sub_32((int32_t*) addr, delta);
 510 #elif SIZEOF_SIZE_T == 8
 511     return (size_t) pmix_atomic_fetch_sub_64((int64_t*) addr, delta);
 512 #else
 513 #error "Unknown size_t size"
 514 #endif
 515 }
 516 
 517 #else
 518 #if SIZEOF_SIZE_T == 4
 519 #define pmix_atomic_add_fetch_size_t(addr, delta) ((size_t) pmix_atomic_add_fetch_32((pmix_atomic_int32_t *) addr, delta))
 520 #define pmix_atomic_fetch_add_size_t(addr, delta) ((size_t) pmix_atomic_fetch_add_32((pmix_atomic_int32_t *) addr, delta))
 521 #define pmix_atomic_sub_fetch_size_t(addr, delta) ((size_t) pmix_atomic_sub_fetch_32((pmix_atomic_int32_t *) addr, delta))
 522 #define pmix_atomic_fetch_sub_size_t(addr, delta) ((size_t) pmix_atomic_fetch_sub_32((pmix_atomic_int32_t *) addr, delta))
 523 #elif SIZEOF_SIZE_T == 8
 524 #define pmix_atomic_add_fetch_size_t(addr, delta) ((size_t) pmix_atomic_add_fetch_64((pmix_atomic_int64_t *) addr, delta))
 525 #define pmix_atomic_fetch_add_size_t(addr, delta) ((size_t) pmix_atomic_fetch_add_64((pmix_atomic_int64_t *) addr, delta))
 526 #define pmix_atomic_sub_fetch_size_t(addr, delta) ((size_t) pmix_atomic_sub_fetch_64((pmix_atomic_int64_t *) addr, delta))
 527 #define pmix_atomic_fetch_sub_size_t(addr, delta) ((size_t) pmix_atomic_fetch_sub_64((pmix_atomic_int64_t *) addr, delta))
 528 #else
 529 #error "Unknown size_t size"
 530 #endif
 531 #endif
 532 
 533 #if defined(DOXYGEN) || (PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 || PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64)
 534 /* these are always done with inline functions, so always mark as
 535    static inline */
 536 
 537 static inline bool pmix_atomic_compare_exchange_strong_xx (pmix_atomic_intptr_t *addr, intptr_t *oldval,
 538                                                            int64_t newval, size_t length);
 539 static inline bool pmix_atomic_compare_exchange_strong_acq_xx (pmix_atomic_intptr_t *addr, intptr_t *oldval,
 540                                                                int64_t newval, size_t length);
 541 static inline bool pmix_atomic_compare_exchange_strong_rel_xx (pmix_atomic_intptr_t *addr, intptr_t *oldval,
 542                                                                int64_t newval, size_t length);
 543 
 544 
 545 static inline bool pmix_atomic_compare_exchange_strong_ptr (pmix_atomic_intptr_t* addr, intptr_t *oldval,
 546                                                             intptr_t newval);
 547 static inline bool pmix_atomic_compare_exchange_strong_acq_ptr (pmix_atomic_intptr_t* addr, intptr_t *oldval,
 548                                                                 intptr_t newval);
 549 static inline bool pmix_atomic_compare_exchange_strong_rel_ptr (pmix_atomic_intptr_t* addr, intptr_t *oldval,
 550                                                                 intptr_t newval);
 551 
 552 /**
 553  * Atomic compare and set of generic type with relaxed semantics. This
 554  * macro detect at compile time the type of the first argument and
 555  * choose the correct function to be called.
 556  *
 557  * \note This macro should only be used for integer types.
 558  *
 559  * @param addr          Address of <TYPE>.
 560  * @param oldval        Comparison value address of <TYPE>.
 561  * @param newval        New value to set if comparision is true <TYPE>.
 562  *
 563  * See pmix_atomic_compare_exchange_* for pseudo-code.
 564  */
 565 #define pmix_atomic_compare_exchange_strong( ADDR, OLDVAL, NEWVAL )                  \
 566     pmix_atomic_compare_exchange_strong_xx( (pmix_atomic_intptr_t*)(ADDR), (intptr_t *)(OLDVAL), \
 567                                             (intptr_t)(NEWVAL), sizeof(*(ADDR)) )
 568 
 569 /**
 570  * Atomic compare and set of generic type with acquire semantics. This
 571  * macro detect at compile time the type of the first argument and
 572  * choose the correct function to be called.
 573  *
 574  * \note This macro should only be used for integer types.
 575  *
 576  * @param addr          Address of <TYPE>.
 577  * @param oldval        Comparison value address of <TYPE>.
 578  * @param newval        New value to set if comparision is true <TYPE>.
 579  *
 580  * See pmix_atomic_compare_exchange_acq_* for pseudo-code.
 581  */
 582 #define pmix_atomic_compare_exchange_strong_acq( ADDR, OLDVAL, NEWVAL )                  \
 583     pmix_atomic_compare_exchange_strong_acq_xx( (pmix_atomic_intptr_t*)(ADDR), (intptr_t *)(OLDVAL), \
 584                                                 (intptr_t)(NEWVAL), sizeof(*(ADDR)) )
 585 
 586 /**
 587  * Atomic compare and set of generic type with release semantics. This
 588  * macro detect at compile time the type of the first argument and
 589  * choose the correct function to be called.
 590  *
 591  * \note This macro should only be used for integer types.
 592  *
 593  * @param addr          Address of <TYPE>.
 594  * @param oldval        Comparison value address of <TYPE>.
 595  * @param newval        New value to set if comparision is true <TYPE>.
 596  *
 597  * See pmix_atomic_compare_exchange_rel_* for pseudo-code.
 598  */
 599 #define pmix_atomic_compare_exchange_strong_rel( ADDR, OLDVAL, NEWVAL ) \
 600     pmix_atomic_compare_exchange_strong_rel_xx( (pmix_atomic_intptr_t*)(ADDR), (intptr_t *)(OLDVAL), \
 601                                                 (intptr_t)(NEWVAL), sizeof(*(ADDR)) )
 602 
 603 
 604 #endif /* (PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 || PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64) */
 605 
 606 #if defined(DOXYGEN) || (PMIX_HAVE_ATOMIC_MATH_32 || PMIX_HAVE_ATOMIC_MATH_64)
 607 
 608 static inline void pmix_atomic_add_xx(pmix_atomic_intptr_t* addr,
 609                                       int32_t value, size_t length);
 610 static inline void pmix_atomic_sub_xx(pmix_atomic_intptr_t* addr,
 611                                       int32_t value, size_t length);
 612 
 613 static inline intptr_t pmix_atomic_add_fetch_ptr( pmix_atomic_intptr_t* addr, void* delta );
 614 static inline intptr_t pmix_atomic_fetch_add_ptr( pmix_atomic_intptr_t* addr, void* delta );
 615 static inline intptr_t pmix_atomic_sub_fetch_ptr( pmix_atomic_intptr_t* addr, void* delta );
 616 static inline intptr_t pmix_atomic_fetch_sub_ptr( pmix_atomic_intptr_t* addr, void* delta );
 617 
 618 /**
 619  * Atomically increment the content depending on the type. This
 620  * macro detect at compile time the type of the first argument
 621  * and choose the correct function to be called.
 622  *
 623  * \note This macro should only be used for integer types.
 624  *
 625  * @param addr          Address of <TYPE>
 626  * @param delta         Value to add (converted to <TYPE>).
 627  */
 628 #define pmix_atomic_add( ADDR, VALUE )                                  \
 629    pmix_atomic_add_xx( (pmix_atomic_intptr_t*)(ADDR), (int32_t)(VALUE), \
 630                        sizeof(*(ADDR)) )
 631 
 632 /**
 633  * Atomically decrement the content depending on the type. This
 634  * macro detect at compile time the type of the first argument
 635  * and choose the correct function to be called.
 636  *
 637  * \note This macro should only be used for integer types.
 638  *
 639  * @param addr          Address of <TYPE>
 640  * @param delta         Value to substract (converted to <TYPE>).
 641  */
 642 #define pmix_atomic_sub( ADDR, VALUE )                                  \
 643    pmix_atomic_sub_xx( (pmix_atomic_intptr_t*)(ADDR), (int32_t)(VALUE),        \
 644                       sizeof(*(ADDR)) )
 645 
 646 #endif /* PMIX_HAVE_ATOMIC_MATH_32 || PMIX_HAVE_ATOMIC_MATH_64 */
 647 
 648 
 649 /*
 650  * Include inline implementations of everything not defined directly
 651  * in assembly
 652  */
 653 #include "src/atomics/sys/atomic_impl.h"
 654 
 655 #endif /* !PMIX_C_HAVE__ATOMIC */
 656 
 657 END_C_DECLS
 658 
 659 #endif /* PMIX_SYS_ATOMIC_H */
/* [<][>][^][v][top][bottom][index][help] */