This source file includes following definitions.
- opal_atomic_mb
- opal_atomic_rmb
- opal_atomic_wmb
- opal_atomic_compare_exchange_strong_acq_32
- opal_atomic_compare_exchange_strong_rel_32
- opal_atomic_compare_exchange_strong_32
- opal_atomic_swap_32
- opal_atomic_fetch_add_32
- opal_atomic_fetch_and_32
- opal_atomic_fetch_or_32
- opal_atomic_fetch_xor_32
- opal_atomic_fetch_sub_32
- opal_atomic_compare_exchange_strong_acq_64
- opal_atomic_compare_exchange_strong_rel_64
- opal_atomic_compare_exchange_strong_64
- opal_atomic_swap_64
- opal_atomic_fetch_add_64
- opal_atomic_fetch_and_64
- opal_atomic_fetch_or_64
- opal_atomic_fetch_xor_64
- opal_atomic_fetch_sub_64
- opal_atomic_compare_exchange_strong_128
- opal_atomic_compare_exchange_strong_128
- opal_atomic_lock_init
- opal_atomic_trylock
- opal_atomic_lock
- opal_atomic_unlock
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27 #ifndef OPAL_SYS_ARCH_ATOMIC_H
28 #define OPAL_SYS_ARCH_ATOMIC_H 1
29
30
31
32
33
34
35 #define OPAL_HAVE_ATOMIC_MEM_BARRIER 1
36
37 #define OPAL_HAVE_ATOMIC_MATH_32 1
38 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
39 #define OPAL_HAVE_ATOMIC_ADD_32 1
40 #define OPAL_HAVE_ATOMIC_AND_32 1
41 #define OPAL_HAVE_ATOMIC_OR_32 1
42 #define OPAL_HAVE_ATOMIC_XOR_32 1
43 #define OPAL_HAVE_ATOMIC_SUB_32 1
44 #define OPAL_HAVE_ATOMIC_SWAP_32 1
45 #define OPAL_HAVE_ATOMIC_MATH_64 1
46 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1
47 #define OPAL_HAVE_ATOMIC_ADD_64 1
48 #define OPAL_HAVE_ATOMIC_AND_64 1
49 #define OPAL_HAVE_ATOMIC_OR_64 1
50 #define OPAL_HAVE_ATOMIC_XOR_64 1
51 #define OPAL_HAVE_ATOMIC_SUB_64 1
52 #define OPAL_HAVE_ATOMIC_SWAP_64 1
53
54
55 static inline void opal_atomic_mb(void)
56 {
57 __atomic_thread_fence (__ATOMIC_SEQ_CST);
58 }
59
60 static inline void opal_atomic_rmb(void)
61 {
62 #if OPAL_ASSEMBLY_ARCH == OPAL_X86_64
63
64
65
66 __atomic_thread_fence (__ATOMIC_SEQ_CST);
67 #else
68 __atomic_thread_fence (__ATOMIC_ACQUIRE);
69 #endif
70 }
71
72 static inline void opal_atomic_wmb(void)
73 {
74 __atomic_thread_fence (__ATOMIC_RELEASE);
75 }
76
77 #define MB() opal_atomic_mb()
78
79
80
81
82
83
84
85
86
87
88
89 #if defined(__SUNPRO_C) || defined(__SUNPRO_CC)
90 #pragma error_messages(off, E_ARG_INCOMPATIBLE_WITH_ARG_L)
91 #endif
92
93 static inline bool opal_atomic_compare_exchange_strong_acq_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
94 {
95 return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
96 }
97
98
99 static inline bool opal_atomic_compare_exchange_strong_rel_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
100 {
101 return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
102 }
103
104 static inline bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
105 {
106 return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
107 }
108
109 static inline int32_t opal_atomic_swap_32 (opal_atomic_int32_t *addr, int32_t newval)
110 {
111 int32_t oldval;
112 __atomic_exchange (addr, &newval, &oldval, __ATOMIC_RELAXED);
113 return oldval;
114 }
115
116 static inline int32_t opal_atomic_fetch_add_32(opal_atomic_int32_t *addr, int32_t delta)
117 {
118 return __atomic_fetch_add (addr, delta, __ATOMIC_RELAXED);
119 }
120
121 static inline int32_t opal_atomic_fetch_and_32(opal_atomic_int32_t *addr, int32_t value)
122 {
123 return __atomic_fetch_and (addr, value, __ATOMIC_RELAXED);
124 }
125
126 static inline int32_t opal_atomic_fetch_or_32(opal_atomic_int32_t *addr, int32_t value)
127 {
128 return __atomic_fetch_or (addr, value, __ATOMIC_RELAXED);
129 }
130
131 static inline int32_t opal_atomic_fetch_xor_32(opal_atomic_int32_t *addr, int32_t value)
132 {
133 return __atomic_fetch_xor (addr, value, __ATOMIC_RELAXED);
134 }
135
136 static inline int32_t opal_atomic_fetch_sub_32(opal_atomic_int32_t *addr, int32_t delta)
137 {
138 return __atomic_fetch_sub (addr, delta, __ATOMIC_RELAXED);
139 }
140
141 static inline bool opal_atomic_compare_exchange_strong_acq_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
142 {
143 return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
144 }
145
146 static inline bool opal_atomic_compare_exchange_strong_rel_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
147 {
148 return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
149 }
150
151
152 static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
153 {
154 return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
155 }
156
157 static inline int64_t opal_atomic_swap_64 (opal_atomic_int64_t *addr, int64_t newval)
158 {
159 int64_t oldval;
160 __atomic_exchange (addr, &newval, &oldval, __ATOMIC_RELAXED);
161 return oldval;
162 }
163
164 static inline int64_t opal_atomic_fetch_add_64(opal_atomic_int64_t *addr, int64_t delta)
165 {
166 return __atomic_fetch_add (addr, delta, __ATOMIC_RELAXED);
167 }
168
169 static inline int64_t opal_atomic_fetch_and_64(opal_atomic_int64_t *addr, int64_t value)
170 {
171 return __atomic_fetch_and (addr, value, __ATOMIC_RELAXED);
172 }
173
174 static inline int64_t opal_atomic_fetch_or_64(opal_atomic_int64_t *addr, int64_t value)
175 {
176 return __atomic_fetch_or (addr, value, __ATOMIC_RELAXED);
177 }
178
179 static inline int64_t opal_atomic_fetch_xor_64(opal_atomic_int64_t *addr, int64_t value)
180 {
181 return __atomic_fetch_xor (addr, value, __ATOMIC_RELAXED);
182 }
183
184 static inline int64_t opal_atomic_fetch_sub_64(opal_atomic_int64_t *addr, int64_t delta)
185 {
186 return __atomic_fetch_sub (addr, delta, __ATOMIC_RELAXED);
187 }
188
189 #if OPAL_HAVE_GCC_BUILTIN_CSWAP_INT128
190
191 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1
192
193 static inline bool opal_atomic_compare_exchange_strong_128 (opal_atomic_int128_t *addr,
194 opal_int128_t *oldval, opal_int128_t newval)
195 {
196 return __atomic_compare_exchange_n (addr, oldval, newval, false,
197 __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
198 }
199
200 #elif defined(OPAL_HAVE_SYNC_BUILTIN_CSWAP_INT128) && OPAL_HAVE_SYNC_BUILTIN_CSWAP_INT128
201
202 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1
203
204
205
206 static inline bool opal_atomic_compare_exchange_strong_128 (opal_atomic_opal_int128_t *addr,
207 opal_int128_t *oldval, opal_int128_t newval)
208 {
209 opal_int128_t prev = __sync_val_compare_and_swap (addr, *oldval, newval);
210 bool ret = prev == *oldval;
211 *oldval = prev;
212 return ret;
213 }
214
215 #endif
216
217 #if defined(__HLE__)
218
219 #include <immintrin.h>
220
221 #define OPAL_HAVE_ATOMIC_SPINLOCKS 1
222
223 static inline void opal_atomic_lock_init (opal_atomic_lock_t* lock, int32_t value)
224 {
225 lock->u.lock = value;
226 }
227
228 static inline int opal_atomic_trylock(opal_atomic_lock_t *lock)
229 {
230 int ret = __atomic_exchange_n (&lock->u.lock, OPAL_ATOMIC_LOCK_LOCKED,
231 __ATOMIC_ACQUIRE | __ATOMIC_HLE_ACQUIRE);
232 if (OPAL_ATOMIC_LOCK_LOCKED == ret) {
233
234 _mm_pause ();
235 return 1;
236 }
237
238 return 0;
239 }
240
241 static inline void opal_atomic_lock (opal_atomic_lock_t *lock)
242 {
243 while (OPAL_ATOMIC_LOCK_LOCKED == __atomic_exchange_n (&lock->u.lock, OPAL_ATOMIC_LOCK_LOCKED,
244 __ATOMIC_ACQUIRE | __ATOMIC_HLE_ACQUIRE)) {
245
246 _mm_pause ();
247 }
248 }
249
250 static inline void opal_atomic_unlock (opal_atomic_lock_t *lock)
251 {
252 __atomic_store_n (&lock->u.lock, OPAL_ATOMIC_LOCK_UNLOCKED,
253 __ATOMIC_RELEASE | __ATOMIC_HLE_RELEASE);
254 }
255
256 #endif
257
258 #if defined(__SUNPRO_C) || defined(__SUNPRO_CC)
259 #pragma error_messages(default, E_ARG_INCOMPATIBLE_WITH_ARG_L)
260 #endif
261
262 #endif