This source file includes following definitions.
- opal_atomic_mb
- opal_atomic_rmb
- opal_atomic_wmb
- opal_atomic_isync
- opal_atomic_compare_exchange_strong_32
- opal_atomic_compare_exchange_strong_acq_32
- opal_atomic_compare_exchange_strong_rel_32
- opal_atomic_swap_32
- OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64
- opal_atomic_swap_64
- opal_atomic_compare_exchange_strong_64
- opal_atomic_compare_exchange_strong_acq_64
- opal_atomic_compare_exchange_strong_rel_64
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23 #ifndef OPAL_SYS_ARCH_ATOMIC_H
24 #define OPAL_SYS_ARCH_ATOMIC_H 1
25
26
27
28
29
30 #define MB() __asm__ __volatile__ ("sync" : : : "memory")
31 #define RMB() __asm__ __volatile__ ("lwsync" : : : "memory")
32 #define WMB() __asm__ __volatile__ ("lwsync" : : : "memory")
33 #define ISYNC() __asm__ __volatile__ ("isync" : : : "memory")
34
35
36
37
38
39
40
41 #define OPAL_HAVE_ATOMIC_MEM_BARRIER 1
42
43 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
44 #define OPAL_HAVE_ATOMIC_SWAP_32 1
45 #define OPAL_HAVE_ATOMIC_LLSC_32 1
46
47 #define OPAL_HAVE_ATOMIC_MATH_32 1
48 #define OPAL_HAVE_ATOMIC_ADD_32 1
49 #define OPAL_HAVE_ATOMIC_AND_32 1
50 #define OPAL_HAVE_ATOMIC_OR_32 1
51 #define OPAL_HAVE_ATOMIC_XOR_32 1
52 #define OPAL_HAVE_ATOMIC_SUB_32 1
53
54
55 #if (OPAL_ASSEMBLY_ARCH == OPAL_POWERPC64) || OPAL_ASM_SUPPORT_64BIT
56 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1
57 #define OPAL_HAVE_ATOMIC_SWAP_64 1
58 #define OPAL_HAVE_ATOMIC_LLSC_64 1
59 #define OPAL_HAVE_ATOMIC_MATH_64 1
60 #define OPAL_HAVE_ATOMIC_ADD_64 1
61 #define OPAL_HAVE_ATOMIC_AND_64 1
62 #define OPAL_HAVE_ATOMIC_OR_64 1
63 #define OPAL_HAVE_ATOMIC_XOR_64 1
64 #define OPAL_HAVE_ATOMIC_SUB_64 1
65 #endif
66
67
68
69
70
71
72
73 #if OPAL_GCC_INLINE_ASSEMBLY
74
75 static inline
76 void opal_atomic_mb(void)
77 {
78 MB();
79 }
80
81
82 static inline
83 void opal_atomic_rmb(void)
84 {
85 RMB();
86 }
87
88
89 static inline
90 void opal_atomic_wmb(void)
91 {
92 WMB();
93 }
94
95 static inline
96 void opal_atomic_isync(void)
97 {
98 ISYNC();
99 }
100
101 #endif
102
103
104
105
106
107
108 #if OPAL_GCC_INLINE_ASSEMBLY
109
110 #ifdef __xlC__
111
112
113 #define OPAL_ASM_ADDR(a) ((uintptr_t)a)
114 #else
115 #define OPAL_ASM_ADDR(a) (a)
116 #endif
117
118 #if defined(__PGI)
119
120
121
122 #define OPAL_ASM_VALUE64(x) (void *)(intptr_t) (x)
123 #else
124 #define OPAL_ASM_VALUE64(x) x
125 #endif
126
127 static inline bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
128 {
129 int32_t prev;
130 bool ret;
131
132 __asm__ __volatile__ (
133 "1: lwarx %0, 0, %2 \n\t"
134 " cmpw 0, %0, %3 \n\t"
135 " bne- 2f \n\t"
136 " stwcx. %4, 0, %2 \n\t"
137 " bne- 1b \n\t"
138 "2:"
139 : "=&r" (prev), "=m" (*addr)
140 : "r" OPAL_ASM_ADDR(addr), "r" (*oldval), "r" (newval), "m" (*addr)
141 : "cc", "memory");
142
143 ret = (prev == *oldval);
144 *oldval = prev;
145 return ret;
146 }
147
148
149
150
151 #define opal_atomic_ll_32(addr, ret) \
152 do { \
153 opal_atomic_int32_t *_addr = (addr); \
154 int32_t _ret; \
155 __asm__ __volatile__ ("lwarx %0, 0, %1 \n\t" \
156 : "=&r" (_ret) \
157 : "r" (_addr) \
158 ); \
159 ret = (typeof(ret)) _ret; \
160 } while (0)
161
162 #define opal_atomic_sc_32(addr, value, ret) \
163 do { \
164 opal_atomic_int32_t *_addr = (addr); \
165 int32_t _ret, _foo, _newval = (int32_t) value; \
166 \
167 __asm__ __volatile__ (" stwcx. %4, 0, %3 \n\t" \
168 " li %0,0 \n\t" \
169 " bne- 1f \n\t" \
170 " ori %0,%0,1 \n\t" \
171 "1:" \
172 : "=r" (_ret), "=m" (*_addr), "=r" (_foo) \
173 : "r" (_addr), "r" (_newval) \
174 : "cc", "memory"); \
175 ret = _ret; \
176 } while (0)
177
178
179
180
181
182
183 static inline bool opal_atomic_compare_exchange_strong_acq_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
184 {
185 bool rc;
186
187 rc = opal_atomic_compare_exchange_strong_32 (addr, oldval, newval);
188 opal_atomic_rmb();
189
190 return rc;
191 }
192
193
194 static inline bool opal_atomic_compare_exchange_strong_rel_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
195 {
196 opal_atomic_wmb();
197 return opal_atomic_compare_exchange_strong_32 (addr, oldval, newval);
198 }
199
200 static inline int32_t opal_atomic_swap_32(opal_atomic_int32_t *addr, int32_t newval)
201 {
202 int32_t ret;
203
204 __asm__ __volatile__ ("1: lwarx %0, 0, %2 \n\t"
205 " stwcx. %3, 0, %2 \n\t"
206 " bne- 1b \n\t"
207 : "=&r" (ret), "=m" (*addr)
208 : "r" (addr), "r" (newval)
209 : "cc", "memory");
210
211 return ret;
212 }
213
214 #endif
215
216
217 #if (OPAL_ASSEMBLY_ARCH == OPAL_POWERPC64)
218
219 #if OPAL_GCC_INLINE_ASSEMBLY
220
221 #define OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(type, instr) \
222 static inline int64_t opal_atomic_fetch_ ## type ## _64(opal_atomic_int64_t* v, int64_t val) \
223 { \
224 int64_t t, old; \
225 \
226 __asm__ __volatile__( \
227 "1: ldarx %1, 0, %4 \n\t" \
228 " " #instr " %0, %3, %1 \n\t" \
229 " stdcx. %0, 0, %4 \n\t" \
230 " bne- 1b \n\t" \
231 : "=&r" (t), "=&r" (old), "=m" (*v) \
232 : "r" (OPAL_ASM_VALUE64(val)), "r" OPAL_ASM_ADDR(v), "m" (*v) \
233 : "cc"); \
234 \
235 return old; \
236 }
237
238 OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(add, add)
239 OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(and, and)
240 OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(or, or)
241 OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(xor, xor)
242 OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(sub, subf)
243
244 static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
245 {
246 int64_t prev;
247 bool ret;
248
249 __asm__ __volatile__ (
250 "1: ldarx %0, 0, %2 \n\t"
251 " cmpd 0, %0, %3 \n\t"
252 " bne- 2f \n\t"
253 " stdcx. %4, 0, %2 \n\t"
254 " bne- 1b \n\t"
255 "2:"
256 : "=&r" (prev), "=m" (*addr)
257 : "r" (addr), "r" (OPAL_ASM_VALUE64(*oldval)), "r" (OPAL_ASM_VALUE64(newval)), "m" (*addr)
258 : "cc", "memory");
259
260 ret = (prev == *oldval);
261 *oldval = prev;
262 return ret;
263 }
264
265 #define opal_atomic_ll_64(addr, ret) \
266 do { \
267 opal_atomic_int64_t *_addr = (addr); \
268 int64_t _ret; \
269 __asm__ __volatile__ ("ldarx %0, 0, %1 \n\t" \
270 : "=&r" (_ret) \
271 : "r" (_addr) \
272 ); \
273 ret = (typeof(ret)) _ret; \
274 } while (0)
275
276 #define opal_atomic_sc_64(addr, value, ret) \
277 do { \
278 opal_atomic_int64_t *_addr = (addr); \
279 int64_t _newval = (int64_t) value; \
280 int32_t _ret; \
281 \
282 __asm__ __volatile__ (" stdcx. %2, 0, %1 \n\t" \
283 " li %0,0 \n\t" \
284 " bne- 1f \n\t" \
285 " ori %0,%0,1 \n\t" \
286 "1:" \
287 : "=r" (_ret) \
288 : "r" (_addr), "r" (OPAL_ASM_VALUE64(_newval)) \
289 : "cc", "memory"); \
290 ret = _ret; \
291 } while (0)
292
293 static inline int64_t opal_atomic_swap_64(opal_atomic_int64_t *addr, int64_t newval)
294 {
295 int64_t ret;
296
297 __asm__ __volatile__ ("1: ldarx %0, 0, %2 \n\t"
298 " stdcx. %3, 0, %2 \n\t"
299 " bne- 1b \n\t"
300 : "=&r" (ret), "=m" (*addr)
301 : "r" (addr), "r" (OPAL_ASM_VALUE64(newval))
302 : "cc", "memory");
303
304 return ret;
305 }
306
307 #endif
308
309 #elif (OPAL_ASSEMBLY_ARCH == OPAL_POWERPC32) && OPAL_ASM_SUPPORT_64BIT
310
311 #ifndef ll_low
312 #define ll_low(x) *(((unsigned int*)&(x))+0)
313 #define ll_high(x) *(((unsigned int*)&(x))+1)
314 #endif
315
316 #if OPAL_GCC_INLINE_ASSEMBLY
317
318 static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
319 {
320 int64_t prev;
321 int ret;
322
323
324
325
326
327
328
329
330
331
332
333
334 __asm__ __volatile__ (
335 "ld r4,%3 \n\t"
336 "ld r5,%4 \n\t"
337 "1: ldarx %1, 0, %2 \n\t"
338 " cmpd 0, %1, r4 \n\t"
339 " bne- 2f \n\t"
340 " stdcx. r5, 0, %2 \n\t"
341 " bne- 1b \n\t"
342 "2: \n\t"
343 "xor r5,r4,%1 \n\t"
344 "subfic r9,r5,0 \n\t"
345 "adde %0,r9,r5 \n\t"
346 : "=&r" (ret), "+r" (prev)
347 : "r"OPAL_ASM_ADDR(addr),
348 "m"(*oldval), "m"(newval)
349 : "r4", "r5", "r9", "cc", "memory");
350 *oldval = prev;
351 return (bool) ret;
352 }
353
354 #endif
355
356 #endif
357
358 #if OPAL_GCC_INLINE_ASSEMBLY
359
360
361
362
363
364
365 static inline bool opal_atomic_compare_exchange_strong_acq_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
366 {
367 bool rc;
368
369 rc = opal_atomic_compare_exchange_strong_64 (addr, oldval, newval);
370 opal_atomic_rmb();
371
372 return rc;
373 }
374
375
376 static inline bool opal_atomic_compare_exchange_strong_rel_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
377 {
378 opal_atomic_wmb();
379 return opal_atomic_compare_exchange_strong_64 (addr, oldval, newval);
380 }
381
382
383 #define OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_32(type, instr) \
384 static inline int32_t opal_atomic_fetch_ ## type ## _32(opal_atomic_int32_t* v, int val) \
385 { \
386 int32_t t, old; \
387 \
388 __asm__ __volatile__( \
389 "1: lwarx %1, 0, %4 \n\t" \
390 " " #instr " %0, %3, %1 \n\t" \
391 " stwcx. %0, 0, %4 \n\t" \
392 " bne- 1b \n\t" \
393 : "=&r" (t), "=&r" (old), "=m" (*v) \
394 : "r" (val), "r" OPAL_ASM_ADDR(v), "m" (*v) \
395 : "cc"); \
396 \
397 return old; \
398 }
399
400 OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_32(add, add)
401 OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_32(and, and)
402 OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_32(or, or)
403 OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_32(xor, xor)
404 OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_32(sub, subf)
405
406 #endif
407
408 #endif