This source file includes following definitions.
- opal_atomic_mb
- opal_atomic_rmb
- opal_atomic_wmb
- opal_atomic_isync
- opal_atomic_compare_exchange_strong_32
- opal_atomic_swap_32
- opal_atomic_compare_exchange_strong_acq_32
- opal_atomic_compare_exchange_strong_rel_32
- opal_atomic_compare_exchange_strong_64
- opal_atomic_swap_64
- opal_atomic_compare_exchange_strong_acq_64
- opal_atomic_compare_exchange_strong_rel_64
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24 #if !defined(OPAL_SYS_ARCH_ATOMIC_H)
25
26 #define OPAL_SYS_ARCH_ATOMIC_H 1
27
28 #if OPAL_GCC_INLINE_ASSEMBLY
29
30 #define OPAL_HAVE_ATOMIC_MEM_BARRIER 1
31 #define OPAL_HAVE_ATOMIC_LLSC_32 1
32 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
33 #define OPAL_HAVE_ATOMIC_SWAP_32 1
34 #define OPAL_HAVE_ATOMIC_MATH_32 1
35 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1
36 #define OPAL_HAVE_ATOMIC_SWAP_64 1
37 #define OPAL_HAVE_ATOMIC_LLSC_64 1
38 #define OPAL_HAVE_ATOMIC_ADD_32 1
39 #define OPAL_HAVE_ATOMIC_AND_32 1
40 #define OPAL_HAVE_ATOMIC_OR_32 1
41 #define OPAL_HAVE_ATOMIC_XOR_32 1
42 #define OPAL_HAVE_ATOMIC_SUB_32 1
43 #define OPAL_HAVE_ATOMIC_ADD_64 1
44 #define OPAL_HAVE_ATOMIC_AND_64 1
45 #define OPAL_HAVE_ATOMIC_OR_64 1
46 #define OPAL_HAVE_ATOMIC_XOR_64 1
47 #define OPAL_HAVE_ATOMIC_SUB_64 1
48
49 #define MB() __asm__ __volatile__ ("dmb sy" : : : "memory")
50 #define RMB() __asm__ __volatile__ ("dmb ld" : : : "memory")
51 #define WMB() __asm__ __volatile__ ("dmb st" : : : "memory")
52
53
54
55
56
57
58
59 static inline void opal_atomic_mb (void)
60 {
61 MB();
62 }
63
64 static inline void opal_atomic_rmb (void)
65 {
66 RMB();
67 }
68
69 static inline void opal_atomic_wmb (void)
70 {
71 WMB();
72 }
73
74 static inline void opal_atomic_isync (void)
75 {
76 __asm__ __volatile__ ("isb");
77 }
78
79
80
81
82
83
84
85 static inline bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
86 {
87 int32_t prev, tmp;
88 bool ret;
89
90 __asm__ __volatile__ ("1: ldaxr %w0, [%2] \n"
91 " cmp %w0, %w3 \n"
92 " bne 2f \n"
93 " stxr %w1, %w4, [%2] \n"
94 " cbnz %w1, 1b \n"
95 "2: \n"
96 : "=&r" (prev), "=&r" (tmp)
97 : "r" (addr), "r" (*oldval), "r" (newval)
98 : "cc", "memory");
99
100 ret = (prev == *oldval);
101 *oldval = prev;
102 return ret;
103 }
104
105 static inline int32_t opal_atomic_swap_32(opal_atomic_int32_t *addr, int32_t newval)
106 {
107 int32_t ret, tmp;
108
109 __asm__ __volatile__ ("1: ldaxr %w0, [%2] \n"
110 " stlxr %w1, %w3, [%2] \n"
111 " cbnz %w1, 1b \n"
112 : "=&r" (ret), "=&r" (tmp)
113 : "r" (addr), "r" (newval)
114 : "cc", "memory");
115
116 return ret;
117 }
118
119
120
121
122
123
124 static inline bool opal_atomic_compare_exchange_strong_acq_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
125 {
126 int32_t prev, tmp;
127 bool ret;
128
129 __asm__ __volatile__ ("1: ldaxr %w0, [%2] \n"
130 " cmp %w0, %w3 \n"
131 " bne 2f \n"
132 " stxr %w1, %w4, [%2] \n"
133 " cbnz %w1, 1b \n"
134 "2: \n"
135 : "=&r" (prev), "=&r" (tmp)
136 : "r" (addr), "r" (*oldval), "r" (newval)
137 : "cc", "memory");
138
139 ret = (prev == *oldval);
140 *oldval = prev;
141 return ret;
142 }
143
144
145 static inline bool opal_atomic_compare_exchange_strong_rel_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
146 {
147 int32_t prev, tmp;
148 bool ret;
149
150 __asm__ __volatile__ ("1: ldxr %w0, [%2] \n"
151 " cmp %w0, %w3 \n"
152 " bne 2f \n"
153 " stlxr %w1, %w4, [%2] \n"
154 " cbnz %w1, 1b \n"
155 "2: \n"
156 : "=&r" (prev), "=&r" (tmp)
157 : "r" (addr), "r" (*oldval), "r" (newval)
158 : "cc", "memory");
159
160 ret = (prev == *oldval);
161 *oldval = prev;
162 return ret;
163 }
164
165 #define opal_atomic_ll_32(addr, ret) \
166 do { \
167 opal_atomic_int32_t *_addr = (addr); \
168 int32_t _ret; \
169 \
170 __asm__ __volatile__ ("ldaxr %w0, [%1] \n" \
171 : "=&r" (_ret) \
172 : "r" (_addr)); \
173 \
174 ret = (typeof(ret)) _ret; \
175 } while (0)
176
177 #define opal_atomic_sc_32(addr, newval, ret) \
178 do { \
179 opal_atomic_int32_t *_addr = (addr); \
180 int32_t _newval = (int32_t) newval; \
181 int _ret; \
182 \
183 __asm__ __volatile__ ("stlxr %w0, %w2, [%1] \n" \
184 : "=&r" (_ret) \
185 : "r" (_addr), "r" (_newval) \
186 : "cc", "memory"); \
187 \
188 ret = (_ret == 0); \
189 } while (0)
190
191 static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
192 {
193 int64_t prev;
194 int tmp;
195 bool ret;
196
197 __asm__ __volatile__ ("1: ldaxr %0, [%2] \n"
198 " cmp %0, %3 \n"
199 " bne 2f \n"
200 " stxr %w1, %4, [%2] \n"
201 " cbnz %w1, 1b \n"
202 "2: \n"
203 : "=&r" (prev), "=&r" (tmp)
204 : "r" (addr), "r" (*oldval), "r" (newval)
205 : "cc", "memory");
206
207 ret = (prev == *oldval);
208 *oldval = prev;
209 return ret;
210 }
211
212 static inline int64_t opal_atomic_swap_64 (opal_atomic_int64_t *addr, int64_t newval)
213 {
214 int64_t ret;
215 int tmp;
216
217 __asm__ __volatile__ ("1: ldaxr %0, [%2] \n"
218 " stlxr %w1, %3, [%2] \n"
219 " cbnz %w1, 1b \n"
220 : "=&r" (ret), "=&r" (tmp)
221 : "r" (addr), "r" (newval)
222 : "cc", "memory");
223
224 return ret;
225 }
226
227
228
229
230
231
232 static inline bool opal_atomic_compare_exchange_strong_acq_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
233 {
234 int64_t prev;
235 int tmp;
236 bool ret;
237
238 __asm__ __volatile__ ("1: ldaxr %0, [%2] \n"
239 " cmp %0, %3 \n"
240 " bne 2f \n"
241 " stxr %w1, %4, [%2] \n"
242 " cbnz %w1, 1b \n"
243 "2: \n"
244 : "=&r" (prev), "=&r" (tmp)
245 : "r" (addr), "r" (*oldval), "r" (newval)
246 : "cc", "memory");
247
248 ret = (prev == *oldval);
249 *oldval = prev;
250 return ret;
251 }
252
253
254 static inline bool opal_atomic_compare_exchange_strong_rel_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
255 {
256 int64_t prev;
257 int tmp;
258 bool ret;
259
260 __asm__ __volatile__ ("1: ldxr %0, [%2] \n"
261 " cmp %0, %3 \n"
262 " bne 2f \n"
263 " stlxr %w1, %4, [%2] \n"
264 " cbnz %w1, 1b \n"
265 "2: \n"
266 : "=&r" (prev), "=&r" (tmp)
267 : "r" (addr), "r" (*oldval), "r" (newval)
268 : "cc", "memory");
269
270 ret = (prev == *oldval);
271 *oldval = prev;
272 return ret;
273 }
274
275 #define opal_atomic_ll_64(addr, ret) \
276 do { \
277 opal_atomic_int64_t *_addr = (addr); \
278 int64_t _ret; \
279 \
280 __asm__ __volatile__ ("ldaxr %0, [%1] \n" \
281 : "=&r" (_ret) \
282 : "r" (_addr)); \
283 \
284 ret = (typeof(ret)) _ret; \
285 } while (0)
286
287 #define opal_atomic_sc_64(addr, newval, ret) \
288 do { \
289 opal_atomic_int64_t *_addr = (addr); \
290 int64_t _newval = (int64_t) newval; \
291 int _ret; \
292 \
293 __asm__ __volatile__ ("stlxr %w0, %2, [%1] \n" \
294 : "=&r" (_ret) \
295 : "r" (_addr), "r" (_newval) \
296 : "cc", "memory"); \
297 \
298 ret = (_ret == 0); \
299 } while (0)
300
301 #define OPAL_ASM_MAKE_ATOMIC(type, bits, name, inst, reg) \
302 static inline type opal_atomic_fetch_ ## name ## _ ## bits (opal_atomic_ ## type *addr, type value) \
303 { \
304 type newval, old; \
305 int32_t tmp; \
306 \
307 __asm__ __volatile__("1: ldxr %" reg "1, [%3] \n" \
308 " " inst " %" reg "0, %" reg "1, %" reg "4 \n" \
309 " stxr %w2, %" reg "0, [%3] \n" \
310 " cbnz %w2, 1b \n" \
311 : "=&r" (newval), "=&r" (old), "=&r" (tmp) \
312 : "r" (addr), "r" (value) \
313 : "cc", "memory"); \
314 \
315 return old; \
316 }
317
318 OPAL_ASM_MAKE_ATOMIC(int32_t, 32, add, "add", "w")
319 OPAL_ASM_MAKE_ATOMIC(int32_t, 32, and, "and", "w")
320 OPAL_ASM_MAKE_ATOMIC(int32_t, 32, or, "orr", "w")
321 OPAL_ASM_MAKE_ATOMIC(int32_t, 32, xor, "eor", "w")
322 OPAL_ASM_MAKE_ATOMIC(int32_t, 32, sub, "sub", "w")
323 OPAL_ASM_MAKE_ATOMIC(int64_t, 64, add, "add", "")
324 OPAL_ASM_MAKE_ATOMIC(int64_t, 64, and, "and", "")
325 OPAL_ASM_MAKE_ATOMIC(int64_t, 64, or, "orr", "")
326 OPAL_ASM_MAKE_ATOMIC(int64_t, 64, xor, "eor", "")
327 OPAL_ASM_MAKE_ATOMIC(int64_t, 64, sub, "sub", "")
328
329 #endif
330
331 #endif