This source file includes following definitions.
- opal_atomic_mb
- opal_atomic_rmb
- opal_atomic_wmb
- opal_atomic_isync
- opal_atomic_compare_exchange_strong_32
- opal_atomic_compare_exchange_strong_64
- opal_atomic_compare_exchange_strong_128
- opal_atomic_swap_32
- opal_atomic_swap_64
- opal_atomic_fetch_add_32
- opal_atomic_fetch_add_64
- opal_atomic_fetch_sub_32
- opal_atomic_fetch_sub_64
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24 #ifndef OPAL_SYS_ARCH_ATOMIC_H
25 #define OPAL_SYS_ARCH_ATOMIC_H 1
26
27
28
29
30
31
32 #define SMPLOCK "lock; "
33 #define MB() __asm__ __volatile__("": : :"memory")
34
35
36
37
38
39
40
41 #define OPAL_HAVE_ATOMIC_MEM_BARRIER 1
42
43 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
44
45 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1
46
47
48
49
50
51
52 #if OPAL_GCC_INLINE_ASSEMBLY
53
54 static inline void opal_atomic_mb(void)
55 {
56 MB();
57 }
58
59
60 static inline void opal_atomic_rmb(void)
61 {
62 MB();
63 }
64
65
66 static inline void opal_atomic_wmb(void)
67 {
68 MB();
69 }
70
71 static inline void opal_atomic_isync(void)
72 {
73 }
74
75 #endif
76
77
78
79
80
81
82
83 #if OPAL_GCC_INLINE_ASSEMBLY
84
85 static inline bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
86 {
87 unsigned char ret;
88 __asm__ __volatile__ (
89 SMPLOCK "cmpxchgl %3,%2 \n\t"
90 "sete %0 \n\t"
91 : "=qm" (ret), "+a" (*oldval), "+m" (*addr)
92 : "q"(newval)
93 : "memory", "cc");
94
95 return (bool) ret;
96 }
97
98 #endif
99
100 #define opal_atomic_compare_exchange_strong_acq_32 opal_atomic_compare_exchange_strong_32
101 #define opal_atomic_compare_exchange_strong_rel_32 opal_atomic_compare_exchange_strong_32
102
103 #if OPAL_GCC_INLINE_ASSEMBLY
104
105 static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
106 {
107 unsigned char ret;
108 __asm__ __volatile__ (
109 SMPLOCK "cmpxchgq %3,%2 \n\t"
110 "sete %0 \n\t"
111 : "=qm" (ret), "+a" (*oldval), "+m" (*((opal_atomic_long_t *)addr))
112 : "q"(newval)
113 : "memory", "cc"
114 );
115
116 return (bool) ret;
117 }
118
119 #endif
120
121 #define opal_atomic_compare_exchange_strong_acq_64 opal_atomic_compare_exchange_strong_64
122 #define opal_atomic_compare_exchange_strong_rel_64 opal_atomic_compare_exchange_strong_64
123
124 #if OPAL_GCC_INLINE_ASSEMBLY && OPAL_HAVE_CMPXCHG16B && HAVE_OPAL_INT128_T
125
126 static inline bool opal_atomic_compare_exchange_strong_128 (opal_atomic_int128_t *addr, opal_int128_t *oldval, opal_int128_t newval)
127 {
128 unsigned char ret;
129
130
131
132
133 __asm__ __volatile__ (SMPLOCK "cmpxchg16b (%%rsi) \n\t"
134 "sete %0 \n\t"
135 : "=qm" (ret), "+a" (((int64_t *)oldval)[0]), "+d" (((int64_t *)oldval)[1])
136 : "S" (addr), "b" (((int64_t *)&newval)[0]), "c" (((int64_t *)&newval)[1])
137 : "memory", "cc");
138
139 return (bool) ret;
140 }
141
142 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1
143
144 #endif
145
146
147 #if OPAL_GCC_INLINE_ASSEMBLY
148
149 #define OPAL_HAVE_ATOMIC_SWAP_32 1
150
151 #define OPAL_HAVE_ATOMIC_SWAP_64 1
152
153 static inline int32_t opal_atomic_swap_32( opal_atomic_int32_t *addr,
154 int32_t newval)
155 {
156 int32_t oldval;
157
158 __asm__ __volatile__("xchg %1, %0" :
159 "=r" (oldval), "+m" (*addr) :
160 "0" (newval) :
161 "memory");
162 return oldval;
163 }
164
165 #endif
166
167 #if OPAL_GCC_INLINE_ASSEMBLY
168
169 static inline int64_t opal_atomic_swap_64( opal_atomic_int64_t *addr,
170 int64_t newval)
171 {
172 int64_t oldval;
173
174 __asm__ __volatile__("xchgq %1, %0" :
175 "=r" (oldval), "+m" (*addr) :
176 "0" (newval) :
177 "memory");
178 return oldval;
179 }
180
181 #endif
182
183
184
185 #if OPAL_GCC_INLINE_ASSEMBLY
186
187 #define OPAL_HAVE_ATOMIC_MATH_32 1
188 #define OPAL_HAVE_ATOMIC_MATH_64 1
189
190 #define OPAL_HAVE_ATOMIC_ADD_32 1
191
192
193
194
195
196
197
198
199 static inline int32_t opal_atomic_fetch_add_32(opal_atomic_int32_t* v, int i)
200 {
201 int ret = i;
202 __asm__ __volatile__(
203 SMPLOCK "xaddl %1,%0"
204 :"+m" (*v), "+r" (ret)
205 :
206 :"memory", "cc"
207 );
208 return ret;
209 }
210
211 #define OPAL_HAVE_ATOMIC_ADD_64 1
212
213
214
215
216
217
218
219
220 static inline int64_t opal_atomic_fetch_add_64(opal_atomic_int64_t* v, int64_t i)
221 {
222 int64_t ret = i;
223 __asm__ __volatile__(
224 SMPLOCK "xaddq %1,%0"
225 :"+m" (*v), "+r" (ret)
226 :
227 :"memory", "cc"
228 );
229 return ret;
230 }
231
232 #define OPAL_HAVE_ATOMIC_SUB_32 1
233
234
235
236
237
238
239
240
241 static inline int32_t opal_atomic_fetch_sub_32(opal_atomic_int32_t* v, int i)
242 {
243 int ret = -i;
244 __asm__ __volatile__(
245 SMPLOCK "xaddl %1,%0"
246 :"+m" (*v), "+r" (ret)
247 :
248 :"memory", "cc"
249 );
250 return ret;
251 }
252
253 #define OPAL_HAVE_ATOMIC_SUB_64 1
254
255
256
257
258
259
260
261
262 static inline int64_t opal_atomic_fetch_sub_64(opal_atomic_int64_t* v, int64_t i)
263 {
264 int64_t ret = -i;
265 __asm__ __volatile__(
266 SMPLOCK "xaddq %1,%0"
267 :"+m" (*v), "+r" (ret)
268 :
269 :"memory", "cc"
270 );
271 return ret;
272 }
273
274 #endif
275
276 #endif