1 | #ifndef _ATOMIC_H
|
---|
2 | #define _ATOMIC_H
|
---|
3 |
|
---|
4 | #include <stdint.h>
|
---|
5 |
|
---|
6 | #include "atomic_arch.h"
|
---|
7 |
|
---|
8 | #ifdef a_ll
|
---|
9 |
|
---|
10 | #ifndef a_pre_llsc
|
---|
11 | #define a_pre_llsc()
|
---|
12 | #endif
|
---|
13 |
|
---|
14 | #ifndef a_post_llsc
|
---|
15 | #define a_post_llsc()
|
---|
16 | #endif
|
---|
17 |
|
---|
18 | #ifndef a_cas
|
---|
19 | #define a_cas a_cas
|
---|
20 | static inline int a_cas(volatile int *p, int t, int s)
|
---|
21 | {
|
---|
22 | int old;
|
---|
23 | a_pre_llsc();
|
---|
24 | do old = a_ll(p);
|
---|
25 | while (old==t && !a_sc(p, s));
|
---|
26 | a_post_llsc();
|
---|
27 | return old;
|
---|
28 | }
|
---|
29 | #endif
|
---|
30 |
|
---|
31 | #ifndef a_swap
|
---|
32 | #define a_swap a_swap
|
---|
33 | static inline int a_swap(volatile int *p, int v)
|
---|
34 | {
|
---|
35 | int old;
|
---|
36 | a_pre_llsc();
|
---|
37 | do old = a_ll(p);
|
---|
38 | while (!a_sc(p, v));
|
---|
39 | a_post_llsc();
|
---|
40 | return old;
|
---|
41 | }
|
---|
42 | #endif
|
---|
43 |
|
---|
44 | #ifndef a_fetch_add
|
---|
45 | #define a_fetch_add a_fetch_add
|
---|
46 | static inline int a_fetch_add(volatile int *p, int v)
|
---|
47 | {
|
---|
48 | int old;
|
---|
49 | a_pre_llsc();
|
---|
50 | do old = a_ll(p);
|
---|
51 | while (!a_sc(p, (unsigned)old + v));
|
---|
52 | a_post_llsc();
|
---|
53 | return old;
|
---|
54 | }
|
---|
55 | #endif
|
---|
56 |
|
---|
57 | #ifndef a_fetch_and
|
---|
58 | #define a_fetch_and a_fetch_and
|
---|
59 | static inline int a_fetch_and(volatile int *p, int v)
|
---|
60 | {
|
---|
61 | int old;
|
---|
62 | a_pre_llsc();
|
---|
63 | do old = a_ll(p);
|
---|
64 | while (!a_sc(p, old & v));
|
---|
65 | a_post_llsc();
|
---|
66 | return old;
|
---|
67 | }
|
---|
68 | #endif
|
---|
69 |
|
---|
70 | #ifndef a_fetch_or
|
---|
71 | #define a_fetch_or a_fetch_or
|
---|
72 | static inline int a_fetch_or(volatile int *p, int v)
|
---|
73 | {
|
---|
74 | int old;
|
---|
75 | a_pre_llsc();
|
---|
76 | do old = a_ll(p);
|
---|
77 | while (!a_sc(p, old | v));
|
---|
78 | a_post_llsc();
|
---|
79 | return old;
|
---|
80 | }
|
---|
81 | #endif
|
---|
82 |
|
---|
83 | #endif
|
---|
84 |
|
---|
85 | #ifdef a_ll_p
|
---|
86 |
|
---|
87 | #ifndef a_cas_p
|
---|
88 | #define a_cas_p a_cas_p
|
---|
89 | static inline void *a_cas_p(volatile void *p, void *t, void *s)
|
---|
90 | {
|
---|
91 | void *old;
|
---|
92 | a_pre_llsc();
|
---|
93 | do old = a_ll_p(p);
|
---|
94 | while (old==t && !a_sc_p(p, s));
|
---|
95 | a_post_llsc();
|
---|
96 | return old;
|
---|
97 | }
|
---|
98 | #endif
|
---|
99 |
|
---|
100 | #endif
|
---|
101 |
|
---|
102 | #ifndef a_cas
|
---|
103 | #error missing definition of a_cas
|
---|
104 | #endif
|
---|
105 |
|
---|
106 | #ifndef a_swap
|
---|
107 | #define a_swap a_swap
|
---|
108 | static inline int a_swap(volatile int *p, int v)
|
---|
109 | {
|
---|
110 | int old;
|
---|
111 | do old = *p;
|
---|
112 | while (a_cas(p, old, v) != old);
|
---|
113 | return old;
|
---|
114 | }
|
---|
115 | #endif
|
---|
116 |
|
---|
117 | #ifndef a_fetch_add
|
---|
118 | #define a_fetch_add a_fetch_add
|
---|
119 | static inline int a_fetch_add(volatile int *p, int v)
|
---|
120 | {
|
---|
121 | int old;
|
---|
122 | do old = *p;
|
---|
123 | while (a_cas(p, old, (unsigned)old+v) != old);
|
---|
124 | return old;
|
---|
125 | }
|
---|
126 | #endif
|
---|
127 |
|
---|
128 | #ifndef a_fetch_and
|
---|
129 | #define a_fetch_and a_fetch_and
|
---|
130 | static inline int a_fetch_and(volatile int *p, int v)
|
---|
131 | {
|
---|
132 | int old;
|
---|
133 | do old = *p;
|
---|
134 | while (a_cas(p, old, old&v) != old);
|
---|
135 | return old;
|
---|
136 | }
|
---|
137 | #endif
|
---|
138 | #ifndef a_fetch_or
|
---|
139 | #define a_fetch_or a_fetch_or
|
---|
140 | static inline int a_fetch_or(volatile int *p, int v)
|
---|
141 | {
|
---|
142 | int old;
|
---|
143 | do old = *p;
|
---|
144 | while (a_cas(p, old, old|v) != old);
|
---|
145 | return old;
|
---|
146 | }
|
---|
147 | #endif
|
---|
148 |
|
---|
149 | #ifndef a_and
|
---|
150 | #define a_and a_and
|
---|
151 | static inline void a_and(volatile int *p, int v)
|
---|
152 | {
|
---|
153 | a_fetch_and(p, v);
|
---|
154 | }
|
---|
155 | #endif
|
---|
156 |
|
---|
157 | #ifndef a_or
|
---|
158 | #define a_or a_or
|
---|
159 | static inline void a_or(volatile int *p, int v)
|
---|
160 | {
|
---|
161 | a_fetch_or(p, v);
|
---|
162 | }
|
---|
163 | #endif
|
---|
164 |
|
---|
165 | #ifndef a_inc
|
---|
166 | #define a_inc a_inc
|
---|
167 | static inline void a_inc(volatile int *p)
|
---|
168 | {
|
---|
169 | a_fetch_add(p, 1);
|
---|
170 | }
|
---|
171 | #endif
|
---|
172 |
|
---|
173 | #ifndef a_dec
|
---|
174 | #define a_dec a_dec
|
---|
175 | static inline void a_dec(volatile int *p)
|
---|
176 | {
|
---|
177 | a_fetch_add(p, -1);
|
---|
178 | }
|
---|
179 | #endif
|
---|
180 |
|
---|
181 | #ifndef a_store
|
---|
182 | #define a_store a_store
|
---|
183 | static inline void a_store(volatile int *p, int v)
|
---|
184 | {
|
---|
185 | #ifdef a_barrier
|
---|
186 | a_barrier();
|
---|
187 | *p = v;
|
---|
188 | a_barrier();
|
---|
189 | #else
|
---|
190 | a_swap(p, v);
|
---|
191 | #endif
|
---|
192 | }
|
---|
193 | #endif
|
---|
194 |
|
---|
195 | #ifndef a_barrier
|
---|
196 | #define a_barrier a_barrier
|
---|
197 | static inline void a_barrier()
|
---|
198 | {
|
---|
199 | volatile int tmp = 0;
|
---|
200 | a_cas(&tmp, 0, 0);
|
---|
201 | }
|
---|
202 | #endif
|
---|
203 |
|
---|
204 | #ifndef a_spin
|
---|
205 | #define a_spin a_barrier
|
---|
206 | #endif
|
---|
207 |
|
---|
208 | #ifndef a_and_64
|
---|
209 | #define a_and_64 a_and_64
|
---|
210 | static inline void a_and_64(volatile uint64_t *p, uint64_t v)
|
---|
211 | {
|
---|
212 | union { uint64_t v; uint32_t r[2]; } u = { v };
|
---|
213 | if (u.r[0]+1) a_and((int *)p, u.r[0]);
|
---|
214 | if (u.r[1]+1) a_and((int *)p+1, u.r[1]);
|
---|
215 | }
|
---|
216 | #endif
|
---|
217 |
|
---|
218 | #ifndef a_or_64
|
---|
219 | #define a_or_64 a_or_64
|
---|
220 | static inline void a_or_64(volatile uint64_t *p, uint64_t v)
|
---|
221 | {
|
---|
222 | union { uint64_t v; uint32_t r[2]; } u = { v };
|
---|
223 | if (u.r[0]) a_or((int *)p, u.r[0]);
|
---|
224 | if (u.r[1]) a_or((int *)p+1, u.r[1]);
|
---|
225 | }
|
---|
226 | #endif
|
---|
227 |
|
---|
228 | #ifndef a_cas_p
|
---|
229 | typedef char a_cas_p_undefined_but_pointer_not_32bit[-sizeof(char) == 0xffffffff ? 1 : -1];
|
---|
230 | #define a_cas_p a_cas_p
|
---|
231 | static inline void *a_cas_p(volatile void *p, void *t, void *s)
|
---|
232 | {
|
---|
233 | return (void *)a_cas((volatile int *)p, (int)t, (int)s);
|
---|
234 | }
|
---|
235 | #endif
|
---|
236 |
|
---|
237 | #ifndef a_or_l
|
---|
238 | #define a_or_l a_or_l
|
---|
239 | static inline void a_or_l(volatile void *p, long v)
|
---|
240 | {
|
---|
241 | if (sizeof(long) == sizeof(int)) a_or(p, v);
|
---|
242 | else a_or_64(p, v);
|
---|
243 | }
|
---|
244 | #endif
|
---|
245 |
|
---|
246 | #ifndef a_crash
|
---|
247 | #define a_crash a_crash
|
---|
248 | static inline void a_crash()
|
---|
249 | {
|
---|
250 | *(volatile char *)0=0;
|
---|
251 | }
|
---|
252 | #endif
|
---|
253 |
|
---|
254 | #ifndef a_ctz_64
|
---|
255 | #define a_ctz_64 a_ctz_64
|
---|
256 | static inline int a_ctz_64(uint64_t x)
|
---|
257 | {
|
---|
258 | static const char debruijn64[64] = {
|
---|
259 | 0, 1, 2, 53, 3, 7, 54, 27, 4, 38, 41, 8, 34, 55, 48, 28,
|
---|
260 | 62, 5, 39, 46, 44, 42, 22, 9, 24, 35, 59, 56, 49, 18, 29, 11,
|
---|
261 | 63, 52, 6, 26, 37, 40, 33, 47, 61, 45, 43, 21, 23, 58, 17, 10,
|
---|
262 | 51, 25, 36, 32, 60, 20, 57, 16, 50, 31, 19, 15, 30, 14, 13, 12
|
---|
263 | };
|
---|
264 | static const char debruijn32[32] = {
|
---|
265 | 0, 1, 23, 2, 29, 24, 19, 3, 30, 27, 25, 11, 20, 8, 4, 13,
|
---|
266 | 31, 22, 28, 18, 26, 10, 7, 12, 21, 17, 9, 6, 16, 5, 15, 14
|
---|
267 | };
|
---|
268 | if (sizeof(long) < 8) {
|
---|
269 | uint32_t y = x;
|
---|
270 | if (!y) {
|
---|
271 | y = x>>32;
|
---|
272 | return 32 + debruijn32[(y&-y)*0x076be629 >> 27];
|
---|
273 | }
|
---|
274 | return debruijn32[(y&-y)*0x076be629 >> 27];
|
---|
275 | }
|
---|
276 | return debruijn64[(x&-x)*0x022fdd63cc95386dull >> 58];
|
---|
277 | }
|
---|
278 | #endif
|
---|
279 |
|
---|
280 | #ifndef a_clz_64
|
---|
281 | #define a_clz_64 a_clz_64
|
---|
282 | static inline int a_clz_64(uint64_t x)
|
---|
283 | {
|
---|
284 | #ifdef a_clz_32
|
---|
285 | if (x>>32)
|
---|
286 | return a_clz_32(x>>32);
|
---|
287 | return a_clz_32(x) + 32;
|
---|
288 | #else
|
---|
289 | uint32_t y;
|
---|
290 | int r;
|
---|
291 | if (x>>32) y=x>>32, r=0; else y=x, r=32;
|
---|
292 | if (y>>16) y>>=16; else r |= 16;
|
---|
293 | if (y>>8) y>>=8; else r |= 8;
|
---|
294 | if (y>>4) y>>=4; else r |= 4;
|
---|
295 | if (y>>2) y>>=2; else r |= 2;
|
---|
296 | return r | !(y>>1);
|
---|
297 | #endif
|
---|
298 | }
|
---|
299 | #endif
|
---|
300 |
|
---|
301 | #ifndef a_ctz_l
|
---|
302 | #define a_ctz_l a_ctz_l
|
---|
303 | static inline int a_ctz_l(unsigned long x)
|
---|
304 | {
|
---|
305 | static const char debruijn32[32] = {
|
---|
306 | 0, 1, 23, 2, 29, 24, 19, 3, 30, 27, 25, 11, 20, 8, 4, 13,
|
---|
307 | 31, 22, 28, 18, 26, 10, 7, 12, 21, 17, 9, 6, 16, 5, 15, 14
|
---|
308 | };
|
---|
309 | if (sizeof(long) == 8) return a_ctz_64(x);
|
---|
310 | return debruijn32[(x&-x)*0x076be629 >> 27];
|
---|
311 | }
|
---|
312 | #endif
|
---|
313 |
|
---|
314 | #endif
|
---|