[444] | 1 | #ifndef _ATOMIC_H
|
---|
| 2 | #define _ATOMIC_H
|
---|
| 3 |
|
---|
| 4 | #include <stdint.h>
|
---|
| 5 |
|
---|
| 6 | #include "atomic_arch.h"
|
---|
| 7 |
|
---|
| 8 | #ifdef a_ll
|
---|
| 9 |
|
---|
| 10 | #ifndef a_pre_llsc
|
---|
| 11 | #define a_pre_llsc()
|
---|
| 12 | #endif
|
---|
| 13 |
|
---|
| 14 | #ifndef a_post_llsc
|
---|
| 15 | #define a_post_llsc()
|
---|
| 16 | #endif
|
---|
| 17 |
|
---|
| 18 | #ifndef a_cas
|
---|
| 19 | #define a_cas a_cas
|
---|
| 20 | static inline int a_cas(volatile int *p, int t, int s)
|
---|
| 21 | {
|
---|
| 22 | int old;
|
---|
| 23 | a_pre_llsc();
|
---|
| 24 | do old = a_ll(p);
|
---|
| 25 | while (old==t && !a_sc(p, s));
|
---|
| 26 | a_post_llsc();
|
---|
| 27 | return old;
|
---|
| 28 | }
|
---|
| 29 | #endif
|
---|
| 30 |
|
---|
| 31 | #ifndef a_swap
|
---|
| 32 | #define a_swap a_swap
|
---|
| 33 | static inline int a_swap(volatile int *p, int v)
|
---|
| 34 | {
|
---|
| 35 | int old;
|
---|
| 36 | a_pre_llsc();
|
---|
| 37 | do old = a_ll(p);
|
---|
| 38 | while (!a_sc(p, v));
|
---|
| 39 | a_post_llsc();
|
---|
| 40 | return old;
|
---|
| 41 | }
|
---|
| 42 | #endif
|
---|
| 43 |
|
---|
| 44 | #ifndef a_fetch_add
|
---|
| 45 | #define a_fetch_add a_fetch_add
|
---|
| 46 | static inline int a_fetch_add(volatile int *p, int v)
|
---|
| 47 | {
|
---|
| 48 | int old;
|
---|
| 49 | a_pre_llsc();
|
---|
| 50 | do old = a_ll(p);
|
---|
| 51 | while (!a_sc(p, (unsigned)old + v));
|
---|
| 52 | a_post_llsc();
|
---|
| 53 | return old;
|
---|
| 54 | }
|
---|
| 55 | #endif
|
---|
| 56 |
|
---|
| 57 | #ifndef a_fetch_and
|
---|
| 58 | #define a_fetch_and a_fetch_and
|
---|
| 59 | static inline int a_fetch_and(volatile int *p, int v)
|
---|
| 60 | {
|
---|
| 61 | int old;
|
---|
| 62 | a_pre_llsc();
|
---|
| 63 | do old = a_ll(p);
|
---|
| 64 | while (!a_sc(p, old & v));
|
---|
| 65 | a_post_llsc();
|
---|
| 66 | return old;
|
---|
| 67 | }
|
---|
| 68 | #endif
|
---|
| 69 |
|
---|
| 70 | #ifndef a_fetch_or
|
---|
| 71 | #define a_fetch_or a_fetch_or
|
---|
| 72 | static inline int a_fetch_or(volatile int *p, int v)
|
---|
| 73 | {
|
---|
| 74 | int old;
|
---|
| 75 | a_pre_llsc();
|
---|
| 76 | do old = a_ll(p);
|
---|
| 77 | while (!a_sc(p, old | v));
|
---|
| 78 | a_post_llsc();
|
---|
| 79 | return old;
|
---|
| 80 | }
|
---|
| 81 | #endif
|
---|
| 82 |
|
---|
| 83 | #endif
|
---|
| 84 |
|
---|
| 85 | #ifdef a_ll_p
|
---|
| 86 |
|
---|
| 87 | #ifndef a_cas_p
|
---|
| 88 | #define a_cas_p a_cas_p
|
---|
| 89 | static inline void *a_cas_p(volatile void *p, void *t, void *s)
|
---|
| 90 | {
|
---|
| 91 | void *old;
|
---|
| 92 | a_pre_llsc();
|
---|
| 93 | do old = a_ll_p(p);
|
---|
| 94 | while (old==t && !a_sc_p(p, s));
|
---|
| 95 | a_post_llsc();
|
---|
| 96 | return old;
|
---|
| 97 | }
|
---|
| 98 | #endif
|
---|
| 99 |
|
---|
| 100 | #endif
|
---|
| 101 |
|
---|
| 102 | #ifndef a_cas
|
---|
| 103 | #error missing definition of a_cas
|
---|
| 104 | #endif
|
---|
| 105 |
|
---|
| 106 | #ifndef a_swap
|
---|
| 107 | #define a_swap a_swap
|
---|
| 108 | static inline int a_swap(volatile int *p, int v)
|
---|
| 109 | {
|
---|
| 110 | int old;
|
---|
| 111 | do old = *p;
|
---|
| 112 | while (a_cas(p, old, v) != old);
|
---|
| 113 | return old;
|
---|
| 114 | }
|
---|
| 115 | #endif
|
---|
| 116 |
|
---|
| 117 | #ifndef a_fetch_add
|
---|
| 118 | #define a_fetch_add a_fetch_add
|
---|
| 119 | static inline int a_fetch_add(volatile int *p, int v)
|
---|
| 120 | {
|
---|
| 121 | int old;
|
---|
| 122 | do old = *p;
|
---|
| 123 | while (a_cas(p, old, (unsigned)old+v) != old);
|
---|
| 124 | return old;
|
---|
| 125 | }
|
---|
| 126 | #endif
|
---|
| 127 |
|
---|
| 128 | #ifndef a_fetch_and
|
---|
| 129 | #define a_fetch_and a_fetch_and
|
---|
| 130 | static inline int a_fetch_and(volatile int *p, int v)
|
---|
| 131 | {
|
---|
| 132 | int old;
|
---|
| 133 | do old = *p;
|
---|
| 134 | while (a_cas(p, old, old&v) != old);
|
---|
| 135 | return old;
|
---|
| 136 | }
|
---|
| 137 | #endif
|
---|
| 138 | #ifndef a_fetch_or
|
---|
| 139 | #define a_fetch_or a_fetch_or
|
---|
| 140 | static inline int a_fetch_or(volatile int *p, int v)
|
---|
| 141 | {
|
---|
| 142 | int old;
|
---|
| 143 | do old = *p;
|
---|
| 144 | while (a_cas(p, old, old|v) != old);
|
---|
| 145 | return old;
|
---|
| 146 | }
|
---|
| 147 | #endif
|
---|
| 148 |
|
---|
| 149 | #ifndef a_and
|
---|
| 150 | #define a_and a_and
|
---|
| 151 | static inline void a_and(volatile int *p, int v)
|
---|
| 152 | {
|
---|
| 153 | a_fetch_and(p, v);
|
---|
| 154 | }
|
---|
| 155 | #endif
|
---|
| 156 |
|
---|
| 157 | #ifndef a_or
|
---|
| 158 | #define a_or a_or
|
---|
| 159 | static inline void a_or(volatile int *p, int v)
|
---|
| 160 | {
|
---|
| 161 | a_fetch_or(p, v);
|
---|
| 162 | }
|
---|
| 163 | #endif
|
---|
| 164 |
|
---|
| 165 | #ifndef a_inc
|
---|
| 166 | #define a_inc a_inc
|
---|
| 167 | static inline void a_inc(volatile int *p)
|
---|
| 168 | {
|
---|
| 169 | a_fetch_add(p, 1);
|
---|
| 170 | }
|
---|
| 171 | #endif
|
---|
| 172 |
|
---|
| 173 | #ifndef a_dec
|
---|
| 174 | #define a_dec a_dec
|
---|
| 175 | static inline void a_dec(volatile int *p)
|
---|
| 176 | {
|
---|
| 177 | a_fetch_add(p, -1);
|
---|
| 178 | }
|
---|
| 179 | #endif
|
---|
| 180 |
|
---|
| 181 | #ifndef a_store
|
---|
| 182 | #define a_store a_store
|
---|
| 183 | static inline void a_store(volatile int *p, int v)
|
---|
| 184 | {
|
---|
| 185 | #ifdef a_barrier
|
---|
| 186 | a_barrier();
|
---|
| 187 | *p = v;
|
---|
| 188 | a_barrier();
|
---|
| 189 | #else
|
---|
| 190 | a_swap(p, v);
|
---|
| 191 | #endif
|
---|
| 192 | }
|
---|
| 193 | #endif
|
---|
| 194 |
|
---|
| 195 | #ifndef a_barrier
|
---|
| 196 | #define a_barrier a_barrier
|
---|
| 197 | static inline void a_barrier()
|
---|
| 198 | {
|
---|
| 199 | volatile int tmp = 0;
|
---|
| 200 | a_cas(&tmp, 0, 0);
|
---|
| 201 | }
|
---|
| 202 | #endif
|
---|
| 203 |
|
---|
| 204 | #ifndef a_spin
|
---|
| 205 | #define a_spin a_barrier
|
---|
| 206 | #endif
|
---|
| 207 |
|
---|
| 208 | #ifndef a_and_64
|
---|
| 209 | #define a_and_64 a_and_64
|
---|
| 210 | static inline void a_and_64(volatile uint64_t *p, uint64_t v)
|
---|
| 211 | {
|
---|
| 212 | union { uint64_t v; uint32_t r[2]; } u = { v };
|
---|
| 213 | if (u.r[0]+1) a_and((int *)p, u.r[0]);
|
---|
| 214 | if (u.r[1]+1) a_and((int *)p+1, u.r[1]);
|
---|
| 215 | }
|
---|
| 216 | #endif
|
---|
| 217 |
|
---|
| 218 | #ifndef a_or_64
|
---|
| 219 | #define a_or_64 a_or_64
|
---|
| 220 | static inline void a_or_64(volatile uint64_t *p, uint64_t v)
|
---|
| 221 | {
|
---|
| 222 | union { uint64_t v; uint32_t r[2]; } u = { v };
|
---|
| 223 | if (u.r[0]) a_or((int *)p, u.r[0]);
|
---|
| 224 | if (u.r[1]) a_or((int *)p+1, u.r[1]);
|
---|
| 225 | }
|
---|
| 226 | #endif
|
---|
| 227 |
|
---|
| 228 | #ifndef a_cas_p
|
---|
| 229 | typedef char a_cas_p_undefined_but_pointer_not_32bit[-sizeof(char) == 0xffffffff ? 1 : -1];
|
---|
| 230 | #define a_cas_p a_cas_p
|
---|
| 231 | static inline void *a_cas_p(volatile void *p, void *t, void *s)
|
---|
| 232 | {
|
---|
| 233 | return (void *)a_cas((volatile int *)p, (int)t, (int)s);
|
---|
| 234 | }
|
---|
| 235 | #endif
|
---|
| 236 |
|
---|
| 237 | #ifndef a_or_l
|
---|
| 238 | #define a_or_l a_or_l
|
---|
| 239 | static inline void a_or_l(volatile void *p, long v)
|
---|
| 240 | {
|
---|
| 241 | if (sizeof(long) == sizeof(int)) a_or(p, v);
|
---|
| 242 | else a_or_64(p, v);
|
---|
| 243 | }
|
---|
| 244 | #endif
|
---|
| 245 |
|
---|
| 246 | #ifndef a_crash
|
---|
| 247 | #define a_crash a_crash
|
---|
| 248 | static inline void a_crash()
|
---|
| 249 | {
|
---|
| 250 | *(volatile char *)0=0;
|
---|
| 251 | }
|
---|
| 252 | #endif
|
---|
| 253 |
|
---|
| 254 | #ifndef a_ctz_64
|
---|
| 255 | #define a_ctz_64 a_ctz_64
|
---|
| 256 | static inline int a_ctz_64(uint64_t x)
|
---|
| 257 | {
|
---|
| 258 | static const char debruijn64[64] = {
|
---|
| 259 | 0, 1, 2, 53, 3, 7, 54, 27, 4, 38, 41, 8, 34, 55, 48, 28,
|
---|
| 260 | 62, 5, 39, 46, 44, 42, 22, 9, 24, 35, 59, 56, 49, 18, 29, 11,
|
---|
| 261 | 63, 52, 6, 26, 37, 40, 33, 47, 61, 45, 43, 21, 23, 58, 17, 10,
|
---|
| 262 | 51, 25, 36, 32, 60, 20, 57, 16, 50, 31, 19, 15, 30, 14, 13, 12
|
---|
| 263 | };
|
---|
| 264 | static const char debruijn32[32] = {
|
---|
| 265 | 0, 1, 23, 2, 29, 24, 19, 3, 30, 27, 25, 11, 20, 8, 4, 13,
|
---|
| 266 | 31, 22, 28, 18, 26, 10, 7, 12, 21, 17, 9, 6, 16, 5, 15, 14
|
---|
| 267 | };
|
---|
| 268 | if (sizeof(long) < 8) {
|
---|
| 269 | uint32_t y = x;
|
---|
| 270 | if (!y) {
|
---|
| 271 | y = x>>32;
|
---|
| 272 | return 32 + debruijn32[(y&-y)*0x076be629 >> 27];
|
---|
| 273 | }
|
---|
| 274 | return debruijn32[(y&-y)*0x076be629 >> 27];
|
---|
| 275 | }
|
---|
| 276 | return debruijn64[(x&-x)*0x022fdd63cc95386dull >> 58];
|
---|
| 277 | }
|
---|
| 278 | #endif
|
---|
| 279 |
|
---|
| 280 | #ifndef a_clz_64
|
---|
| 281 | #define a_clz_64 a_clz_64
|
---|
| 282 | static inline int a_clz_64(uint64_t x)
|
---|
| 283 | {
|
---|
| 284 | #ifdef a_clz_32
|
---|
| 285 | if (x>>32)
|
---|
| 286 | return a_clz_32(x>>32);
|
---|
| 287 | return a_clz_32(x) + 32;
|
---|
| 288 | #else
|
---|
| 289 | uint32_t y;
|
---|
| 290 | int r;
|
---|
| 291 | if (x>>32) y=x>>32, r=0; else y=x, r=32;
|
---|
| 292 | if (y>>16) y>>=16; else r |= 16;
|
---|
| 293 | if (y>>8) y>>=8; else r |= 8;
|
---|
| 294 | if (y>>4) y>>=4; else r |= 4;
|
---|
| 295 | if (y>>2) y>>=2; else r |= 2;
|
---|
| 296 | return r | !(y>>1);
|
---|
| 297 | #endif
|
---|
| 298 | }
|
---|
| 299 | #endif
|
---|
| 300 |
|
---|
| 301 | #ifndef a_ctz_l
|
---|
| 302 | #define a_ctz_l a_ctz_l
|
---|
| 303 | static inline int a_ctz_l(unsigned long x)
|
---|
| 304 | {
|
---|
| 305 | static const char debruijn32[32] = {
|
---|
| 306 | 0, 1, 23, 2, 29, 24, 19, 3, 30, 27, 25, 11, 20, 8, 4, 13,
|
---|
| 307 | 31, 22, 28, 18, 26, 10, 7, 12, 21, 17, 9, 6, 16, 5, 15, 14
|
---|
| 308 | };
|
---|
| 309 | if (sizeof(long) == 8) return a_ctz_64(x);
|
---|
| 310 | return debruijn32[(x&-x)*0x076be629 >> 27];
|
---|
| 311 | }
|
---|
| 312 | #endif
|
---|
| 313 |
|
---|
| 314 | #endif
|
---|