1 /* $OpenBSD: atomic.h,v 1.23 2024/01/16 23:38:13 jsg Exp $ */
2 /**
3 * \file drm_atomic.h
4 * Atomic operations used in the DRM which may or may not be provided by the OS.
5 *
6 * \author Eric Anholt <anholt@FreeBSD.org>
7 */
8
9 /*-
10 * Copyright 2004 Eric Anholt
11 * All Rights Reserved.
12 *
13 * Permission is hereby granted, free of charge, to any person obtaining a
14 * copy of this software and associated documentation files (the "Software"),
15 * to deal in the Software without restriction, including without limitation
16 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
17 * and/or sell copies of the Software, and to permit persons to whom the
18 * Software is furnished to do so, subject to the following conditions:
19 *
20 * The above copyright notice and this permission notice (including the next
21 * paragraph) shall be included in all copies or substantial portions of the
22 * Software.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
25 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
26 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
27 * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
28 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
29 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
30 * OTHER DEALINGS IN THE SOFTWARE.
31 */
32
33 #ifndef _DRM_LINUX_ATOMIC_H_
34 #define _DRM_LINUX_ATOMIC_H_
35
36 #include <sys/types.h>
37 #include <sys/mutex.h>
38 #include <machine/intr.h>
39 #include <linux/types.h>
40 #include <linux/compiler.h> /* via x86/include/asm/atomic.h */
41
42 #define ATOMIC_INIT(x) (x)
43
44 #define atomic_set(p, v) WRITE_ONCE(*(p), (v))
45 #define atomic_read(p) READ_ONCE(*(p))
46 #define atomic_inc(p) __sync_fetch_and_add(p, 1)
47 #define atomic_dec(p) __sync_fetch_and_sub(p, 1)
48 #define atomic_add(n, p) __sync_fetch_and_add(p, n)
49 #define atomic_sub(n, p) __sync_fetch_and_sub(p, n)
50 #define atomic_and(n, p) __sync_fetch_and_and(p, n)
51 #define atomic_or(n, p) atomic_setbits_int(p, n)
52 #define atomic_add_return(n, p) __sync_add_and_fetch(p, n)
53 #define atomic_sub_return(n, p) __sync_sub_and_fetch(p, n)
54 #define atomic_sub_and_test(n, p) (atomic_sub_return(n, p) == 0)
55 #define atomic_inc_return(v) atomic_add_return(1, (v))
56 #define atomic_dec_return(v) atomic_sub_return(1, (v))
57 #define atomic_dec_and_test(v) (atomic_dec_return(v) == 0)
58 #define atomic_inc_and_test(v) (atomic_inc_return(v) == 0)
59 #define atomic_cmpxchg(p, o, n) __sync_val_compare_and_swap(p, o, n)
60 #define cmpxchg(p, o, n) __sync_val_compare_and_swap(p, o, n)
61 #define cmpxchg64(p, o, n) __sync_val_compare_and_swap(p, o, n)
62 #define atomic_set_release(p, v) atomic_set((p), (v))
63 #define atomic_andnot(bits, p) atomic_clearbits_int(p,bits)
64 #define atomic_fetch_inc(p) __sync_fetch_and_add(p, 1)
65 #define atomic_fetch_xor(n, p) __sync_fetch_and_xor(p, n)
66
67 #define try_cmpxchg(p, op, n) \
68 ({ \
69 __typeof(p) __op = (__typeof((p)))(op); \
70 __typeof(*(p)) __o = *__op; \
71 __typeof(*(p)) __p = __sync_val_compare_and_swap((p), (__o), (n)); \
72 if (__p != __o) \
73 *__op = __p; \
74 (__p == __o); \
75 })
76
77 static inline bool
atomic_try_cmpxchg(volatile int * p,int * op,int n)78 atomic_try_cmpxchg(volatile int *p, int *op, int n)
79 {
80 return try_cmpxchg(p, op, n);
81 }
82
83 static inline int
atomic_xchg(volatile int * v,int n)84 atomic_xchg(volatile int *v, int n)
85 {
86 __sync_synchronize();
87 return __sync_lock_test_and_set(v, n);
88 }
89
90 #define xchg(v, n) __sync_lock_test_and_set(v, n)
91
92 static inline int
atomic_add_unless(volatile int * v,int n,int u)93 atomic_add_unless(volatile int *v, int n, int u)
94 {
95 int o;
96
97 do {
98 o = *v;
99 if (o == u)
100 return 0;
101 } while (__sync_val_compare_and_swap(v, o, o +n) != o);
102
103 return 1;
104 }
105
106 #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
107
108 static inline int
atomic_dec_if_positive(volatile int * v)109 atomic_dec_if_positive(volatile int *v)
110 {
111 int r, o;
112
113 do {
114 o = *v;
115 r = o - 1;
116 if (r < 0)
117 break;
118 } while (__sync_val_compare_and_swap(v, o, r) != o);
119
120 return r;
121 }
122
123 #define atomic_long_read(p) READ_ONCE(*(p))
124
125 /* 32 bit powerpc lacks 64 bit atomics */
126 #if !defined(__powerpc__) || defined(__powerpc64__)
127
128 typedef int64_t atomic64_t;
129
130 #define ATOMIC64_INIT(x) (x)
131
132 #define atomic64_set(p, v) WRITE_ONCE(*(p), (v))
133 #define atomic64_read(p) READ_ONCE(*(p))
134
135 static inline int64_t
atomic64_xchg(volatile int64_t * v,int64_t n)136 atomic64_xchg(volatile int64_t *v, int64_t n)
137 {
138 __sync_synchronize();
139 return __sync_lock_test_and_set(v, n);
140 }
141
142 static inline int64_t
atomic64_cmpxchg(volatile int64_t * v,int64_t o,int64_t n)143 atomic64_cmpxchg(volatile int64_t *v, int64_t o, int64_t n)
144 {
145 return __sync_val_compare_and_swap(v, o, n);
146 }
147
148 #define atomic64_add(n, p) __sync_fetch_and_add_8(p, n)
149 #define atomic64_sub(n, p) __sync_fetch_and_sub_8(p, n)
150 #define atomic64_inc(p) __sync_fetch_and_add_8(p, 1)
151 #define atomic64_add_return(n, p) __sync_add_and_fetch_8(p, n)
152 #define atomic64_inc_return(p) __sync_add_and_fetch_8(p, 1)
153
154 #else
155
156 extern struct mutex atomic64_mtx;
157
158 typedef struct {
159 volatile int64_t val;
160 } atomic64_t;
161
162 #define ATOMIC64_INIT(x) { (x) }
163
164 static inline void
atomic64_set(atomic64_t * v,int64_t i)165 atomic64_set(atomic64_t *v, int64_t i)
166 {
167 mtx_enter(&atomic64_mtx);
168 v->val = i;
169 mtx_leave(&atomic64_mtx);
170 }
171
172 static inline int64_t
atomic64_read(atomic64_t * v)173 atomic64_read(atomic64_t *v)
174 {
175 int64_t val;
176
177 mtx_enter(&atomic64_mtx);
178 val = v->val;
179 mtx_leave(&atomic64_mtx);
180
181 return val;
182 }
183
184 static inline int64_t
atomic64_xchg(atomic64_t * v,int64_t n)185 atomic64_xchg(atomic64_t *v, int64_t n)
186 {
187 int64_t val;
188
189 mtx_enter(&atomic64_mtx);
190 val = v->val;
191 v->val = n;
192 mtx_leave(&atomic64_mtx);
193
194 return val;
195 }
196
197 static inline void
atomic64_add(int i,atomic64_t * v)198 atomic64_add(int i, atomic64_t *v)
199 {
200 mtx_enter(&atomic64_mtx);
201 v->val += i;
202 mtx_leave(&atomic64_mtx);
203 }
204
205 #define atomic64_inc(p) atomic64_add(p, 1)
206
207 static inline int64_t
atomic64_add_return(int i,atomic64_t * v)208 atomic64_add_return(int i, atomic64_t *v)
209 {
210 int64_t val;
211
212 mtx_enter(&atomic64_mtx);
213 val = v->val + i;
214 v->val = val;
215 mtx_leave(&atomic64_mtx);
216
217 return val;
218 }
219
220 #define atomic64_inc_return(p) atomic64_add_return(1, p)
221
222 static inline void
atomic64_sub(int i,atomic64_t * v)223 atomic64_sub(int i, atomic64_t *v)
224 {
225 mtx_enter(&atomic64_mtx);
226 v->val -= i;
227 mtx_leave(&atomic64_mtx);
228 }
229 #endif
230
231 #ifdef __LP64__
232 typedef int64_t atomic_long_t;
233 #define atomic_long_set(p, v) atomic64_set(p, v)
234 #define atomic_long_xchg(v, n) atomic64_xchg(v, n)
235 #define atomic_long_cmpxchg(p, o, n) atomic_cmpxchg(p, o, n)
236 #define atomic_long_add(i, v) atomic64_add(i, v)
237 #define atomic_long_sub(i, v) atomic64_sub(i, v)
238 #else
239 typedef int32_t atomic_long_t;
240 #define atomic_long_set(p, v) atomic_set(p, v)
241 #define atomic_long_xchg(v, n) atomic_xchg(v, n)
242 #define atomic_long_cmpxchg(p, o, n) atomic_cmpxchg(p, o, n)
243 #define atomic_long_add(i, v) atomic_add(i, v)
244 #define atomic_long_sub(i, v) atomic_sub(i, v)
245 #endif
246
247 static inline atomic_t
test_and_set_bit(u_int b,volatile void * p)248 test_and_set_bit(u_int b, volatile void *p)
249 {
250 unsigned int m = 1 << (b & 0x1f);
251 unsigned int prev = __sync_fetch_and_or((volatile u_int *)p + (b >> 5), m);
252 return (prev & m) != 0;
253 }
254
255 static inline void
clear_bit(u_int b,volatile void * p)256 clear_bit(u_int b, volatile void *p)
257 {
258 atomic_clearbits_int(((volatile u_int *)p) + (b >> 5), 1 << (b & 0x1f));
259 }
260
261 static inline void
clear_bit_unlock(u_int b,volatile void * p)262 clear_bit_unlock(u_int b, volatile void *p)
263 {
264 membar_enter();
265 clear_bit(b, p);
266 }
267
268 static inline void
set_bit(u_int b,volatile void * p)269 set_bit(u_int b, volatile void *p)
270 {
271 atomic_setbits_int(((volatile u_int *)p) + (b >> 5), 1 << (b & 0x1f));
272 }
273
274 static inline void
__clear_bit(u_int b,volatile void * p)275 __clear_bit(u_int b, volatile void *p)
276 {
277 volatile u_int *ptr = (volatile u_int *)p;
278 ptr[b >> 5] &= ~(1 << (b & 0x1f));
279 }
280
281 static inline void
__set_bit(u_int b,volatile void * p)282 __set_bit(u_int b, volatile void *p)
283 {
284 volatile u_int *ptr = (volatile u_int *)p;
285 ptr[b >> 5] |= (1 << (b & 0x1f));
286 }
287
288 static inline int
test_bit(u_int b,const volatile void * p)289 test_bit(u_int b, const volatile void *p)
290 {
291 return !!(((volatile u_int *)p)[b >> 5] & (1 << (b & 0x1f)));
292 }
293
294 static inline int
__test_and_set_bit(u_int b,volatile void * p)295 __test_and_set_bit(u_int b, volatile void *p)
296 {
297 unsigned int m = 1 << (b & 0x1f);
298 volatile u_int *ptr = (volatile u_int *)p;
299 unsigned int prev = ptr[b >> 5];
300 ptr[b >> 5] |= m;
301
302 return (prev & m) != 0;
303 }
304
305 static inline int
test_and_clear_bit(u_int b,volatile void * p)306 test_and_clear_bit(u_int b, volatile void *p)
307 {
308 unsigned int m = 1 << (b & 0x1f);
309 unsigned int prev = __sync_fetch_and_and((volatile u_int *)p + (b >> 5), ~m);
310 return (prev & m) != 0;
311 }
312
313 static inline int
__test_and_clear_bit(u_int b,volatile void * p)314 __test_and_clear_bit(u_int b, volatile void *p)
315 {
316 volatile u_int *ptr = (volatile u_int *)p;
317 int rv = !!(ptr[b >> 5] & (1 << (b & 0x1f)));
318 ptr[b >> 5] &= ~(1 << (b & 0x1f));
319 return rv;
320 }
321
322 static inline int
find_first_zero_bit(volatile void * p,int max)323 find_first_zero_bit(volatile void *p, int max)
324 {
325 int b;
326 volatile u_int *ptr = (volatile u_int *)p;
327
328 for (b = 0; b < max; b += 32) {
329 if (ptr[b >> 5] != ~0) {
330 for (;;) {
331 if ((ptr[b >> 5] & (1 << (b & 0x1f))) == 0)
332 return b;
333 b++;
334 }
335 }
336 }
337 return max;
338 }
339
340 static inline int
find_next_zero_bit(volatile void * p,int max,int b)341 find_next_zero_bit(volatile void *p, int max, int b)
342 {
343 volatile u_int *ptr = (volatile u_int *)p;
344
345 for (; b < max; b += 32) {
346 if (ptr[b >> 5] != ~0) {
347 for (;;) {
348 if ((ptr[b >> 5] & (1 << (b & 0x1f))) == 0)
349 return b;
350 b++;
351 }
352 }
353 }
354 return max;
355 }
356
357 static inline int
find_first_bit(volatile void * p,int max)358 find_first_bit(volatile void *p, int max)
359 {
360 int b;
361 volatile u_int *ptr = (volatile u_int *)p;
362
363 for (b = 0; b < max; b += 32) {
364 if (ptr[b >> 5] != 0) {
365 for (;;) {
366 if (ptr[b >> 5] & (1 << (b & 0x1f)))
367 return b;
368 b++;
369 }
370 }
371 }
372 return max;
373 }
374
375 static inline int
find_next_bit(const volatile void * p,int max,int b)376 find_next_bit(const volatile void *p, int max, int b)
377 {
378 volatile u_int *ptr = (volatile u_int *)p;
379
380 for (; b < max; b+= 32) {
381 if (ptr[b >> 5] != 0) {
382 for (;;) {
383 if (ptr[b >> 5] & (1 << (b & 0x1f)))
384 return b;
385 b++;
386 }
387 }
388 }
389 return max;
390 }
391
392 #define for_each_set_bit(b, p, max) \
393 for ((b) = find_first_bit((p), (max)); \
394 (b) < (max); \
395 (b) = find_next_bit((p), (max), (b) + 1))
396
397 #define for_each_clear_bit(b, p, max) \
398 for ((b) = find_first_zero_bit((p), (max)); \
399 (b) < (max); \
400 (b) = find_next_zero_bit((p), (max), (b) + 1))
401
402 #if defined(__i386__)
403 #define rmb() __asm volatile("lock; addl $0,-4(%%esp)" : : : "memory", "cc")
404 #define wmb() __asm volatile("lock; addl $0,-4(%%esp)" : : : "memory", "cc")
405 #define mb() __asm volatile("lock; addl $0,-4(%%esp)" : : : "memory", "cc")
406 #define smp_mb() __asm volatile("lock; addl $0,-4(%%esp)" : : : "memory", "cc")
407 #define smp_rmb() __membar("")
408 #define smp_wmb() __membar("")
409 #define __smp_store_mb(var, value) do { (void)xchg(&var, value); } while (0)
410 #define smp_mb__after_atomic() do { } while (0)
411 #define smp_mb__before_atomic() do { } while (0)
412 #elif defined(__amd64__)
413 #define rmb() __membar("lfence")
414 #define wmb() __membar("sfence")
415 #define mb() __membar("mfence")
416 #define smp_mb() __asm volatile("lock; addl $0,-4(%%rsp)" : : : "memory", "cc")
417 #define smp_rmb() __membar("")
418 #define smp_wmb() __membar("")
419 #define __smp_store_mb(var, value) do { (void)xchg(&var, value); } while (0)
420 #define smp_mb__after_atomic() do { } while (0)
421 #define smp_mb__before_atomic() do { } while (0)
422 #elif defined(__aarch64__)
423 #define rmb() __membar("dsb ld")
424 #define wmb() __membar("dsb st")
425 #define mb() __membar("dsb sy")
426 #define dma_rmb() __membar("dmb oshld")
427 #define dma_wmb() __membar("dmb oshst")
428 #define dma_mb() __membar("dmb osh")
429 #define smp_mb() __membar("dmb ish")
430 #elif defined(__arm__)
431 #define rmb() __membar("dsb sy")
432 #define wmb() __membar("dsb sy")
433 #define mb() __membar("dsb sy")
434 #elif defined(__mips64__)
435 #define rmb() mips_sync()
436 #define wmb() mips_sync()
437 #define mb() mips_sync()
438 #elif defined(__powerpc64__)
439 #define rmb() __membar("sync")
440 #define wmb() __membar("sync")
441 #define mb() __membar("sync")
442 #define smp_rmb() __membar("lwsync")
443 #define smp_wmb() __membar("lwsync")
444 #define smp_mb() __membar("sync")
445 #elif defined(__powerpc__)
446 #define rmb() __membar("sync")
447 #define wmb() __membar("sync")
448 #define mb() __membar("sync")
449 #define smp_wmb() __membar("eieio")
450 #elif defined(__riscv)
451 #define rmb() __membar("fence ir,ir")
452 #define wmb() __membar("fence ow,ow")
453 #define mb() __membar("fence iorw,iorw")
454 #define smp_rmb() __membar("fence r,r")
455 #define smp_wmb() __membar("fence w,w")
456 #define smp_mb() __membar("fence rw,rw")
457 #elif defined(__sparc64__)
458 #define rmb() membar_sync()
459 #define wmb() membar_sync()
460 #define mb() membar_sync()
461 #endif
462
463 #ifndef smp_rmb
464 #define smp_rmb() rmb()
465 #endif
466
467 #ifndef smp_wmb
468 #define smp_wmb() wmb()
469 #endif
470
471 #ifndef mmiowb
472 #define mmiowb() wmb()
473 #endif
474
475 #ifndef smp_mb__before_atomic
476 #define smp_mb__before_atomic() mb()
477 #endif
478
479 #ifndef smp_mb__after_atomic
480 #define smp_mb__after_atomic() mb()
481 #endif
482
483 #ifndef smp_store_mb
484 #define smp_store_mb(x, v) do { x = v; mb(); } while (0)
485 #endif
486
487 #ifndef smp_store_release
488 #define smp_store_release(x, v) do { smp_mb(); WRITE_ONCE(*x, v); } while(0)
489 #endif
490
491 #ifndef smp_load_acquire
492 #define smp_load_acquire(x) \
493 ({ \
494 __typeof(*x) _v = READ_ONCE(*x); \
495 smp_mb(); \
496 _v; \
497 })
498 #endif
499
500 #endif
501