1 /*-
2 * Copyright (c) 2015-2024 Ruslan Bukin <br@bsdpad.com>
3 * All rights reserved.
4 *
5 * Portions of this software were developed by SRI International and the
6 * University of Cambridge Computer Laboratory under DARPA/AFRL contract
7 * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme.
8 *
9 * Portions of this software were developed by the University of Cambridge
10 * Computer Laboratory as part of the CTSRD Project, with support from the
11 * UK Higher Education Innovation Fund (HEIF).
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 * 1. Redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer.
18 * 2. Redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 */
34
35 #ifndef _MACHINE_ATOMIC_H_
36 #define _MACHINE_ATOMIC_H_
37
38 #include <sys/atomic_common.h>
39
40 #define fence() __asm __volatile("fence" ::: "memory");
41 #define mb() fence()
42 #define rmb() fence()
43 #define wmb() fence()
44
45 static __inline int atomic_cmpset_8(__volatile uint8_t *, uint8_t, uint8_t);
46 static __inline int atomic_fcmpset_8(__volatile uint8_t *, uint8_t *, uint8_t);
47 static __inline int atomic_cmpset_16(__volatile uint16_t *, uint16_t, uint16_t);
48 static __inline int atomic_fcmpset_16(__volatile uint16_t *, uint16_t *,
49 uint16_t);
50
51 #define ATOMIC_ACQ_REL(NAME, WIDTH) \
52 static __inline void \
53 atomic_##NAME##_acq_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\
54 { \
55 atomic_##NAME##_##WIDTH(p, v); \
56 fence(); \
57 } \
58 \
59 static __inline void \
60 atomic_##NAME##_rel_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\
61 { \
62 fence(); \
63 atomic_##NAME##_##WIDTH(p, v); \
64 }
65
66 #define ATOMIC_CMPSET_ACQ_REL(WIDTH) \
67 static __inline int \
68 atomic_cmpset_acq_##WIDTH(__volatile uint##WIDTH##_t *p, \
69 uint##WIDTH##_t cmpval, uint##WIDTH##_t newval) \
70 { \
71 int retval; \
72 \
73 retval = atomic_cmpset_##WIDTH(p, cmpval, newval); \
74 fence(); \
75 return (retval); \
76 } \
77 \
78 static __inline int \
79 atomic_cmpset_rel_##WIDTH(__volatile uint##WIDTH##_t *p, \
80 uint##WIDTH##_t cmpval, uint##WIDTH##_t newval) \
81 { \
82 fence(); \
83 return (atomic_cmpset_##WIDTH(p, cmpval, newval)); \
84 }
85
86 #define ATOMIC_FCMPSET_ACQ_REL(WIDTH) \
87 static __inline int \
88 atomic_fcmpset_acq_##WIDTH(__volatile uint##WIDTH##_t *p, \
89 uint##WIDTH##_t *cmpval, uint##WIDTH##_t newval) \
90 { \
91 int retval; \
92 \
93 retval = atomic_fcmpset_##WIDTH(p, cmpval, newval); \
94 fence(); \
95 return (retval); \
96 } \
97 \
98 static __inline int \
99 atomic_fcmpset_rel_##WIDTH(__volatile uint##WIDTH##_t *p, \
100 uint##WIDTH##_t *cmpval, uint##WIDTH##_t newval) \
101 { \
102 fence(); \
103 return (atomic_fcmpset_##WIDTH(p, cmpval, newval)); \
104 }
105
106 ATOMIC_CMPSET_ACQ_REL(8);
107 ATOMIC_FCMPSET_ACQ_REL(8);
108
109 #define atomic_cmpset_char atomic_cmpset_8
110 #define atomic_cmpset_acq_char atomic_cmpset_acq_8
111 #define atomic_cmpset_rel_char atomic_cmpset_rel_8
112 #define atomic_fcmpset_char atomic_fcmpset_8
113 #define atomic_fcmpset_acq_char atomic_fcmpset_acq_8
114 #define atomic_fcmpset_rel_char atomic_fcmpset_rel_8
115
116 #define atomic_cmpset_short atomic_cmpset_16
117 #define atomic_fcmpset_short atomic_fcmpset_16
118
119 ATOMIC_CMPSET_ACQ_REL(16);
120 ATOMIC_FCMPSET_ACQ_REL(16);
121
122 #define atomic_load_acq_16 atomic_load_acq_16
123 static __inline uint16_t
atomic_load_acq_16(volatile uint16_t * p)124 atomic_load_acq_16(volatile uint16_t *p)
125 {
126 uint16_t ret;
127
128 ret = *p;
129
130 fence();
131
132 return (ret);
133 }
134
135 static __inline void
atomic_store_rel_16(volatile uint16_t * p,uint16_t val)136 atomic_store_rel_16(volatile uint16_t *p, uint16_t val)
137 {
138
139 fence();
140
141 *p = val;
142 }
143
144 #define atomic_cmpset_acq_short atomic_cmpset_acq_16
145 #define atomic_fcmpset_acq_short atomic_fcmpset_acq_16
146 #define atomic_load_acq_short atomic_load_acq_16
147
148 #define atomic_cmpset_rel_short atomic_cmpset_rel_16
149 #define atomic_fcmpset_rel_short atomic_fcmpset_rel_16
150 #define atomic_store_rel_short atomic_store_rel_16
151
152 static __inline void
atomic_add_32(volatile uint32_t * p,uint32_t val)153 atomic_add_32(volatile uint32_t *p, uint32_t val)
154 {
155
156 __asm __volatile("amoadd.w zero, %1, %0"
157 : "+A" (*p)
158 : "r" (val)
159 : "memory");
160 }
161
162 static __inline void
atomic_subtract_32(volatile uint32_t * p,uint32_t val)163 atomic_subtract_32(volatile uint32_t *p, uint32_t val)
164 {
165
166 __asm __volatile("amoadd.w zero, %1, %0"
167 : "+A" (*p)
168 : "r" (-val)
169 : "memory");
170 }
171
172 static __inline void
atomic_set_32(volatile uint32_t * p,uint32_t val)173 atomic_set_32(volatile uint32_t *p, uint32_t val)
174 {
175
176 __asm __volatile("amoor.w zero, %1, %0"
177 : "+A" (*p)
178 : "r" (val)
179 : "memory");
180 }
181
182 static __inline void
atomic_clear_32(volatile uint32_t * p,uint32_t val)183 atomic_clear_32(volatile uint32_t *p, uint32_t val)
184 {
185
186 __asm __volatile("amoand.w zero, %1, %0"
187 : "+A" (*p)
188 : "r" (~val)
189 : "memory");
190 }
191
192 static __inline int
atomic_cmpset_32(volatile uint32_t * p,uint32_t cmpval,uint32_t newval)193 atomic_cmpset_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval)
194 {
195 uint32_t tmp;
196 int res;
197
198 res = 0;
199
200 __asm __volatile(
201 "0:"
202 "li %1, 1\n" /* Preset to fail */
203 "lr.w %0, %2\n"
204 "bne %0, %z3, 1f\n"
205 "sc.w %1, %z4, %2\n"
206 "bnez %1, 0b\n"
207 "1:"
208 : "=&r" (tmp), "=&r" (res), "+A" (*p)
209 : "rJ" ((long)(int32_t)cmpval), "rJ" (newval)
210 : "memory");
211
212 return (!res);
213 }
214
215 static __inline int
atomic_fcmpset_32(volatile uint32_t * p,uint32_t * cmpval,uint32_t newval)216 atomic_fcmpset_32(volatile uint32_t *p, uint32_t *cmpval, uint32_t newval)
217 {
218 uint32_t tmp;
219 int res;
220
221 res = 0;
222
223 __asm __volatile(
224 "0:"
225 "li %1, 1\n" /* Preset to fail */
226 "lr.w %0, %2\n" /* Load old value */
227 "bne %0, %z4, 1f\n" /* Compare */
228 "sc.w %1, %z5, %2\n" /* Try to store new value */
229 "j 2f\n"
230 "1:"
231 "sw %0, %3\n" /* Save old value */
232 "2:"
233 : "=&r" (tmp), "=&r" (res), "+A" (*p), "+A" (*cmpval)
234 : "rJ" ((long)(int32_t)*cmpval), "rJ" (newval)
235 : "memory");
236
237 return (!res);
238 }
239
240 static __inline uint32_t
atomic_fetchadd_32(volatile uint32_t * p,uint32_t val)241 atomic_fetchadd_32(volatile uint32_t *p, uint32_t val)
242 {
243 uint32_t ret;
244
245 __asm __volatile("amoadd.w %0, %2, %1"
246 : "=&r" (ret), "+A" (*p)
247 : "r" (val)
248 : "memory");
249
250 return (ret);
251 }
252
253 static __inline uint32_t
atomic_readandclear_32(volatile uint32_t * p)254 atomic_readandclear_32(volatile uint32_t *p)
255 {
256 uint32_t ret;
257 uint32_t val;
258
259 val = 0;
260
261 __asm __volatile("amoswap.w %0, %2, %1"
262 : "=&r"(ret), "+A" (*p)
263 : "r" (val)
264 : "memory");
265
266 return (ret);
267 }
268
269 #define atomic_add_int atomic_add_32
270 #define atomic_clear_int atomic_clear_32
271 #define atomic_cmpset_int atomic_cmpset_32
272 #define atomic_fcmpset_int atomic_fcmpset_32
273 #define atomic_fetchadd_int atomic_fetchadd_32
274 #define atomic_readandclear_int atomic_readandclear_32
275 #define atomic_set_int atomic_set_32
276 #define atomic_subtract_int atomic_subtract_32
277
278 ATOMIC_ACQ_REL(set, 32)
279 ATOMIC_ACQ_REL(clear, 32)
280 ATOMIC_ACQ_REL(add, 32)
281 ATOMIC_ACQ_REL(subtract, 32)
282
283 ATOMIC_CMPSET_ACQ_REL(32);
284 ATOMIC_FCMPSET_ACQ_REL(32);
285
286 static __inline uint32_t
atomic_load_acq_32(volatile uint32_t * p)287 atomic_load_acq_32(volatile uint32_t *p)
288 {
289 uint32_t ret;
290
291 ret = *p;
292
293 fence();
294
295 return (ret);
296 }
297
298 static __inline void
atomic_store_rel_32(volatile uint32_t * p,uint32_t val)299 atomic_store_rel_32(volatile uint32_t *p, uint32_t val)
300 {
301
302 fence();
303
304 *p = val;
305 }
306
307 #define atomic_add_acq_int atomic_add_acq_32
308 #define atomic_clear_acq_int atomic_clear_acq_32
309 #define atomic_cmpset_acq_int atomic_cmpset_acq_32
310 #define atomic_fcmpset_acq_int atomic_fcmpset_acq_32
311 #define atomic_load_acq_int atomic_load_acq_32
312 #define atomic_set_acq_int atomic_set_acq_32
313 #define atomic_subtract_acq_int atomic_subtract_acq_32
314
315 #define atomic_add_rel_int atomic_add_rel_32
316 #define atomic_clear_rel_int atomic_clear_rel_32
317 #define atomic_cmpset_rel_int atomic_cmpset_rel_32
318 #define atomic_fcmpset_rel_int atomic_fcmpset_rel_32
319 #define atomic_set_rel_int atomic_set_rel_32
320 #define atomic_subtract_rel_int atomic_subtract_rel_32
321 #define atomic_store_rel_int atomic_store_rel_32
322
323 static __inline void
atomic_add_64(volatile uint64_t * p,uint64_t val)324 atomic_add_64(volatile uint64_t *p, uint64_t val)
325 {
326
327 __asm __volatile("amoadd.d zero, %1, %0"
328 : "+A" (*p)
329 : "r" (val)
330 : "memory");
331 }
332
333 static __inline void
atomic_subtract_64(volatile uint64_t * p,uint64_t val)334 atomic_subtract_64(volatile uint64_t *p, uint64_t val)
335 {
336
337 __asm __volatile("amoadd.d zero, %1, %0"
338 : "+A" (*p)
339 : "r" (-val)
340 : "memory");
341 }
342
343 static __inline void
atomic_set_64(volatile uint64_t * p,uint64_t val)344 atomic_set_64(volatile uint64_t *p, uint64_t val)
345 {
346
347 __asm __volatile("amoor.d zero, %1, %0"
348 : "+A" (*p)
349 : "r" (val)
350 : "memory");
351 }
352
353 static __inline void
atomic_clear_64(volatile uint64_t * p,uint64_t val)354 atomic_clear_64(volatile uint64_t *p, uint64_t val)
355 {
356
357 __asm __volatile("amoand.d zero, %1, %0"
358 : "+A" (*p)
359 : "r" (~val)
360 : "memory");
361 }
362
363 static __inline int
atomic_cmpset_64(volatile uint64_t * p,uint64_t cmpval,uint64_t newval)364 atomic_cmpset_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval)
365 {
366 uint64_t tmp;
367 int res;
368
369 res = 0;
370
371 __asm __volatile(
372 "0:"
373 "li %1, 1\n" /* Preset to fail */
374 "lr.d %0, %2\n"
375 "bne %0, %z3, 1f\n"
376 "sc.d %1, %z4, %2\n"
377 "bnez %1, 0b\n"
378 "1:"
379 : "=&r" (tmp), "=&r" (res), "+A" (*p)
380 : "rJ" (cmpval), "rJ" (newval)
381 : "memory");
382
383 return (!res);
384 }
385
386 static __inline int
atomic_fcmpset_64(volatile uint64_t * p,uint64_t * cmpval,uint64_t newval)387 atomic_fcmpset_64(volatile uint64_t *p, uint64_t *cmpval, uint64_t newval)
388 {
389 uint64_t tmp;
390 int res;
391
392 res = 0;
393
394 __asm __volatile(
395 "0:"
396 "li %1, 1\n" /* Preset to fail */
397 "lr.d %0, %2\n" /* Load old value */
398 "bne %0, %z4, 1f\n" /* Compare */
399 "sc.d %1, %z5, %2\n" /* Try to store new value */
400 "j 2f\n"
401 "1:"
402 "sd %0, %3\n" /* Save old value */
403 "2:"
404 : "=&r" (tmp), "=&r" (res), "+A" (*p), "+A" (*cmpval)
405 : "rJ" (*cmpval), "rJ" (newval)
406 : "memory");
407
408 return (!res);
409 }
410
411 static __inline uint64_t
atomic_fetchadd_64(volatile uint64_t * p,uint64_t val)412 atomic_fetchadd_64(volatile uint64_t *p, uint64_t val)
413 {
414 uint64_t ret;
415
416 __asm __volatile("amoadd.d %0, %2, %1"
417 : "=&r" (ret), "+A" (*p)
418 : "r" (val)
419 : "memory");
420
421 return (ret);
422 }
423
424 static __inline uint64_t
atomic_readandclear_64(volatile uint64_t * p)425 atomic_readandclear_64(volatile uint64_t *p)
426 {
427 uint64_t ret;
428 uint64_t val;
429
430 val = 0;
431
432 __asm __volatile("amoswap.d %0, %2, %1"
433 : "=&r"(ret), "+A" (*p)
434 : "r" (val)
435 : "memory");
436
437 return (ret);
438 }
439
440 static __inline uint32_t
atomic_swap_32(volatile uint32_t * p,uint32_t val)441 atomic_swap_32(volatile uint32_t *p, uint32_t val)
442 {
443 uint32_t old;
444
445 __asm __volatile("amoswap.w %0, %2, %1"
446 : "=&r"(old), "+A" (*p)
447 : "r" (val)
448 : "memory");
449
450 return (old);
451 }
452
453 static __inline uint64_t
atomic_swap_64(volatile uint64_t * p,uint64_t val)454 atomic_swap_64(volatile uint64_t *p, uint64_t val)
455 {
456 uint64_t old;
457
458 __asm __volatile("amoswap.d %0, %2, %1"
459 : "=&r"(old), "+A" (*p)
460 : "r" (val)
461 : "memory");
462
463 return (old);
464 }
465
466 #define atomic_swap_int atomic_swap_32
467
468 #define atomic_add_long atomic_add_64
469 #define atomic_clear_long atomic_clear_64
470 #define atomic_cmpset_long atomic_cmpset_64
471 #define atomic_fcmpset_long atomic_fcmpset_64
472 #define atomic_fetchadd_long atomic_fetchadd_64
473 #define atomic_readandclear_long atomic_readandclear_64
474 #define atomic_set_long atomic_set_64
475 #define atomic_subtract_long atomic_subtract_64
476 #define atomic_swap_long atomic_swap_64
477
478 #define atomic_add_ptr atomic_add_64
479 #define atomic_clear_ptr atomic_clear_64
480 #define atomic_cmpset_ptr atomic_cmpset_64
481 #define atomic_fcmpset_ptr atomic_fcmpset_64
482 #define atomic_fetchadd_ptr atomic_fetchadd_64
483 #define atomic_readandclear_ptr atomic_readandclear_64
484 #define atomic_set_ptr atomic_set_64
485 #define atomic_subtract_ptr atomic_subtract_64
486 #define atomic_swap_ptr atomic_swap_64
487
488 ATOMIC_ACQ_REL(set, 64)
489 ATOMIC_ACQ_REL(clear, 64)
490 ATOMIC_ACQ_REL(add, 64)
491 ATOMIC_ACQ_REL(subtract, 64)
492
493 ATOMIC_CMPSET_ACQ_REL(64);
494 ATOMIC_FCMPSET_ACQ_REL(64);
495
496 static __inline uint64_t
atomic_load_acq_64(volatile uint64_t * p)497 atomic_load_acq_64(volatile uint64_t *p)
498 {
499 uint64_t ret;
500
501 ret = *p;
502
503 fence();
504
505 return (ret);
506 }
507
508 static __inline void
atomic_store_rel_64(volatile uint64_t * p,uint64_t val)509 atomic_store_rel_64(volatile uint64_t *p, uint64_t val)
510 {
511
512 fence();
513
514 *p = val;
515 }
516
517 #define atomic_add_acq_long atomic_add_acq_64
518 #define atomic_clear_acq_long atomic_clear_acq_64
519 #define atomic_cmpset_acq_long atomic_cmpset_acq_64
520 #define atomic_fcmpset_acq_long atomic_fcmpset_acq_64
521 #define atomic_load_acq_long atomic_load_acq_64
522 #define atomic_set_acq_long atomic_set_acq_64
523 #define atomic_subtract_acq_long atomic_subtract_acq_64
524
525 #define atomic_add_acq_ptr atomic_add_acq_64
526 #define atomic_clear_acq_ptr atomic_clear_acq_64
527 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_64
528 #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_64
529 #define atomic_load_acq_ptr atomic_load_acq_64
530 #define atomic_set_acq_ptr atomic_set_acq_64
531 #define atomic_subtract_acq_ptr atomic_subtract_acq_64
532
533 #undef ATOMIC_ACQ_REL
534
535 static __inline void
atomic_thread_fence_acq(void)536 atomic_thread_fence_acq(void)
537 {
538
539 fence();
540 }
541
542 static __inline void
atomic_thread_fence_rel(void)543 atomic_thread_fence_rel(void)
544 {
545
546 fence();
547 }
548
549 static __inline void
atomic_thread_fence_acq_rel(void)550 atomic_thread_fence_acq_rel(void)
551 {
552
553 fence();
554 }
555
556 static __inline void
atomic_thread_fence_seq_cst(void)557 atomic_thread_fence_seq_cst(void)
558 {
559
560 fence();
561 }
562
563 #define atomic_add_rel_long atomic_add_rel_64
564 #define atomic_clear_rel_long atomic_clear_rel_64
565
566 #define atomic_add_rel_long atomic_add_rel_64
567 #define atomic_clear_rel_long atomic_clear_rel_64
568 #define atomic_cmpset_rel_long atomic_cmpset_rel_64
569 #define atomic_fcmpset_rel_long atomic_fcmpset_rel_64
570 #define atomic_set_rel_long atomic_set_rel_64
571 #define atomic_subtract_rel_long atomic_subtract_rel_64
572 #define atomic_store_rel_long atomic_store_rel_64
573
574 #define atomic_add_rel_ptr atomic_add_rel_64
575 #define atomic_clear_rel_ptr atomic_clear_rel_64
576 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_64
577 #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_64
578 #define atomic_set_rel_ptr atomic_set_rel_64
579 #define atomic_subtract_rel_ptr atomic_subtract_rel_64
580 #define atomic_store_rel_ptr atomic_store_rel_64
581
582 #include <sys/_atomic_subword.h>
583
584 #endif /* _MACHINE_ATOMIC_H_ */
585