1 /* $NetBSD: atomic.h,v 1.26 2022/07/31 11:28:46 martin Exp $ */
2
3 /*-
4 * Copyright (c) 2007, 2008 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Jason R. Thorpe.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32 #ifndef _SYS_ATOMIC_H_
33 #define _SYS_ATOMIC_H_
34
35 #include <sys/types.h>
36 #if !defined(_KERNEL) && !defined(_STANDALONE)
37 #include <stdint.h>
38 #endif
39
40 #if defined(_KERNEL) && defined(_KERNEL_OPT)
41 #include "opt_kasan.h"
42 #include "opt_kcsan.h"
43 #include "opt_kmsan.h"
44 #endif
45
46 #if defined(KASAN)
47 #define ATOMIC_PROTO_ADD(name, tret, targ1, targ2) \
48 void kasan_atomic_add_##name(volatile targ1 *, targ2); \
49 tret kasan_atomic_add_##name##_nv(volatile targ1 *, targ2)
50 #define ATOMIC_PROTO_AND(name, tret, targ1, targ2) \
51 void kasan_atomic_and_##name(volatile targ1 *, targ2); \
52 tret kasan_atomic_and_##name##_nv(volatile targ1 *, targ2)
53 #define ATOMIC_PROTO_OR(name, tret, targ1, targ2) \
54 void kasan_atomic_or_##name(volatile targ1 *, targ2); \
55 tret kasan_atomic_or_##name##_nv(volatile targ1 *, targ2)
56 #define ATOMIC_PROTO_CAS(name, tret, targ1, targ2) \
57 tret kasan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \
58 tret kasan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2)
59 #define ATOMIC_PROTO_SWAP(name, tret, targ1, targ2) \
60 tret kasan_atomic_swap_##name(volatile targ1 *, targ2)
61 #define ATOMIC_PROTO_DEC(name, tret, targ1) \
62 void kasan_atomic_dec_##name(volatile targ1 *); \
63 tret kasan_atomic_dec_##name##_nv(volatile targ1 *)
64 #define ATOMIC_PROTO_INC(name, tret, targ1) \
65 void kasan_atomic_inc_##name(volatile targ1 *); \
66 tret kasan_atomic_inc_##name##_nv(volatile targ1 *)
67 #elif defined(KCSAN)
68 #define ATOMIC_PROTO_ADD(name, tret, targ1, targ2) \
69 void kcsan_atomic_add_##name(volatile targ1 *, targ2); \
70 tret kcsan_atomic_add_##name##_nv(volatile targ1 *, targ2)
71 #define ATOMIC_PROTO_AND(name, tret, targ1, targ2) \
72 void kcsan_atomic_and_##name(volatile targ1 *, targ2); \
73 tret kcsan_atomic_and_##name##_nv(volatile targ1 *, targ2)
74 #define ATOMIC_PROTO_OR(name, tret, targ1, targ2) \
75 void kcsan_atomic_or_##name(volatile targ1 *, targ2); \
76 tret kcsan_atomic_or_##name##_nv(volatile targ1 *, targ2)
77 #define ATOMIC_PROTO_CAS(name, tret, targ1, targ2) \
78 tret kcsan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \
79 tret kcsan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2)
80 #define ATOMIC_PROTO_SWAP(name, tret, targ1, targ2) \
81 tret kcsan_atomic_swap_##name(volatile targ1 *, targ2)
82 #define ATOMIC_PROTO_DEC(name, tret, targ1) \
83 void kcsan_atomic_dec_##name(volatile targ1 *); \
84 tret kcsan_atomic_dec_##name##_nv(volatile targ1 *)
85 #define ATOMIC_PROTO_INC(name, tret, targ1) \
86 void kcsan_atomic_inc_##name(volatile targ1 *); \
87 tret kcsan_atomic_inc_##name##_nv(volatile targ1 *)
88 #elif defined(KMSAN)
89 #define ATOMIC_PROTO_ADD(name, tret, targ1, targ2) \
90 void kmsan_atomic_add_##name(volatile targ1 *, targ2); \
91 tret kmsan_atomic_add_##name##_nv(volatile targ1 *, targ2)
92 #define ATOMIC_PROTO_AND(name, tret, targ1, targ2) \
93 void kmsan_atomic_and_##name(volatile targ1 *, targ2); \
94 tret kmsan_atomic_and_##name##_nv(volatile targ1 *, targ2)
95 #define ATOMIC_PROTO_OR(name, tret, targ1, targ2) \
96 void kmsan_atomic_or_##name(volatile targ1 *, targ2); \
97 tret kmsan_atomic_or_##name##_nv(volatile targ1 *, targ2)
98 #define ATOMIC_PROTO_CAS(name, tret, targ1, targ2) \
99 tret kmsan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \
100 tret kmsan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2)
101 #define ATOMIC_PROTO_SWAP(name, tret, targ1, targ2) \
102 tret kmsan_atomic_swap_##name(volatile targ1 *, targ2)
103 #define ATOMIC_PROTO_DEC(name, tret, targ1) \
104 void kmsan_atomic_dec_##name(volatile targ1 *); \
105 tret kmsan_atomic_dec_##name##_nv(volatile targ1 *)
106 #define ATOMIC_PROTO_INC(name, tret, targ1) \
107 void kmsan_atomic_inc_##name(volatile targ1 *); \
108 tret kmsan_atomic_inc_##name##_nv(volatile targ1 *)
109 #else
110 #define ATOMIC_PROTO_ADD(name, tret, targ1, targ2) \
111 void atomic_add_##name(volatile targ1 *, targ2); \
112 tret atomic_add_##name##_nv(volatile targ1 *, targ2)
113 #define ATOMIC_PROTO_AND(name, tret, targ1, targ2) \
114 void atomic_and_##name(volatile targ1 *, targ2); \
115 tret atomic_and_##name##_nv(volatile targ1 *, targ2)
116 #define ATOMIC_PROTO_OR(name, tret, targ1, targ2) \
117 void atomic_or_##name(volatile targ1 *, targ2); \
118 tret atomic_or_##name##_nv(volatile targ1 *, targ2)
119 #define ATOMIC_PROTO_CAS(name, tret, targ1, targ2) \
120 tret atomic_cas_##name(volatile targ1 *, targ2, targ2); \
121 tret atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2)
122 #define ATOMIC_PROTO_SWAP(name, tret, targ1, targ2) \
123 tret atomic_swap_##name(volatile targ1 *, targ2)
124 #define ATOMIC_PROTO_DEC(name, tret, targ1) \
125 void atomic_dec_##name(volatile targ1 *); \
126 tret atomic_dec_##name##_nv(volatile targ1 *)
127 #define ATOMIC_PROTO_INC(name, tret, targ1) \
128 void atomic_inc_##name(volatile targ1 *); \
129 tret atomic_inc_##name##_nv(volatile targ1 *)
130 #endif
131
132 __BEGIN_DECLS
133
134 ATOMIC_PROTO_ADD(32, uint32_t, uint32_t, int32_t);
135 ATOMIC_PROTO_ADD(64, uint64_t, uint64_t, int64_t);
136 ATOMIC_PROTO_ADD(int, unsigned int, unsigned int, int);
137 ATOMIC_PROTO_ADD(long, unsigned long, unsigned long, long);
138 ATOMIC_PROTO_ADD(ptr, void *, void, ssize_t);
139
140 ATOMIC_PROTO_AND(32, uint32_t, uint32_t, uint32_t);
141 ATOMIC_PROTO_AND(64, uint64_t, uint64_t, uint64_t);
142 ATOMIC_PROTO_AND(uint, unsigned int, unsigned int, unsigned int);
143 ATOMIC_PROTO_AND(ulong, unsigned long, unsigned long, unsigned long);
144
145 ATOMIC_PROTO_OR(32, uint32_t, uint32_t, uint32_t);
146 ATOMIC_PROTO_OR(64, uint64_t, uint64_t, uint64_t);
147 ATOMIC_PROTO_OR(uint, unsigned int, unsigned int, unsigned int);
148 ATOMIC_PROTO_OR(ulong, unsigned long, unsigned long, unsigned long);
149
150 ATOMIC_PROTO_CAS(32, uint32_t, uint32_t, uint32_t);
151 ATOMIC_PROTO_CAS(64, uint64_t, uint64_t, uint64_t);
152 ATOMIC_PROTO_CAS(uint, unsigned int, unsigned int, unsigned int);
153 ATOMIC_PROTO_CAS(ulong, unsigned long, unsigned long, unsigned long);
154 ATOMIC_PROTO_CAS(ptr, void *, void, void *);
155
156 ATOMIC_PROTO_SWAP(32, uint32_t, uint32_t, uint32_t);
157 ATOMIC_PROTO_SWAP(64, uint64_t, uint64_t, uint64_t);
158 ATOMIC_PROTO_SWAP(uint, unsigned int, unsigned int, unsigned int);
159 ATOMIC_PROTO_SWAP(ulong, unsigned long, unsigned long, unsigned long);
160 ATOMIC_PROTO_SWAP(ptr, void *, void, void *);
161
162 ATOMIC_PROTO_DEC(32, uint32_t, uint32_t);
163 ATOMIC_PROTO_DEC(64, uint64_t, uint64_t);
164 ATOMIC_PROTO_DEC(uint, unsigned int, unsigned int);
165 ATOMIC_PROTO_DEC(ulong, unsigned long, unsigned long);
166 ATOMIC_PROTO_DEC(ptr, void *, void);
167
168 ATOMIC_PROTO_INC(32, uint32_t, uint32_t);
169 ATOMIC_PROTO_INC(64, uint64_t, uint64_t);
170 ATOMIC_PROTO_INC(uint, unsigned int, unsigned int);
171 ATOMIC_PROTO_INC(ulong, unsigned long, unsigned long);
172 ATOMIC_PROTO_INC(ptr, void *, void);
173
174 /*
175 * These operations will be provided for userland, but may not be
176 * implemented efficiently.
177 */
178 uint16_t atomic_cas_16(volatile uint16_t *, uint16_t, uint16_t);
179 uint8_t atomic_cas_8(volatile uint8_t *, uint8_t, uint8_t);
180
181 /*
182 * Memory barrier operations
183 */
184 void membar_acquire(void);
185 void membar_release(void);
186 void membar_producer(void);
187 void membar_consumer(void);
188 void membar_sync(void);
189
190 /*
191 * Deprecated memory barriers
192 */
193 void membar_enter(void);
194 void membar_exit(void);
195
196 #ifdef __HAVE_MEMBAR_DATADEP_CONSUMER
197 void membar_datadep_consumer(void);
198 #else
199 #define membar_datadep_consumer() ((void)0)
200 #endif
201
202 __END_DECLS
203
204 #if defined(KASAN)
205 #define atomic_add_32 kasan_atomic_add_32
206 #define atomic_add_int kasan_atomic_add_int
207 #define atomic_add_long kasan_atomic_add_long
208 #define atomic_add_ptr kasan_atomic_add_ptr
209 #define atomic_add_64 kasan_atomic_add_64
210 #define atomic_add_32_nv kasan_atomic_add_32_nv
211 #define atomic_add_int_nv kasan_atomic_add_int_nv
212 #define atomic_add_long_nv kasan_atomic_add_long_nv
213 #define atomic_add_ptr_nv kasan_atomic_add_ptr_nv
214 #define atomic_add_64_nv kasan_atomic_add_64_nv
215 #define atomic_and_32 kasan_atomic_and_32
216 #define atomic_and_uint kasan_atomic_and_uint
217 #define atomic_and_ulong kasan_atomic_and_ulong
218 #define atomic_and_64 kasan_atomic_and_64
219 #define atomic_and_32_nv kasan_atomic_and_32_nv
220 #define atomic_and_uint_nv kasan_atomic_and_uint_nv
221 #define atomic_and_ulong_nv kasan_atomic_and_ulong_nv
222 #define atomic_and_64_nv kasan_atomic_and_64_nv
223 #define atomic_or_32 kasan_atomic_or_32
224 #define atomic_or_uint kasan_atomic_or_uint
225 #define atomic_or_ulong kasan_atomic_or_ulong
226 #define atomic_or_64 kasan_atomic_or_64
227 #define atomic_or_32_nv kasan_atomic_or_32_nv
228 #define atomic_or_uint_nv kasan_atomic_or_uint_nv
229 #define atomic_or_ulong_nv kasan_atomic_or_ulong_nv
230 #define atomic_or_64_nv kasan_atomic_or_64_nv
231 #define atomic_cas_32 kasan_atomic_cas_32
232 #define atomic_cas_uint kasan_atomic_cas_uint
233 #define atomic_cas_ulong kasan_atomic_cas_ulong
234 #define atomic_cas_ptr kasan_atomic_cas_ptr
235 #define atomic_cas_64 kasan_atomic_cas_64
236 #define atomic_cas_32_ni kasan_atomic_cas_32_ni
237 #define atomic_cas_uint_ni kasan_atomic_cas_uint_ni
238 #define atomic_cas_ulong_ni kasan_atomic_cas_ulong_ni
239 #define atomic_cas_ptr_ni kasan_atomic_cas_ptr_ni
240 #define atomic_cas_64_ni kasan_atomic_cas_64_ni
241 #define atomic_swap_32 kasan_atomic_swap_32
242 #define atomic_swap_uint kasan_atomic_swap_uint
243 #define atomic_swap_ulong kasan_atomic_swap_ulong
244 #define atomic_swap_ptr kasan_atomic_swap_ptr
245 #define atomic_swap_64 kasan_atomic_swap_64
246 #define atomic_dec_32 kasan_atomic_dec_32
247 #define atomic_dec_uint kasan_atomic_dec_uint
248 #define atomic_dec_ulong kasan_atomic_dec_ulong
249 #define atomic_dec_ptr kasan_atomic_dec_ptr
250 #define atomic_dec_64 kasan_atomic_dec_64
251 #define atomic_dec_32_nv kasan_atomic_dec_32_nv
252 #define atomic_dec_uint_nv kasan_atomic_dec_uint_nv
253 #define atomic_dec_ulong_nv kasan_atomic_dec_ulong_nv
254 #define atomic_dec_ptr_nv kasan_atomic_dec_ptr_nv
255 #define atomic_dec_64_nv kasan_atomic_dec_64_nv
256 #define atomic_inc_32 kasan_atomic_inc_32
257 #define atomic_inc_uint kasan_atomic_inc_uint
258 #define atomic_inc_ulong kasan_atomic_inc_ulong
259 #define atomic_inc_ptr kasan_atomic_inc_ptr
260 #define atomic_inc_64 kasan_atomic_inc_64
261 #define atomic_inc_32_nv kasan_atomic_inc_32_nv
262 #define atomic_inc_uint_nv kasan_atomic_inc_uint_nv
263 #define atomic_inc_ulong_nv kasan_atomic_inc_ulong_nv
264 #define atomic_inc_ptr_nv kasan_atomic_inc_ptr_nv
265 #define atomic_inc_64_nv kasan_atomic_inc_64_nv
266 #elif defined(KCSAN)
267 #define atomic_add_32 kcsan_atomic_add_32
268 #define atomic_add_int kcsan_atomic_add_int
269 #define atomic_add_long kcsan_atomic_add_long
270 #define atomic_add_ptr kcsan_atomic_add_ptr
271 #define atomic_add_64 kcsan_atomic_add_64
272 #define atomic_add_32_nv kcsan_atomic_add_32_nv
273 #define atomic_add_int_nv kcsan_atomic_add_int_nv
274 #define atomic_add_long_nv kcsan_atomic_add_long_nv
275 #define atomic_add_ptr_nv kcsan_atomic_add_ptr_nv
276 #define atomic_add_64_nv kcsan_atomic_add_64_nv
277 #define atomic_and_32 kcsan_atomic_and_32
278 #define atomic_and_uint kcsan_atomic_and_uint
279 #define atomic_and_ulong kcsan_atomic_and_ulong
280 #define atomic_and_64 kcsan_atomic_and_64
281 #define atomic_and_32_nv kcsan_atomic_and_32_nv
282 #define atomic_and_uint_nv kcsan_atomic_and_uint_nv
283 #define atomic_and_ulong_nv kcsan_atomic_and_ulong_nv
284 #define atomic_and_64_nv kcsan_atomic_and_64_nv
285 #define atomic_or_32 kcsan_atomic_or_32
286 #define atomic_or_uint kcsan_atomic_or_uint
287 #define atomic_or_ulong kcsan_atomic_or_ulong
288 #define atomic_or_64 kcsan_atomic_or_64
289 #define atomic_or_32_nv kcsan_atomic_or_32_nv
290 #define atomic_or_uint_nv kcsan_atomic_or_uint_nv
291 #define atomic_or_ulong_nv kcsan_atomic_or_ulong_nv
292 #define atomic_or_64_nv kcsan_atomic_or_64_nv
293 #define atomic_cas_32 kcsan_atomic_cas_32
294 #define atomic_cas_uint kcsan_atomic_cas_uint
295 #define atomic_cas_ulong kcsan_atomic_cas_ulong
296 #define atomic_cas_ptr kcsan_atomic_cas_ptr
297 #define atomic_cas_64 kcsan_atomic_cas_64
298 #define atomic_cas_32_ni kcsan_atomic_cas_32_ni
299 #define atomic_cas_uint_ni kcsan_atomic_cas_uint_ni
300 #define atomic_cas_ulong_ni kcsan_atomic_cas_ulong_ni
301 #define atomic_cas_ptr_ni kcsan_atomic_cas_ptr_ni
302 #define atomic_cas_64_ni kcsan_atomic_cas_64_ni
303 #define atomic_swap_32 kcsan_atomic_swap_32
304 #define atomic_swap_uint kcsan_atomic_swap_uint
305 #define atomic_swap_ulong kcsan_atomic_swap_ulong
306 #define atomic_swap_ptr kcsan_atomic_swap_ptr
307 #define atomic_swap_64 kcsan_atomic_swap_64
308 #define atomic_dec_32 kcsan_atomic_dec_32
309 #define atomic_dec_uint kcsan_atomic_dec_uint
310 #define atomic_dec_ulong kcsan_atomic_dec_ulong
311 #define atomic_dec_ptr kcsan_atomic_dec_ptr
312 #define atomic_dec_64 kcsan_atomic_dec_64
313 #define atomic_dec_32_nv kcsan_atomic_dec_32_nv
314 #define atomic_dec_uint_nv kcsan_atomic_dec_uint_nv
315 #define atomic_dec_ulong_nv kcsan_atomic_dec_ulong_nv
316 #define atomic_dec_ptr_nv kcsan_atomic_dec_ptr_nv
317 #define atomic_dec_64_nv kcsan_atomic_dec_64_nv
318 #define atomic_inc_32 kcsan_atomic_inc_32
319 #define atomic_inc_uint kcsan_atomic_inc_uint
320 #define atomic_inc_ulong kcsan_atomic_inc_ulong
321 #define atomic_inc_ptr kcsan_atomic_inc_ptr
322 #define atomic_inc_64 kcsan_atomic_inc_64
323 #define atomic_inc_32_nv kcsan_atomic_inc_32_nv
324 #define atomic_inc_uint_nv kcsan_atomic_inc_uint_nv
325 #define atomic_inc_ulong_nv kcsan_atomic_inc_ulong_nv
326 #define atomic_inc_ptr_nv kcsan_atomic_inc_ptr_nv
327 #define atomic_inc_64_nv kcsan_atomic_inc_64_nv
328 #elif defined(KMSAN)
329 #define atomic_add_32 kmsan_atomic_add_32
330 #define atomic_add_int kmsan_atomic_add_int
331 #define atomic_add_long kmsan_atomic_add_long
332 #define atomic_add_ptr kmsan_atomic_add_ptr
333 #define atomic_add_64 kmsan_atomic_add_64
334 #define atomic_add_32_nv kmsan_atomic_add_32_nv
335 #define atomic_add_int_nv kmsan_atomic_add_int_nv
336 #define atomic_add_long_nv kmsan_atomic_add_long_nv
337 #define atomic_add_ptr_nv kmsan_atomic_add_ptr_nv
338 #define atomic_add_64_nv kmsan_atomic_add_64_nv
339 #define atomic_and_32 kmsan_atomic_and_32
340 #define atomic_and_uint kmsan_atomic_and_uint
341 #define atomic_and_ulong kmsan_atomic_and_ulong
342 #define atomic_and_64 kmsan_atomic_and_64
343 #define atomic_and_32_nv kmsan_atomic_and_32_nv
344 #define atomic_and_uint_nv kmsan_atomic_and_uint_nv
345 #define atomic_and_ulong_nv kmsan_atomic_and_ulong_nv
346 #define atomic_and_64_nv kmsan_atomic_and_64_nv
347 #define atomic_or_32 kmsan_atomic_or_32
348 #define atomic_or_uint kmsan_atomic_or_uint
349 #define atomic_or_ulong kmsan_atomic_or_ulong
350 #define atomic_or_64 kmsan_atomic_or_64
351 #define atomic_or_32_nv kmsan_atomic_or_32_nv
352 #define atomic_or_uint_nv kmsan_atomic_or_uint_nv
353 #define atomic_or_ulong_nv kmsan_atomic_or_ulong_nv
354 #define atomic_or_64_nv kmsan_atomic_or_64_nv
355 #define atomic_cas_32 kmsan_atomic_cas_32
356 #define atomic_cas_uint kmsan_atomic_cas_uint
357 #define atomic_cas_ulong kmsan_atomic_cas_ulong
358 #define atomic_cas_ptr kmsan_atomic_cas_ptr
359 #define atomic_cas_64 kmsan_atomic_cas_64
360 #define atomic_cas_32_ni kmsan_atomic_cas_32_ni
361 #define atomic_cas_uint_ni kmsan_atomic_cas_uint_ni
362 #define atomic_cas_ulong_ni kmsan_atomic_cas_ulong_ni
363 #define atomic_cas_ptr_ni kmsan_atomic_cas_ptr_ni
364 #define atomic_cas_64_ni kmsan_atomic_cas_64_ni
365 #define atomic_swap_32 kmsan_atomic_swap_32
366 #define atomic_swap_uint kmsan_atomic_swap_uint
367 #define atomic_swap_ulong kmsan_atomic_swap_ulong
368 #define atomic_swap_ptr kmsan_atomic_swap_ptr
369 #define atomic_swap_64 kmsan_atomic_swap_64
370 #define atomic_dec_32 kmsan_atomic_dec_32
371 #define atomic_dec_uint kmsan_atomic_dec_uint
372 #define atomic_dec_ulong kmsan_atomic_dec_ulong
373 #define atomic_dec_ptr kmsan_atomic_dec_ptr
374 #define atomic_dec_64 kmsan_atomic_dec_64
375 #define atomic_dec_32_nv kmsan_atomic_dec_32_nv
376 #define atomic_dec_uint_nv kmsan_atomic_dec_uint_nv
377 #define atomic_dec_ulong_nv kmsan_atomic_dec_ulong_nv
378 #define atomic_dec_ptr_nv kmsan_atomic_dec_ptr_nv
379 #define atomic_dec_64_nv kmsan_atomic_dec_64_nv
380 #define atomic_inc_32 kmsan_atomic_inc_32
381 #define atomic_inc_uint kmsan_atomic_inc_uint
382 #define atomic_inc_ulong kmsan_atomic_inc_ulong
383 #define atomic_inc_ptr kmsan_atomic_inc_ptr
384 #define atomic_inc_64 kmsan_atomic_inc_64
385 #define atomic_inc_32_nv kmsan_atomic_inc_32_nv
386 #define atomic_inc_uint_nv kmsan_atomic_inc_uint_nv
387 #define atomic_inc_ulong_nv kmsan_atomic_inc_ulong_nv
388 #define atomic_inc_ptr_nv kmsan_atomic_inc_ptr_nv
389 #define atomic_inc_64_nv kmsan_atomic_inc_64_nv
390 #endif
391
392 #ifdef _KERNEL
393
394 #if 1 // XXX: __STDC_VERSION__ < 201112L
395
396 /* Pre-C11 definitions */
397
398 #include <sys/cdefs.h>
399
400 #include <lib/libkern/libkern.h>
401
402 #ifdef _LP64
403 #define __HAVE_ATOMIC64_LOADSTORE 1
404 #define __ATOMIC_SIZE_MAX 8
405 #else
406 #define __ATOMIC_SIZE_MAX 4
407 #endif
408
409 /*
410 * We assume that access to an aligned pointer to a volatile object of
411 * at most __ATOMIC_SIZE_MAX bytes is guaranteed to be atomic. This is
412 * an assumption that may be wrong, but we hope it won't be wrong
413 * before we just adopt the C11 atomic API.
414 */
415 #define __ATOMIC_PTR_CHECK(p) do \
416 { \
417 CTASSERT(sizeof(*(p)) <= __ATOMIC_SIZE_MAX); \
418 KASSERT(((uintptr_t)(p) & (sizeof(*(p)) - 1)) == 0); \
419 } while (0)
420
421 #ifdef KCSAN
422 void kcsan_atomic_load(const volatile void *, void *, int);
423 void kcsan_atomic_store(volatile void *, const void *, int);
424 #define __BEGIN_ATOMIC_LOAD(p, v) \
425 union { __typeof__(*(p)) __al_val; char __al_buf[1]; } v; \
426 kcsan_atomic_load(p, v.__al_buf, sizeof(v.__al_val))
427 #define __END_ATOMIC_LOAD(v) \
428 (v).__al_val
429 #define __DO_ATOMIC_STORE(p, v) \
430 kcsan_atomic_store(p, __UNVOLATILE(&v), sizeof(v))
431 #else
432 #define __BEGIN_ATOMIC_LOAD(p, v) \
433 __typeof__(*(p)) v = *(p)
434 #define __END_ATOMIC_LOAD(v) \
435 v
436 #ifdef __HAVE_HASHLOCKED_ATOMICS
437 #define __DO_ATOMIC_STORE(p, v) \
438 __do_atomic_store(p, __UNVOLATILE(&v), sizeof(v))
439 #else /* !__HAVE_HASHLOCKED_ATOMICS */
440 #define __DO_ATOMIC_STORE(p, v) \
441 *p = v
442 #endif
443 #endif
444
445 #define atomic_load_relaxed(p) \
446 ({ \
447 const volatile __typeof__(*(p)) *__al_ptr = (p); \
448 __ATOMIC_PTR_CHECK(__al_ptr); \
449 __BEGIN_ATOMIC_LOAD(__al_ptr, __al_val); \
450 __END_ATOMIC_LOAD(__al_val); \
451 })
452
453 #define atomic_load_consume(p) \
454 ({ \
455 const volatile __typeof__(*(p)) *__al_ptr = (p); \
456 __ATOMIC_PTR_CHECK(__al_ptr); \
457 __BEGIN_ATOMIC_LOAD(__al_ptr, __al_val); \
458 membar_datadep_consumer(); \
459 __END_ATOMIC_LOAD(__al_val); \
460 })
461
462 #define atomic_load_acquire(p) \
463 ({ \
464 const volatile __typeof__(*(p)) *__al_ptr = (p); \
465 __ATOMIC_PTR_CHECK(__al_ptr); \
466 __BEGIN_ATOMIC_LOAD(__al_ptr, __al_val); \
467 membar_acquire(); \
468 __END_ATOMIC_LOAD(__al_val); \
469 })
470
471 #define atomic_store_relaxed(p,v) \
472 ({ \
473 volatile __typeof__(*(p)) *__as_ptr = (p); \
474 __typeof__(*(p)) __as_val = (v); \
475 __ATOMIC_PTR_CHECK(__as_ptr); \
476 __DO_ATOMIC_STORE(__as_ptr, __as_val); \
477 })
478
479 #define atomic_store_release(p,v) \
480 ({ \
481 volatile __typeof__(*(p)) *__as_ptr = (p); \
482 __typeof__(*(p)) __as_val = (v); \
483 __ATOMIC_PTR_CHECK(__as_ptr); \
484 membar_release(); \
485 __DO_ATOMIC_STORE(__as_ptr, __as_val); \
486 })
487
488 #ifdef __HAVE_HASHLOCKED_ATOMICS
489 static __inline __always_inline void
__do_atomic_store(volatile void * p,const void * q,size_t size)490 __do_atomic_store(volatile void *p, const void *q, size_t size)
491 {
492 switch (size) {
493 case 1: {
494 uint8_t v;
495 unsigned s = 8 * ((uintptr_t)p & 3);
496 uint32_t o, n, m = ~(0xffU << s);
497 memcpy(&v, q, 1);
498 do {
499 o = atomic_load_relaxed((const volatile uint32_t *)p);
500 n = (o & m) | ((uint32_t)v << s);
501 } while (atomic_cas_32((volatile uint32_t *)p, o, n) != o);
502 break;
503 }
504 case 2: {
505 uint16_t v;
506 unsigned s = 8 * (((uintptr_t)p & 2) >> 1);
507 uint32_t o, n, m = ~(0xffffU << s);
508 memcpy(&v, q, 2);
509 do {
510 o = atomic_load_relaxed((const volatile uint32_t *)p);
511 n = (o & m) | ((uint32_t)v << s);
512 } while (atomic_cas_32((volatile uint32_t *)p, o, n) != o);
513 break;
514 }
515 case 4: {
516 uint32_t v;
517 memcpy(&v, q, 4);
518 (void)atomic_swap_32(p, v);
519 break;
520 }
521 #ifdef __HAVE_ATOMIC64_LOADSTORE
522 case 8: {
523 uint64_t v;
524 memcpy(&v, q, 8);
525 (void)atomic_swap_64(p, v);
526 break;
527 }
528 #endif
529 }
530 }
531 #endif /* __HAVE_HASHLOCKED_ATOMICS */
532
533 #else /* __STDC_VERSION__ >= 201112L */
534
535 /* C11 definitions, not yet available */
536
537 #include <stdatomic.h>
538
539 #define atomic_load_relaxed(p) \
540 atomic_load_explicit((p), memory_order_relaxed)
541 #if 0 /* memory_order_consume is not there yet */
542 #define atomic_load_consume(p) \
543 atomic_load_explicit((p), memory_order_consume)
544 #else
545 #define atomic_load_consume(p) \
546 ({ \
547 const __typeof__(*(p)) __al_val = atomic_load_relaxed(p); \
548 membar_datadep_consumer(); \
549 __al_val; \
550 })
551 #endif
552 #define atomic_load_acquire(p) \
553 atomic_load_explicit((p), memory_order_acquire)
554 #define atomic_store_relaxed(p, v) \
555 atomic_store_explicit((p), (v), memory_order_relaxed)
556 #define atomic_store_release(p, v) \
557 atomic_store_explicit((p), (v), memory_order_release)
558
559 #endif /* __STDC_VERSION__ */
560
561 #endif /* _KERNEL */
562
563 #endif /* ! _SYS_ATOMIC_H_ */
564