1 /* $NetBSD: atomic_init_testset.c,v 1.15 2014/02/27 09:39:00 matt Exp $ */
2
3 /*-
4 * Copyright (c) 2008 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
17 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
18 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26 * POSSIBILITY OF SUCH DAMAGE.
27 */
28
29 /*
30 * libc glue for atomic operations where the hardware does not provide
31 * compare-and-swap. It's assumed that this will only be used on 32-bit
32 * platforms.
33 *
34 * This should be compiled with '-fno-reorder-blocks -fomit-frame-pointer'
35 * if using gcc.
36 */
37
38 #include <sys/cdefs.h>
39 __RCSID("$NetBSD: atomic_init_testset.c,v 1.15 2014/02/27 09:39:00 matt Exp $");
40
41 #include "atomic_op_namespace.h"
42
43 #include <sys/types.h>
44 #include <sys/atomic.h>
45 #include <sys/lock.h>
46 #include <sys/ras.h>
47 #include <sys/sysctl.h>
48
49 #include <string.h>
50
51 #define I2 __SIMPLELOCK_UNLOCKED, __SIMPLELOCK_UNLOCKED,
52 #define I16 I2 I2 I2 I2 I2 I2 I2 I2
53 #define I128 I16 I16 I16 I16 I16 I16 I16 I16
54
55 static __cpu_simple_lock_t atomic_locks[128] = { I128 };
56 /*
57 * Pick a lock out of above array depending on the object address
58 * passed. Most variables used atomically will not be in the same
59 * cacheline - and if they are, using the same lock is fine.
60 */
61 #define HASH(PTR) (((uintptr_t)(PTR) >> 3) & 127)
62
63 #ifdef __HAVE_ASM_ATOMIC_CAS_UP
64 extern uint32_t _atomic_cas_up(volatile uint32_t *, uint32_t, uint32_t);
65 #else
66 static uint32_t _atomic_cas_up(volatile uint32_t *, uint32_t, uint32_t);
67 #endif
68 static uint32_t (*_atomic_cas_fn)(volatile uint32_t *, uint32_t, uint32_t) =
69 _atomic_cas_up;
70 RAS_DECL(_atomic_cas);
71
72 #ifdef __HAVE_ATOMIC_CAS_64_UP
73 #ifdef __HAVE_ASM_ATOMIC_CAS_64_UP
74 extern uint64_t _atomic_cas_64_up(volatile uint64_t *, uint64_t, uint64_t);
75 #else
76 static uint64_t _atomic_cas_64_up(volatile uint64_t *, uint64_t, uint64_t);
77 #endif
78 static uint64_t (*_atomic_cas_64_fn)(volatile uint64_t *, uint64_t, uint64_t) =
79 _atomic_cas_64_up;
80 RAS_DECL(_atomic_cas_64);
81 #endif
82
83 #ifdef __HAVE_ASM_ATOMIC_CAS_16_UP
84 extern uint16_t _atomic_cas_16_up(volatile uint16_t *, uint16_t, uint16_t);
85 #else
86 static uint16_t _atomic_cas_16_up(volatile uint16_t *, uint16_t, uint16_t);
87 #endif
88 static uint16_t (*_atomic_cas_16_fn)(volatile uint16_t *, uint16_t, uint16_t) =
89 _atomic_cas_16_up;
90 RAS_DECL(_atomic_cas_16);
91
92 #ifdef __HAVE_ASM_ATOMIC_CAS_8_UP
93 extern uint8_t _atomic_cas_8_up(volatile uint8_t *, uint8_t, uint8_t);
94 #else
95 static uint8_t _atomic_cas_8_up(volatile uint8_t *, uint8_t, uint8_t);
96 #endif
97 static uint8_t (*_atomic_cas_8_fn)(volatile uint8_t *, uint8_t, uint8_t) =
98 _atomic_cas_8_up;
99 RAS_DECL(_atomic_cas_8);
100
101 void __libc_atomic_init(void) __attribute__ ((visibility("hidden")));
102
103 #ifndef __HAVE_ASM_ATOMIC_CAS_UP
104 static uint32_t
_atomic_cas_up(volatile uint32_t * ptr,uint32_t old,uint32_t new)105 _atomic_cas_up(volatile uint32_t *ptr, uint32_t old, uint32_t new)
106 {
107 uint32_t ret;
108
109 RAS_START(_atomic_cas);
110 ret = *ptr;
111 if (__predict_false(ret != old)) {
112 return ret;
113 }
114 *ptr = new;
115 RAS_END(_atomic_cas);
116
117 return ret;
118 }
119 #endif
120
121 #if defined(__HAVE_ATOMIC_CAS_64_UP) && !defined(__HAVE_ASM_ATOMIC_CAS_64_UP)
122 static uint64_t
_atomic_cas_64_up(volatile uint64_t * ptr,uint64_t old,uint64_t new)123 _atomic_cas_64_up(volatile uint64_t *ptr, uint64_t old, uint64_t new)
124 {
125 uint64_t ret;
126
127 RAS_START(_atomic_cas_64);
128 ret = *ptr;
129 if (__predict_false(ret != old)) {
130 return ret;
131 }
132 *ptr = new;
133 RAS_END(_atomic_cas_64);
134
135 return ret;
136 }
137 #endif
138
139 #ifndef __HAVE_ASM_ATOMIC_CAS_16_UP
140 static uint16_t
_atomic_cas_16_up(volatile uint16_t * ptr,uint16_t old,uint16_t new)141 _atomic_cas_16_up(volatile uint16_t *ptr, uint16_t old, uint16_t new)
142 {
143 uint16_t ret;
144
145 RAS_START(_atomic_cas_16);
146 ret = *ptr;
147 if (__predict_false(ret != old)) {
148 return ret;
149 }
150 *ptr = new;
151 RAS_END(_atomic_cas_16);
152
153 return ret;
154 }
155 #endif
156
157 #ifndef __HAVE_ASM_ATOMIC_CAS_8_UP
158 static uint8_t
_atomic_cas_8_up(volatile uint8_t * ptr,uint8_t old,uint8_t new)159 _atomic_cas_8_up(volatile uint8_t *ptr, uint8_t old, uint8_t new)
160 {
161 uint8_t ret;
162
163 RAS_START(_atomic_cas_8);
164 ret = *ptr;
165 if (__predict_false(ret != old)) {
166 return ret;
167 }
168 *ptr = new;
169 RAS_END(_atomic_cas_8);
170
171 return ret;
172 }
173 #endif
174
175 static uint32_t
_atomic_cas_mp(volatile uint32_t * ptr,uint32_t old,uint32_t new)176 _atomic_cas_mp(volatile uint32_t *ptr, uint32_t old, uint32_t new)
177 {
178 __cpu_simple_lock_t *lock;
179 uint32_t ret;
180
181 lock = &atomic_locks[HASH(ptr)];
182 __cpu_simple_lock(lock);
183 ret = *ptr;
184 if (__predict_true(ret == old)) {
185 *ptr = new;
186 }
187 __cpu_simple_unlock(lock);
188
189 return ret;
190 }
191
192 #ifdef __HAVE_ATOMIC_CAS_64_UP
193 static uint64_t
_atomic_cas_64_mp(volatile uint64_t * ptr,uint64_t old,uint64_t new)194 _atomic_cas_64_mp(volatile uint64_t *ptr, uint64_t old, uint64_t new)
195 {
196 __cpu_simple_lock_t *lock;
197 uint64_t ret;
198
199 lock = &atomic_locks[HASH(ptr)];
200 __cpu_simple_lock(lock);
201 ret = *ptr;
202 if (__predict_true(ret == old)) {
203 *ptr = new;
204 }
205 __cpu_simple_unlock(lock);
206
207 return ret;
208 }
209 #endif
210
211 static uint16_t
_atomic_cas_16_mp(volatile uint16_t * ptr,uint16_t old,uint16_t new)212 _atomic_cas_16_mp(volatile uint16_t *ptr, uint16_t old, uint16_t new)
213 {
214 __cpu_simple_lock_t *lock;
215 uint16_t ret;
216
217 lock = &atomic_locks[HASH(ptr)];
218 __cpu_simple_lock(lock);
219 ret = *ptr;
220 if (__predict_true(ret == old)) {
221 *ptr = new;
222 }
223 __cpu_simple_unlock(lock);
224
225 return ret;
226 }
227
228 static uint8_t
_atomic_cas_8_mp(volatile uint8_t * ptr,uint8_t old,uint8_t new)229 _atomic_cas_8_mp(volatile uint8_t *ptr, uint8_t old, uint8_t new)
230 {
231 __cpu_simple_lock_t *lock;
232 uint8_t ret;
233
234 lock = &atomic_locks[HASH(ptr)];
235 __cpu_simple_lock(lock);
236 ret = *ptr;
237 if (__predict_true(ret == old)) {
238 *ptr = new;
239 }
240 __cpu_simple_unlock(lock);
241
242 return ret;
243 }
244
245 uint32_t
_atomic_cas_32(volatile uint32_t * ptr,uint32_t old,uint32_t new)246 _atomic_cas_32(volatile uint32_t *ptr, uint32_t old, uint32_t new)
247 {
248
249 return (*_atomic_cas_fn)(ptr, old, new);
250 }
251
252 #ifdef __HAVE_ATOMIC_CAS_64_UP
253 uint64_t _atomic_cas_64(volatile uint64_t *, uint64_t, uint64_t);
254
255 uint64_t
_atomic_cas_64(volatile uint64_t * ptr,uint64_t old,uint64_t new)256 _atomic_cas_64(volatile uint64_t *ptr, uint64_t old, uint64_t new)
257 {
258
259 return (*_atomic_cas_64_fn)(ptr, old, new);
260 }
261 #endif
262
263 uint16_t
_atomic_cas_16(volatile uint16_t * ptr,uint16_t old,uint16_t new)264 _atomic_cas_16(volatile uint16_t *ptr, uint16_t old, uint16_t new)
265 {
266
267 return (*_atomic_cas_16_fn)(ptr, old, new);
268 }
269
270 uint8_t _atomic_cas_8(volatile uint8_t *, uint8_t, uint8_t);
271
272 uint8_t
_atomic_cas_8(volatile uint8_t * ptr,uint8_t old,uint8_t new)273 _atomic_cas_8(volatile uint8_t *ptr, uint8_t old, uint8_t new)
274 {
275
276 return (*_atomic_cas_8_fn)(ptr, old, new);
277 }
278
279 void __section(".text.startup")
__libc_atomic_init(void)280 __libc_atomic_init(void)
281 {
282 #if !defined(__minix)
283 int ncpu, mib[2];
284 size_t len;
285 #endif /* !defined(__minix) */
286
287 _atomic_cas_fn = _atomic_cas_mp;
288 #ifdef __HAVE_ATOMIC_CAS_64_UP
289 _atomic_cas_64_fn = _atomic_cas_64_mp;
290 #endif
291 _atomic_cas_16_fn = _atomic_cas_16_mp;
292 _atomic_cas_8_fn = _atomic_cas_8_mp;
293
294 #if !defined(__minix)
295 mib[0] = CTL_HW;
296 mib[1] = HW_NCPU;
297 len = sizeof(ncpu);
298 if (sysctl(mib, 2, &ncpu, &len, NULL, 0) == -1)
299 return;
300 if (ncpu > 1)
301 return;
302 if (rasctl(RAS_ADDR(_atomic_cas), RAS_SIZE(_atomic_cas),
303 RAS_INSTALL) == 0) {
304 _atomic_cas_fn = _atomic_cas_up;
305 return;
306 }
307
308 #ifdef __HAVE_ATOMIC_CAS_64_UP
309 if (rasctl(RAS_ADDR(_atomic_cas_64), RAS_SIZE(_atomic_cas_64),
310 RAS_INSTALL) == 0) {
311 _atomic_cas_64_fn = _atomic_cas_64_up;
312 return;
313 }
314 #endif
315
316 if (rasctl(RAS_ADDR(_atomic_cas_16), RAS_SIZE(_atomic_cas_16),
317 RAS_INSTALL) == 0) {
318 _atomic_cas_16_fn = _atomic_cas_16_up;
319 return;
320 }
321
322 if (rasctl(RAS_ADDR(_atomic_cas_8), RAS_SIZE(_atomic_cas_8),
323 RAS_INSTALL) == 0) {
324 _atomic_cas_8_fn = _atomic_cas_8_up;
325 return;
326 }
327 #endif /* !defined(__minix) */
328 }
329
330 #undef atomic_cas_32
331 #undef atomic_cas_uint
332 #undef atomic_cas_ulong
333 #undef atomic_cas_ptr
334 #undef atomic_cas_32_ni
335 #undef atomic_cas_uint_ni
336 #undef atomic_cas_ulong_ni
337 #undef atomic_cas_ptr_ni
338
339 atomic_op_alias(atomic_cas_32,_atomic_cas_32)
340 atomic_op_alias(atomic_cas_uint,_atomic_cas_32)
341 __strong_alias(_atomic_cas_uint,_atomic_cas_32)
342 atomic_op_alias(atomic_cas_ulong,_atomic_cas_32)
343 __strong_alias(_atomic_cas_ulong,_atomic_cas_32)
344 atomic_op_alias(atomic_cas_ptr,_atomic_cas_32)
345 __strong_alias(_atomic_cas_ptr,_atomic_cas_32)
346
347 atomic_op_alias(atomic_cas_32_ni,_atomic_cas_32)
348 __strong_alias(_atomic_cas_32_ni,_atomic_cas_32)
349 atomic_op_alias(atomic_cas_uint_ni,_atomic_cas_32)
350 __strong_alias(_atomic_cas_uint_ni,_atomic_cas_32)
351 atomic_op_alias(atomic_cas_ulong_ni,_atomic_cas_32)
352 __strong_alias(_atomic_cas_ulong_ni,_atomic_cas_32)
353 atomic_op_alias(atomic_cas_ptr_ni,_atomic_cas_32)
354 __strong_alias(_atomic_cas_ptr_ni,_atomic_cas_32)
355
356 //atomic_op_alias(atomic_cas_16,_atomic_cas_16)
357 //atomic_op_alias(atomic_cas_16_ni,_atomic_cas_16)
358 //atomic_op_alias(atomic_cas_8,_atomic_cas_8)
359 //atomic_op_alias(atomic_cas_8_ni,_atomic_cas_8)
360 #ifdef __HAVE_ATOMIC_CAS_64_UP
361 //atomic_op_alias(atomic_cas_64_ni,_atomic_cas_64)
362 crt_alias(__sync_val_compare_and_swap_8,_atomic_cas_64)
363 #endif
364 crt_alias(__sync_val_compare_and_swap_4,_atomic_cas_32)
365 crt_alias(__sync_val_compare_and_swap_2,_atomic_cas_16)
366 crt_alias(__sync_val_compare_and_swap_1,_atomic_cas_8)
367