1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2008 Marcel Moolenaar
5 * Copyright (c) 2001 Benno Rice
6 * Copyright (c) 2001 David E. O'Brien
7 * Copyright (c) 1998 Doug Rabson
8 * All rights reserved.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * SUCH DAMAGE.
30 */
31
32 #ifndef _MACHINE_ATOMIC_H_
33 #define _MACHINE_ATOMIC_H_
34
35 #include <sys/atomic_common.h>
36
37 #ifndef __powerpc64__
38 #include <sys/_atomic64e.h>
39 #endif
40
41 /*
42 * The __ATOMIC_REL/ACQ() macros provide memory barriers only in conjunction
43 * with the atomic lXarx/stXcx. sequences below. They are not exposed outside
44 * of this file. See also Appendix B.2 of Book II of the architecture manual.
45 *
46 * Note that not all Book-E processors accept the light-weight sync variant.
47 * In particular, early models of E500 cores are known to wedge. Bank on all
48 * 64-bit capable CPUs to accept lwsync properly and pressimize 32-bit CPUs
49 * to use the heavier-weight sync.
50 */
51
52 #ifdef __powerpc64__
53 #define mb() __asm __volatile("sync" : : : "memory")
54 #define rmb() __asm __volatile("lwsync" : : : "memory")
55 #define wmb() __asm __volatile("lwsync" : : : "memory")
56 #define __ATOMIC_REL() __asm __volatile("lwsync" : : : "memory")
57 #define __ATOMIC_ACQ() __asm __volatile("isync" : : : "memory")
58 #else
59 #define mb() __asm __volatile("sync" : : : "memory")
60 #define rmb() __asm __volatile("sync" : : : "memory")
61 #define wmb() __asm __volatile("sync" : : : "memory")
62 #define __ATOMIC_REL() __asm __volatile("sync" : : : "memory")
63 #define __ATOMIC_ACQ() __asm __volatile("isync" : : : "memory")
64 #endif
65
66 static __inline void
powerpc_lwsync(void)67 powerpc_lwsync(void)
68 {
69
70 #ifdef __powerpc64__
71 __asm __volatile("lwsync" : : : "memory");
72 #else
73 __asm __volatile("sync" : : : "memory");
74 #endif
75 }
76
77 /*
78 * atomic_add(p, v)
79 * { *p += v; }
80 */
81
82 #define __atomic_add_int(p, v, t) \
83 __asm __volatile( \
84 "1: lwarx %0, 0, %2\n" \
85 " add %0, %3, %0\n" \
86 " stwcx. %0, 0, %2\n" \
87 " bne- 1b\n" \
88 : "=&r" (t), "=m" (*p) \
89 : "r" (p), "r" (v), "m" (*p) \
90 : "cr0", "memory") \
91 /* __atomic_add_int */
92
93 #ifdef __powerpc64__
94 #define __atomic_add_long(p, v, t) \
95 __asm __volatile( \
96 "1: ldarx %0, 0, %2\n" \
97 " add %0, %3, %0\n" \
98 " stdcx. %0, 0, %2\n" \
99 " bne- 1b\n" \
100 : "=&r" (t), "=m" (*p) \
101 : "r" (p), "r" (v), "m" (*p) \
102 : "cr0", "memory") \
103 /* __atomic_add_long */
104 #else
105 #define __atomic_add_long(p, v, t) \
106 __asm __volatile( \
107 "1: lwarx %0, 0, %2\n" \
108 " add %0, %3, %0\n" \
109 " stwcx. %0, 0, %2\n" \
110 " bne- 1b\n" \
111 : "=&r" (t), "=m" (*p) \
112 : "r" (p), "r" (v), "m" (*p) \
113 : "cr0", "memory") \
114 /* __atomic_add_long */
115 #endif
116
117 #define _ATOMIC_ADD(type) \
118 static __inline void \
119 atomic_add_##type(volatile u_##type *p, u_##type v) { \
120 u_##type t; \
121 __atomic_add_##type(p, v, t); \
122 } \
123 \
124 static __inline void \
125 atomic_add_acq_##type(volatile u_##type *p, u_##type v) { \
126 u_##type t; \
127 __atomic_add_##type(p, v, t); \
128 __ATOMIC_ACQ(); \
129 } \
130 \
131 static __inline void \
132 atomic_add_rel_##type(volatile u_##type *p, u_##type v) { \
133 u_##type t; \
134 __ATOMIC_REL(); \
135 __atomic_add_##type(p, v, t); \
136 } \
137 /* _ATOMIC_ADD */
138
139 _ATOMIC_ADD(int)
_ATOMIC_ADD(long)140 _ATOMIC_ADD(long)
141
142 #define atomic_add_32 atomic_add_int
143 #define atomic_add_acq_32 atomic_add_acq_int
144 #define atomic_add_rel_32 atomic_add_rel_int
145
146 #ifdef __powerpc64__
147 #define atomic_add_64 atomic_add_long
148 #define atomic_add_acq_64 atomic_add_acq_long
149 #define atomic_add_rel_64 atomic_add_rel_long
150
151 #define atomic_add_ptr atomic_add_long
152 #define atomic_add_acq_ptr atomic_add_acq_long
153 #define atomic_add_rel_ptr atomic_add_rel_long
154 #else
155 #define atomic_add_ptr atomic_add_int
156 #define atomic_add_acq_ptr atomic_add_acq_int
157 #define atomic_add_rel_ptr atomic_add_rel_int
158 #endif
159 #undef _ATOMIC_ADD
160 #undef __atomic_add_long
161 #undef __atomic_add_int
162
163 /*
164 * atomic_clear(p, v)
165 * { *p &= ~v; }
166 */
167
168 #define __atomic_clear_int(p, v, t) \
169 __asm __volatile( \
170 "1: lwarx %0, 0, %2\n" \
171 " andc %0, %0, %3\n" \
172 " stwcx. %0, 0, %2\n" \
173 " bne- 1b\n" \
174 : "=&r" (t), "=m" (*p) \
175 : "r" (p), "r" (v), "m" (*p) \
176 : "cr0", "memory") \
177 /* __atomic_clear_int */
178
179 #ifdef __powerpc64__
180 #define __atomic_clear_long(p, v, t) \
181 __asm __volatile( \
182 "1: ldarx %0, 0, %2\n" \
183 " andc %0, %0, %3\n" \
184 " stdcx. %0, 0, %2\n" \
185 " bne- 1b\n" \
186 : "=&r" (t), "=m" (*p) \
187 : "r" (p), "r" (v), "m" (*p) \
188 : "cr0", "memory") \
189 /* __atomic_clear_long */
190 #else
191 #define __atomic_clear_long(p, v, t) \
192 __asm __volatile( \
193 "1: lwarx %0, 0, %2\n" \
194 " andc %0, %0, %3\n" \
195 " stwcx. %0, 0, %2\n" \
196 " bne- 1b\n" \
197 : "=&r" (t), "=m" (*p) \
198 : "r" (p), "r" (v), "m" (*p) \
199 : "cr0", "memory") \
200 /* __atomic_clear_long */
201 #endif
202
203 #define _ATOMIC_CLEAR(type) \
204 static __inline void \
205 atomic_clear_##type(volatile u_##type *p, u_##type v) { \
206 u_##type t; \
207 __atomic_clear_##type(p, v, t); \
208 } \
209 \
210 static __inline void \
211 atomic_clear_acq_##type(volatile u_##type *p, u_##type v) { \
212 u_##type t; \
213 __atomic_clear_##type(p, v, t); \
214 __ATOMIC_ACQ(); \
215 } \
216 \
217 static __inline void \
218 atomic_clear_rel_##type(volatile u_##type *p, u_##type v) { \
219 u_##type t; \
220 __ATOMIC_REL(); \
221 __atomic_clear_##type(p, v, t); \
222 } \
223 /* _ATOMIC_CLEAR */
224
225 _ATOMIC_CLEAR(int)
226 _ATOMIC_CLEAR(long)
227
228 #define atomic_clear_32 atomic_clear_int
229 #define atomic_clear_acq_32 atomic_clear_acq_int
230 #define atomic_clear_rel_32 atomic_clear_rel_int
231
232 #ifdef __powerpc64__
233 #define atomic_clear_64 atomic_clear_long
234 #define atomic_clear_acq_64 atomic_clear_acq_long
235 #define atomic_clear_rel_64 atomic_clear_rel_long
236
237 #define atomic_clear_ptr atomic_clear_long
238 #define atomic_clear_acq_ptr atomic_clear_acq_long
239 #define atomic_clear_rel_ptr atomic_clear_rel_long
240 #else
241 #define atomic_clear_ptr atomic_clear_int
242 #define atomic_clear_acq_ptr atomic_clear_acq_int
243 #define atomic_clear_rel_ptr atomic_clear_rel_int
244 #endif
245 #undef _ATOMIC_CLEAR
246 #undef __atomic_clear_long
247 #undef __atomic_clear_int
248
249 /*
250 * atomic_cmpset(p, o, n)
251 */
252 /* TODO -- see below */
253
254 /*
255 * atomic_load_acq(p)
256 */
257 /* TODO -- see below */
258
259 /*
260 * atomic_readandclear(p)
261 */
262 /* TODO -- see below */
263
264 /*
265 * atomic_set(p, v)
266 * { *p |= v; }
267 */
268
269 #define __atomic_set_int(p, v, t) \
270 __asm __volatile( \
271 "1: lwarx %0, 0, %2\n" \
272 " or %0, %3, %0\n" \
273 " stwcx. %0, 0, %2\n" \
274 " bne- 1b\n" \
275 : "=&r" (t), "=m" (*p) \
276 : "r" (p), "r" (v), "m" (*p) \
277 : "cr0", "memory") \
278 /* __atomic_set_int */
279
280 #ifdef __powerpc64__
281 #define __atomic_set_long(p, v, t) \
282 __asm __volatile( \
283 "1: ldarx %0, 0, %2\n" \
284 " or %0, %3, %0\n" \
285 " stdcx. %0, 0, %2\n" \
286 " bne- 1b\n" \
287 : "=&r" (t), "=m" (*p) \
288 : "r" (p), "r" (v), "m" (*p) \
289 : "cr0", "memory") \
290 /* __atomic_set_long */
291 #else
292 #define __atomic_set_long(p, v, t) \
293 __asm __volatile( \
294 "1: lwarx %0, 0, %2\n" \
295 " or %0, %3, %0\n" \
296 " stwcx. %0, 0, %2\n" \
297 " bne- 1b\n" \
298 : "=&r" (t), "=m" (*p) \
299 : "r" (p), "r" (v), "m" (*p) \
300 : "cr0", "memory") \
301 /* __atomic_set_long */
302 #endif
303
304 #define _ATOMIC_SET(type) \
305 static __inline void \
306 atomic_set_##type(volatile u_##type *p, u_##type v) { \
307 u_##type t; \
308 __atomic_set_##type(p, v, t); \
309 } \
310 \
311 static __inline void \
312 atomic_set_acq_##type(volatile u_##type *p, u_##type v) { \
313 u_##type t; \
314 __atomic_set_##type(p, v, t); \
315 __ATOMIC_ACQ(); \
316 } \
317 \
318 static __inline void \
319 atomic_set_rel_##type(volatile u_##type *p, u_##type v) { \
320 u_##type t; \
321 __ATOMIC_REL(); \
322 __atomic_set_##type(p, v, t); \
323 } \
324 /* _ATOMIC_SET */
325
326 _ATOMIC_SET(int)
327 _ATOMIC_SET(long)
328
329 #define atomic_set_32 atomic_set_int
330 #define atomic_set_acq_32 atomic_set_acq_int
331 #define atomic_set_rel_32 atomic_set_rel_int
332
333 #ifdef __powerpc64__
334 #define atomic_set_64 atomic_set_long
335 #define atomic_set_acq_64 atomic_set_acq_long
336 #define atomic_set_rel_64 atomic_set_rel_long
337
338 #define atomic_set_ptr atomic_set_long
339 #define atomic_set_acq_ptr atomic_set_acq_long
340 #define atomic_set_rel_ptr atomic_set_rel_long
341 #else
342 #define atomic_set_ptr atomic_set_int
343 #define atomic_set_acq_ptr atomic_set_acq_int
344 #define atomic_set_rel_ptr atomic_set_rel_int
345 #endif
346 #undef _ATOMIC_SET
347 #undef __atomic_set_long
348 #undef __atomic_set_int
349
350 /*
351 * atomic_subtract(p, v)
352 * { *p -= v; }
353 */
354
355 #define __atomic_subtract_int(p, v, t) \
356 __asm __volatile( \
357 "1: lwarx %0, 0, %2\n" \
358 " subf %0, %3, %0\n" \
359 " stwcx. %0, 0, %2\n" \
360 " bne- 1b\n" \
361 : "=&r" (t), "=m" (*p) \
362 : "r" (p), "r" (v), "m" (*p) \
363 : "cr0", "memory") \
364 /* __atomic_subtract_int */
365
366 #ifdef __powerpc64__
367 #define __atomic_subtract_long(p, v, t) \
368 __asm __volatile( \
369 "1: ldarx %0, 0, %2\n" \
370 " subf %0, %3, %0\n" \
371 " stdcx. %0, 0, %2\n" \
372 " bne- 1b\n" \
373 : "=&r" (t), "=m" (*p) \
374 : "r" (p), "r" (v), "m" (*p) \
375 : "cr0", "memory") \
376 /* __atomic_subtract_long */
377 #else
378 #define __atomic_subtract_long(p, v, t) \
379 __asm __volatile( \
380 "1: lwarx %0, 0, %2\n" \
381 " subf %0, %3, %0\n" \
382 " stwcx. %0, 0, %2\n" \
383 " bne- 1b\n" \
384 : "=&r" (t), "=m" (*p) \
385 : "r" (p), "r" (v), "m" (*p) \
386 : "cr0", "memory") \
387 /* __atomic_subtract_long */
388 #endif
389
390 #define _ATOMIC_SUBTRACT(type) \
391 static __inline void \
392 atomic_subtract_##type(volatile u_##type *p, u_##type v) { \
393 u_##type t; \
394 __atomic_subtract_##type(p, v, t); \
395 } \
396 \
397 static __inline void \
398 atomic_subtract_acq_##type(volatile u_##type *p, u_##type v) { \
399 u_##type t; \
400 __atomic_subtract_##type(p, v, t); \
401 __ATOMIC_ACQ(); \
402 } \
403 \
404 static __inline void \
405 atomic_subtract_rel_##type(volatile u_##type *p, u_##type v) { \
406 u_##type t; \
407 __ATOMIC_REL(); \
408 __atomic_subtract_##type(p, v, t); \
409 } \
410 /* _ATOMIC_SUBTRACT */
411
412 _ATOMIC_SUBTRACT(int)
413 _ATOMIC_SUBTRACT(long)
414
415 #define atomic_subtract_32 atomic_subtract_int
416 #define atomic_subtract_acq_32 atomic_subtract_acq_int
417 #define atomic_subtract_rel_32 atomic_subtract_rel_int
418
419 #ifdef __powerpc64__
420 #define atomic_subtract_64 atomic_subtract_long
421 #define atomic_subtract_acq_64 atomic_subract_acq_long
422 #define atomic_subtract_rel_64 atomic_subtract_rel_long
423
424 #define atomic_subtract_ptr atomic_subtract_long
425 #define atomic_subtract_acq_ptr atomic_subtract_acq_long
426 #define atomic_subtract_rel_ptr atomic_subtract_rel_long
427 #else
428 #define atomic_subtract_ptr atomic_subtract_int
429 #define atomic_subtract_acq_ptr atomic_subtract_acq_int
430 #define atomic_subtract_rel_ptr atomic_subtract_rel_int
431 #endif
432 #undef _ATOMIC_SUBTRACT
433 #undef __atomic_subtract_long
434 #undef __atomic_subtract_int
435
436 /*
437 * atomic_store_rel(p, v)
438 */
439 /* TODO -- see below */
440
441 /*
442 * Old/original implementations that still need revisiting.
443 */
444
445 static __inline u_int
446 atomic_readandclear_int(volatile u_int *addr)
447 {
448 u_int result,temp;
449
450 __asm __volatile (
451 "\tsync\n" /* drain writes */
452 "1:\tlwarx %0, 0, %3\n\t" /* load old value */
453 "li %1, 0\n\t" /* load new value */
454 "stwcx. %1, 0, %3\n\t" /* attempt to store */
455 "bne- 1b\n\t" /* spin if failed */
456 : "=&r"(result), "=&r"(temp), "=m" (*addr)
457 : "r" (addr), "m" (*addr)
458 : "cr0", "memory");
459
460 return (result);
461 }
462
463 #ifdef __powerpc64__
464 static __inline u_long
atomic_readandclear_long(volatile u_long * addr)465 atomic_readandclear_long(volatile u_long *addr)
466 {
467 u_long result,temp;
468
469 __asm __volatile (
470 "\tsync\n" /* drain writes */
471 "1:\tldarx %0, 0, %3\n\t" /* load old value */
472 "li %1, 0\n\t" /* load new value */
473 "stdcx. %1, 0, %3\n\t" /* attempt to store */
474 "bne- 1b\n\t" /* spin if failed */
475 : "=&r"(result), "=&r"(temp), "=m" (*addr)
476 : "r" (addr), "m" (*addr)
477 : "cr0", "memory");
478
479 return (result);
480 }
481 #endif
482
483 #define atomic_readandclear_32 atomic_readandclear_int
484
485 #ifdef __powerpc64__
486 #define atomic_readandclear_64 atomic_readandclear_long
487
488 #define atomic_readandclear_ptr atomic_readandclear_long
489 #else
490 static __inline u_long
atomic_readandclear_long(volatile u_long * addr)491 atomic_readandclear_long(volatile u_long *addr)
492 {
493
494 return ((u_long)atomic_readandclear_int((volatile u_int *)addr));
495 }
496
497 #define atomic_readandclear_ptr atomic_readandclear_int
498 #endif
499
500 /*
501 * We assume that a = b will do atomic loads and stores.
502 */
503 #define ATOMIC_STORE_LOAD(TYPE) \
504 static __inline u_##TYPE \
505 atomic_load_acq_##TYPE(volatile u_##TYPE *p) \
506 { \
507 u_##TYPE v; \
508 \
509 v = *p; \
510 powerpc_lwsync(); \
511 return (v); \
512 } \
513 \
514 static __inline void \
515 atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \
516 { \
517 \
518 powerpc_lwsync(); \
519 *p = v; \
520 }
521
522 ATOMIC_STORE_LOAD(int)
523
524 #define atomic_load_acq_32 atomic_load_acq_int
525 #define atomic_store_rel_32 atomic_store_rel_int
526
527 #ifdef __powerpc64__
ATOMIC_STORE_LOAD(long)528 ATOMIC_STORE_LOAD(long)
529
530 #define atomic_load_acq_64 atomic_load_acq_long
531 #define atomic_store_rel_64 atomic_store_rel_long
532
533 #define atomic_load_acq_ptr atomic_load_acq_long
534 #define atomic_store_rel_ptr atomic_store_rel_long
535 #else
536 static __inline u_long
537 atomic_load_acq_long(volatile u_long *addr)
538 {
539
540 return ((u_long)atomic_load_acq_int((volatile u_int *)addr));
541 }
542
543 static __inline void
544 atomic_store_rel_long(volatile u_long *addr, u_long val)
545 {
546
547 atomic_store_rel_int((volatile u_int *)addr, (u_int)val);
548 }
549
550 #define atomic_load_acq_ptr atomic_load_acq_int
551 #define atomic_store_rel_ptr atomic_store_rel_int
552 #endif
553 #undef ATOMIC_STORE_LOAD
554
555 /*
556 * Atomically compare the value stored at *p with cmpval and if the
557 * two values are equal, update the value of *p with newval. Returns
558 * zero if the compare failed, nonzero otherwise.
559 */
560 #ifdef ISA_206_ATOMICS
561 static __inline int
562 atomic_cmpset_char(volatile u_char *p, u_char cmpval, u_char newval)
563 {
564 int ret;
565
566 __asm __volatile (
567 "1:\tlbarx %0, 0, %2\n\t" /* load old value */
568 "cmplw %3, %0\n\t" /* compare */
569 "bne- 2f\n\t" /* exit if not equal */
570 "stbcx. %4, 0, %2\n\t" /* attempt to store */
571 "bne- 1b\n\t" /* spin if failed */
572 "li %0, 1\n\t" /* success - retval = 1 */
573 "b 3f\n\t" /* we've succeeded */
574 "2:\n\t"
575 "stbcx. %0, 0, %2\n\t" /* clear reservation (74xx) */
576 "li %0, 0\n\t" /* failure - retval = 0 */
577 "3:\n\t"
578 : "=&r" (ret), "=m" (*p)
579 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p)
580 : "cr0", "memory");
581
582 return (ret);
583 }
584
585 static __inline int
atomic_cmpset_short(volatile u_short * p,u_short cmpval,u_short newval)586 atomic_cmpset_short(volatile u_short *p, u_short cmpval, u_short newval)
587 {
588 int ret;
589
590 __asm __volatile (
591 "1:\tlharx %0, 0, %2\n\t" /* load old value */
592 "cmplw %3, %0\n\t" /* compare */
593 "bne- 2f\n\t" /* exit if not equal */
594 "sthcx. %4, 0, %2\n\t" /* attempt to store */
595 "bne- 1b\n\t" /* spin if failed */
596 "li %0, 1\n\t" /* success - retval = 1 */
597 "b 3f\n\t" /* we've succeeded */
598 "2:\n\t"
599 "sthcx. %0, 0, %2\n\t" /* clear reservation (74xx) */
600 "li %0, 0\n\t" /* failure - retval = 0 */
601 "3:\n\t"
602 : "=&r" (ret), "=m" (*p)
603 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p)
604 : "cr0", "memory");
605
606 return (ret);
607 }
608 #else
609 static __inline int
610 atomic_cmpset_masked(uint32_t *p, uint32_t cmpval, uint32_t newval,
611 uint32_t mask)
612 {
613 int ret;
614 uint32_t tmp;
615
616 __asm __volatile (
617 "1:\tlwarx %2, 0, %3\n\t" /* load old value */
618 "and %0, %2, %7\n\t"
619 "cmplw %4, %0\n\t" /* compare */
620 "bne- 2f\n\t" /* exit if not equal */
621 "andc %2, %2, %7\n\t"
622 "or %2, %2, %5\n\t"
623 "stwcx. %2, 0, %3\n\t" /* attempt to store */
624 "bne- 1b\n\t" /* spin if failed */
625 "li %0, 1\n\t" /* success - retval = 1 */
626 "b 3f\n\t" /* we've succeeded */
627 "2:\n\t"
628 "stwcx. %2, 0, %3\n\t" /* clear reservation (74xx) */
629 "li %0, 0\n\t" /* failure - retval = 0 */
630 "3:\n\t"
631 : "=&r" (ret), "=m" (*p), "+&r" (tmp)
632 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p),
633 "r" (mask)
634 : "cr0", "memory");
635
636 return (ret);
637 }
638
639 #define _atomic_cmpset_masked_word(a,o,v,m) atomic_cmpset_masked(a, o, v, m)
640 #endif
641
642 static __inline int
atomic_cmpset_int(volatile u_int * p,u_int cmpval,u_int newval)643 atomic_cmpset_int(volatile u_int* p, u_int cmpval, u_int newval)
644 {
645 int ret;
646
647 __asm __volatile (
648 "1:\tlwarx %0, 0, %2\n\t" /* load old value */
649 "cmplw %3, %0\n\t" /* compare */
650 "bne- 2f\n\t" /* exit if not equal */
651 "stwcx. %4, 0, %2\n\t" /* attempt to store */
652 "bne- 1b\n\t" /* spin if failed */
653 "li %0, 1\n\t" /* success - retval = 1 */
654 "b 3f\n\t" /* we've succeeded */
655 "2:\n\t"
656 "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */
657 "li %0, 0\n\t" /* failure - retval = 0 */
658 "3:\n\t"
659 : "=&r" (ret), "=m" (*p)
660 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p)
661 : "cr0", "memory");
662
663 return (ret);
664 }
665 static __inline int
atomic_cmpset_long(volatile u_long * p,u_long cmpval,u_long newval)666 atomic_cmpset_long(volatile u_long* p, u_long cmpval, u_long newval)
667 {
668 int ret;
669
670 __asm __volatile (
671 #ifdef __powerpc64__
672 "1:\tldarx %0, 0, %2\n\t" /* load old value */
673 "cmpld %3, %0\n\t" /* compare */
674 "bne- 2f\n\t" /* exit if not equal */
675 "stdcx. %4, 0, %2\n\t" /* attempt to store */
676 #else
677 "1:\tlwarx %0, 0, %2\n\t" /* load old value */
678 "cmplw %3, %0\n\t" /* compare */
679 "bne- 2f\n\t" /* exit if not equal */
680 "stwcx. %4, 0, %2\n\t" /* attempt to store */
681 #endif
682 "bne- 1b\n\t" /* spin if failed */
683 "li %0, 1\n\t" /* success - retval = 1 */
684 "b 3f\n\t" /* we've succeeded */
685 "2:\n\t"
686 #ifdef __powerpc64__
687 "stdcx. %0, 0, %2\n\t" /* clear reservation (74xx) */
688 #else
689 "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */
690 #endif
691 "li %0, 0\n\t" /* failure - retval = 0 */
692 "3:\n\t"
693 : "=&r" (ret), "=m" (*p)
694 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p)
695 : "cr0", "memory");
696
697 return (ret);
698 }
699
700 #define ATOMIC_CMPSET_ACQ_REL(type) \
701 static __inline int \
702 atomic_cmpset_acq_##type(volatile u_##type *p, \
703 u_##type cmpval, u_##type newval)\
704 {\
705 u_##type retval; \
706 retval = atomic_cmpset_##type(p, cmpval, newval);\
707 __ATOMIC_ACQ();\
708 return (retval);\
709 }\
710 static __inline int \
711 atomic_cmpset_rel_##type(volatile u_##type *p, \
712 u_##type cmpval, u_##type newval)\
713 {\
714 __ATOMIC_REL();\
715 return (atomic_cmpset_##type(p, cmpval, newval));\
716 }\
717 struct hack
718
719 ATOMIC_CMPSET_ACQ_REL(int);
720 ATOMIC_CMPSET_ACQ_REL(long);
721
722 #ifdef ISA_206_ATOMICS
723 #define atomic_cmpset_8 atomic_cmpset_char
724 #endif
725 #define atomic_cmpset_acq_8 atomic_cmpset_acq_char
726 #define atomic_cmpset_rel_8 atomic_cmpset_rel_char
727
728 #ifdef ISA_206_ATOMICS
729 #define atomic_cmpset_16 atomic_cmpset_short
730 #endif
731 #define atomic_cmpset_acq_16 atomic_cmpset_acq_short
732 #define atomic_cmpset_rel_16 atomic_cmpset_rel_short
733
734 #define atomic_cmpset_32 atomic_cmpset_int
735 #define atomic_cmpset_acq_32 atomic_cmpset_acq_int
736 #define atomic_cmpset_rel_32 atomic_cmpset_rel_int
737
738 #ifdef __powerpc64__
739 #define atomic_cmpset_64 atomic_cmpset_long
740 #define atomic_cmpset_acq_64 atomic_cmpset_acq_long
741 #define atomic_cmpset_rel_64 atomic_cmpset_rel_long
742
743 #define atomic_cmpset_ptr atomic_cmpset_long
744 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_long
745 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_long
746 #else
747 #define atomic_cmpset_ptr atomic_cmpset_int
748 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_int
749 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_int
750 #endif
751
752 /*
753 * Atomically compare the value stored at *p with *cmpval and if the
754 * two values are equal, update the value of *p with newval. Returns
755 * zero if the compare failed and sets *cmpval to the read value from *p,
756 * nonzero otherwise.
757 */
758 #ifdef ISA_206_ATOMICS
759 static __inline int
atomic_fcmpset_char(volatile u_char * p,u_char * cmpval,u_char newval)760 atomic_fcmpset_char(volatile u_char *p, u_char *cmpval, u_char newval)
761 {
762 int ret;
763
764 __asm __volatile (
765 "lbarx %0, 0, %3\n\t" /* load old value */
766 "cmplw %4, %0\n\t" /* compare */
767 "bne- 1f\n\t" /* exit if not equal */
768 "stbcx. %5, 0, %3\n\t" /* attempt to store */
769 "bne- 1f\n\t" /* exit if failed */
770 "li %0, 1\n\t" /* success - retval = 1 */
771 "b 2f\n\t" /* we've succeeded */
772 "1:\n\t"
773 "stbcx. %0, 0, %3\n\t" /* clear reservation (74xx) */
774 "stbx %0, 0, %7\n\t"
775 "li %0, 0\n\t" /* failure - retval = 0 */
776 "2:\n\t"
777 : "=&r" (ret), "=m" (*p), "=m" (*cmpval)
778 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval)
779 : "cr0", "memory");
780
781 return (ret);
782 }
783
784 static __inline int
atomic_fcmpset_short(volatile u_short * p,u_short * cmpval,u_short newval)785 atomic_fcmpset_short(volatile u_short *p, u_short *cmpval, u_short newval)
786 {
787 int ret;
788
789 __asm __volatile (
790 "lharx %0, 0, %3\n\t" /* load old value */
791 "cmplw %4, %0\n\t" /* compare */
792 "bne- 1f\n\t" /* exit if not equal */
793 "sthcx. %5, 0, %3\n\t" /* attempt to store */
794 "bne- 1f\n\t" /* exit if failed */
795 "li %0, 1\n\t" /* success - retval = 1 */
796 "b 2f\n\t" /* we've succeeded */
797 "1:\n\t"
798 "sthcx. %0, 0, %3\n\t" /* clear reservation (74xx) */
799 "sthx %0, 0, %7\n\t"
800 "li %0, 0\n\t" /* failure - retval = 0 */
801 "2:\n\t"
802 : "=&r" (ret), "=m" (*p), "=m" (*cmpval)
803 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval)
804 : "cr0", "memory");
805
806 return (ret);
807 }
808 #endif /* ISA_206_ATOMICS */
809
810 static __inline int
atomic_fcmpset_int(volatile u_int * p,u_int * cmpval,u_int newval)811 atomic_fcmpset_int(volatile u_int *p, u_int *cmpval, u_int newval)
812 {
813 int ret;
814
815 __asm __volatile (
816 "lwarx %0, 0, %3\n\t" /* load old value */
817 "cmplw %4, %0\n\t" /* compare */
818 "bne- 1f\n\t" /* exit if not equal */
819 "stwcx. %5, 0, %3\n\t" /* attempt to store */
820 "bne- 1f\n\t" /* exit if failed */
821 "li %0, 1\n\t" /* success - retval = 1 */
822 "b 2f\n\t" /* we've succeeded */
823 "1:\n\t"
824 "stwcx. %0, 0, %3\n\t" /* clear reservation (74xx) */
825 "stwx %0, 0, %7\n\t"
826 "li %0, 0\n\t" /* failure - retval = 0 */
827 "2:\n\t"
828 : "=&r" (ret), "=m" (*p), "=m" (*cmpval)
829 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval)
830 : "cr0", "memory");
831
832 return (ret);
833 }
834 static __inline int
atomic_fcmpset_long(volatile u_long * p,u_long * cmpval,u_long newval)835 atomic_fcmpset_long(volatile u_long *p, u_long *cmpval, u_long newval)
836 {
837 int ret;
838
839 __asm __volatile (
840 #ifdef __powerpc64__
841 "ldarx %0, 0, %3\n\t" /* load old value */
842 "cmpld %4, %0\n\t" /* compare */
843 "bne- 1f\n\t" /* exit if not equal */
844 "stdcx. %5, 0, %3\n\t" /* attempt to store */
845 #else
846 "lwarx %0, 0, %3\n\t" /* load old value */
847 "cmplw %4, %0\n\t" /* compare */
848 "bne- 1f\n\t" /* exit if not equal */
849 "stwcx. %5, 0, %3\n\t" /* attempt to store */
850 #endif
851 "bne- 1f\n\t" /* exit if failed */
852 "li %0, 1\n\t" /* success - retval = 1 */
853 "b 2f\n\t" /* we've succeeded */
854 "1:\n\t"
855 #ifdef __powerpc64__
856 "stdcx. %0, 0, %3\n\t" /* clear reservation (74xx) */
857 "stdx %0, 0, %7\n\t"
858 #else
859 "stwcx. %0, 0, %3\n\t" /* clear reservation (74xx) */
860 "stwx %0, 0, %7\n\t"
861 #endif
862 "li %0, 0\n\t" /* failure - retval = 0 */
863 "2:\n\t"
864 : "=&r" (ret), "=m" (*p), "=m" (*cmpval)
865 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval)
866 : "cr0", "memory");
867
868 return (ret);
869 }
870
871 #define ATOMIC_FCMPSET_ACQ_REL(type) \
872 static __inline int \
873 atomic_fcmpset_acq_##type(volatile u_##type *p, \
874 u_##type *cmpval, u_##type newval)\
875 {\
876 u_##type retval; \
877 retval = atomic_fcmpset_##type(p, cmpval, newval);\
878 __ATOMIC_ACQ();\
879 return (retval);\
880 }\
881 static __inline int \
882 atomic_fcmpset_rel_##type(volatile u_##type *p, \
883 u_##type *cmpval, u_##type newval)\
884 {\
885 __ATOMIC_REL();\
886 return (atomic_fcmpset_##type(p, cmpval, newval));\
887 }\
888 struct hack
889
890 ATOMIC_FCMPSET_ACQ_REL(int);
891 ATOMIC_FCMPSET_ACQ_REL(long);
892
893 #ifdef ISA_206_ATOMICS
894 #define atomic_fcmpset_8 atomic_fcmpset_char
895 #endif
896 #define atomic_fcmpset_acq_8 atomic_fcmpset_acq_char
897 #define atomic_fcmpset_rel_8 atomic_fcmpset_rel_char
898
899 #ifdef ISA_206_ATOMICS
900 #define atomic_fcmpset_16 atomic_fcmpset_short
901 #endif
902 #define atomic_fcmpset_acq_16 atomic_fcmpset_acq_short
903 #define atomic_fcmpset_rel_16 atomic_fcmpset_rel_short
904
905 #define atomic_fcmpset_32 atomic_fcmpset_int
906 #define atomic_fcmpset_acq_32 atomic_fcmpset_acq_int
907 #define atomic_fcmpset_rel_32 atomic_fcmpset_rel_int
908
909 #ifdef __powerpc64__
910 #define atomic_fcmpset_64 atomic_fcmpset_long
911 #define atomic_fcmpset_acq_64 atomic_fcmpset_acq_long
912 #define atomic_fcmpset_rel_64 atomic_fcmpset_rel_long
913
914 #define atomic_fcmpset_ptr atomic_fcmpset_long
915 #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_long
916 #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_long
917 #else
918 #define atomic_fcmpset_ptr atomic_fcmpset_int
919 #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_int
920 #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_int
921 #endif
922
923 static __inline u_int
atomic_fetchadd_int(volatile u_int * p,u_int v)924 atomic_fetchadd_int(volatile u_int *p, u_int v)
925 {
926 u_int value;
927
928 do {
929 value = *p;
930 } while (!atomic_cmpset_int(p, value, value + v));
931 return (value);
932 }
933
934 static __inline u_long
atomic_fetchadd_long(volatile u_long * p,u_long v)935 atomic_fetchadd_long(volatile u_long *p, u_long v)
936 {
937 u_long value;
938
939 do {
940 value = *p;
941 } while (!atomic_cmpset_long(p, value, value + v));
942 return (value);
943 }
944
945 static __inline u_int
atomic_swap_32(volatile u_int * p,u_int v)946 atomic_swap_32(volatile u_int *p, u_int v)
947 {
948 u_int prev;
949
950 __asm __volatile(
951 "1: lwarx %0,0,%2\n"
952 " stwcx. %3,0,%2\n"
953 " bne- 1b\n"
954 : "=&r" (prev), "+m" (*(volatile u_int *)p)
955 : "r" (p), "r" (v)
956 : "cr0", "memory");
957
958 return (prev);
959 }
960
961 #ifdef __powerpc64__
962 static __inline u_long
atomic_swap_64(volatile u_long * p,u_long v)963 atomic_swap_64(volatile u_long *p, u_long v)
964 {
965 u_long prev;
966
967 __asm __volatile(
968 "1: ldarx %0,0,%2\n"
969 " stdcx. %3,0,%2\n"
970 " bne- 1b\n"
971 : "=&r" (prev), "+m" (*(volatile u_long *)p)
972 : "r" (p), "r" (v)
973 : "cr0", "memory");
974
975 return (prev);
976 }
977 #endif
978
979 #define atomic_fetchadd_32 atomic_fetchadd_int
980 #define atomic_swap_int atomic_swap_32
981
982 #ifdef __powerpc64__
983 #define atomic_fetchadd_64 atomic_fetchadd_long
984 #define atomic_swap_long atomic_swap_64
985 #define atomic_swap_ptr atomic_swap_64
986 #else
987 #define atomic_swap_long(p,v) atomic_swap_32((volatile u_int *)(p), v)
988 #define atomic_swap_ptr(p,v) atomic_swap_32((volatile u_int *)(p), v)
989 #endif
990
991 static __inline int
atomic_testandset_int(volatile u_int * p,u_int v)992 atomic_testandset_int(volatile u_int *p, u_int v)
993 {
994 u_int m = (1u << (v & 0x1f));
995 u_int res;
996 u_int tmp;
997
998 __asm __volatile(
999 "1: lwarx %0,0,%3\n"
1000 " and %1,%0,%4\n"
1001 " or %0,%0,%4\n"
1002 " stwcx. %0,0,%3\n"
1003 " bne- 1b\n"
1004 : "=&r"(tmp), "=&r"(res), "+m"(*p)
1005 : "r"(p), "r"(m)
1006 : "cr0", "memory");
1007
1008 return (res != 0);
1009 }
1010
1011 static __inline int
atomic_testandclear_int(volatile u_int * p,u_int v)1012 atomic_testandclear_int(volatile u_int *p, u_int v)
1013 {
1014 u_int m = (1u << (v & 0x1f));
1015 u_int res;
1016 u_int tmp;
1017
1018 __asm __volatile(
1019 "1: lwarx %0,0,%3\n"
1020 " and %1,%0,%4\n"
1021 " andc %0,%0,%4\n"
1022 " stwcx. %0,0,%3\n"
1023 " bne- 1b\n"
1024 : "=&r"(tmp), "=&r"(res), "+m"(*p)
1025 : "r"(p), "r"(m)
1026 : "cr0", "memory");
1027
1028 return (res != 0);
1029 }
1030
1031 #ifdef __powerpc64__
1032 static __inline int
atomic_testandset_long(volatile u_long * p,u_int v)1033 atomic_testandset_long(volatile u_long *p, u_int v)
1034 {
1035 u_long m = (1ul << (v & 0x3f));
1036 u_long res;
1037 u_long tmp;
1038
1039 __asm __volatile(
1040 "1: ldarx %0,0,%3\n"
1041 " and %1,%0,%4\n"
1042 " or %0,%0,%4\n"
1043 " stdcx. %0,0,%3\n"
1044 " bne- 1b\n"
1045 : "=&r"(tmp), "=&r"(res), "+m"(*(volatile u_long *)p)
1046 : "r"(p), "r"(m)
1047 : "cr0", "memory");
1048
1049 return (res != 0);
1050 }
1051
1052 static __inline int
atomic_testandclear_long(volatile u_long * p,u_int v)1053 atomic_testandclear_long(volatile u_long *p, u_int v)
1054 {
1055 u_long m = (1ul << (v & 0x3f));
1056 u_long res;
1057 u_long tmp;
1058
1059 __asm __volatile(
1060 "1: ldarx %0,0,%3\n"
1061 " and %1,%0,%4\n"
1062 " andc %0,%0,%4\n"
1063 " stdcx. %0,0,%3\n"
1064 " bne- 1b\n"
1065 : "=&r"(tmp), "=&r"(res), "+m"(*p)
1066 : "r"(p), "r"(m)
1067 : "cr0", "memory");
1068
1069 return (res != 0);
1070 }
1071 #else
1072 static __inline int
atomic_testandset_long(volatile u_long * p,u_int v)1073 atomic_testandset_long(volatile u_long *p, u_int v)
1074 {
1075 return (atomic_testandset_int((volatile u_int *)p, v));
1076 }
1077
1078 static __inline int
atomic_testandclear_long(volatile u_long * p,u_int v)1079 atomic_testandclear_long(volatile u_long *p, u_int v)
1080 {
1081 return (atomic_testandclear_int((volatile u_int *)p, v));
1082 }
1083 #endif
1084
1085 #define atomic_testandclear_32 atomic_testandclear_int
1086 #define atomic_testandset_32 atomic_testandset_int
1087
1088 static __inline int
atomic_testandset_acq_long(volatile u_long * p,u_int v)1089 atomic_testandset_acq_long(volatile u_long *p, u_int v)
1090 {
1091 u_int a = atomic_testandset_long(p, v);
1092 __ATOMIC_ACQ();
1093 return (a);
1094 }
1095
1096 #define atomic_testandclear_int atomic_testandclear_int
1097 #define atomic_testandset_int atomic_testandset_int
1098 #define atomic_testandclear_long atomic_testandclear_long
1099 #define atomic_testandset_long atomic_testandset_long
1100 #define atomic_testandset_acq_long atomic_testandset_acq_long
1101
1102 static __inline void
atomic_thread_fence_acq(void)1103 atomic_thread_fence_acq(void)
1104 {
1105
1106 powerpc_lwsync();
1107 }
1108
1109 static __inline void
atomic_thread_fence_rel(void)1110 atomic_thread_fence_rel(void)
1111 {
1112
1113 powerpc_lwsync();
1114 }
1115
1116 static __inline void
atomic_thread_fence_acq_rel(void)1117 atomic_thread_fence_acq_rel(void)
1118 {
1119
1120 powerpc_lwsync();
1121 }
1122
1123 static __inline void
atomic_thread_fence_seq_cst(void)1124 atomic_thread_fence_seq_cst(void)
1125 {
1126
1127 __asm __volatile("sync" : : : "memory");
1128 }
1129
1130 #ifndef ISA_206_ATOMICS
1131 #include <sys/_atomic_subword.h>
1132 #define atomic_cmpset_char atomic_cmpset_8
1133 #define atomic_cmpset_short atomic_cmpset_16
1134 #define atomic_fcmpset_char atomic_fcmpset_8
1135 #define atomic_fcmpset_short atomic_fcmpset_16
1136 #endif
1137
1138 /* These need sys/_atomic_subword.h on non-ISA-2.06-atomic platforms. */
1139 ATOMIC_CMPSET_ACQ_REL(char);
1140 ATOMIC_CMPSET_ACQ_REL(short);
1141
1142 ATOMIC_FCMPSET_ACQ_REL(char);
1143 ATOMIC_FCMPSET_ACQ_REL(short);
1144
1145 #undef __ATOMIC_REL
1146 #undef __ATOMIC_ACQ
1147
1148 #endif /* ! _MACHINE_ATOMIC_H_ */
1149