1*dd81489dSjsg /* $OpenBSD: atomic.h,v 1.3 2022/08/29 02:01:18 jsg Exp $ */
213861200Skettenis
313861200Skettenis /*
413861200Skettenis * Copyright (c) 2015 Martin Pieuchot
513861200Skettenis *
613861200Skettenis * Permission to use, copy, modify, and distribute this software for any
713861200Skettenis * purpose with or without fee is hereby granted, provided that the above
813861200Skettenis * copyright notice and this permission notice appear in all copies.
913861200Skettenis *
1013861200Skettenis * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
1113861200Skettenis * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
1213861200Skettenis * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
1313861200Skettenis * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
1413861200Skettenis * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
1513861200Skettenis * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
1613861200Skettenis * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
1713861200Skettenis */
1813861200Skettenis
1913861200Skettenis
2013861200Skettenis #ifndef _MACHINE_ATOMIC_H_
2113861200Skettenis #define _MACHINE_ATOMIC_H_
2213861200Skettenis
2313861200Skettenis #if defined(_KERNEL)
2413861200Skettenis
2513861200Skettenis static __inline void
atomic_setbits_int(volatile unsigned int * uip,unsigned int v)2613861200Skettenis atomic_setbits_int(volatile unsigned int *uip, unsigned int v)
2713861200Skettenis {
2813861200Skettenis unsigned int tmp;
2913861200Skettenis
3013861200Skettenis __asm volatile (
3113861200Skettenis "1: lwarx %0, 0, %2 \n"
3213861200Skettenis " or %0, %1, %0 \n"
3313861200Skettenis " stwcx. %0, 0, %2 \n"
3413861200Skettenis " bne- 1b \n"
3513861200Skettenis " sync" : "=&r" (tmp) : "r" (v), "r" (uip) : "cc", "memory");
3613861200Skettenis }
3713861200Skettenis
3813861200Skettenis static __inline void
atomic_clearbits_int(volatile unsigned int * uip,unsigned int v)3913861200Skettenis atomic_clearbits_int(volatile unsigned int *uip, unsigned int v)
4013861200Skettenis {
4113861200Skettenis unsigned int tmp;
4213861200Skettenis
4313861200Skettenis __asm volatile (
4413861200Skettenis "1: lwarx %0, 0, %2 \n"
4513861200Skettenis " andc %0, %0, %1 \n"
4613861200Skettenis " stwcx. %0, 0, %2 \n"
4713861200Skettenis " bne- 1b \n"
4813861200Skettenis " sync" : "=&r" (tmp) : "r" (v), "r" (uip) : "cc", "memory");
4913861200Skettenis }
5013861200Skettenis
5113861200Skettenis #endif /* defined(_KERNEL) */
5213861200Skettenis
5313861200Skettenis static inline unsigned int
_atomic_cas_uint(volatile unsigned int * p,unsigned int o,unsigned int n)5413861200Skettenis _atomic_cas_uint(volatile unsigned int *p, unsigned int o, unsigned int n)
5513861200Skettenis {
5613861200Skettenis unsigned int rv;
5713861200Skettenis
5813861200Skettenis __asm volatile (
5913861200Skettenis "1: lwarx %0, 0, %2 \n"
6013861200Skettenis " cmpw 0, %0, %4 \n"
6113861200Skettenis " bne- 2f \n"
6213861200Skettenis " stwcx. %3, 0, %2 \n"
6313861200Skettenis " bne- 1b \n"
6413861200Skettenis "2: \n"
6513861200Skettenis : "=&r" (rv), "+m" (*p)
6613861200Skettenis : "r" (p), "r" (n), "r" (o)
6713861200Skettenis : "cc");
6813861200Skettenis
6913861200Skettenis return (rv);
7013861200Skettenis }
7113861200Skettenis #define atomic_cas_uint(_p, _o, _n) _atomic_cas_uint((_p), (_o), (_n))
7213861200Skettenis
7313861200Skettenis static inline unsigned long
_atomic_cas_ulong(volatile unsigned long * p,unsigned long o,unsigned long n)7413861200Skettenis _atomic_cas_ulong(volatile unsigned long *p, unsigned long o, unsigned long n)
7513861200Skettenis {
7613861200Skettenis unsigned long rv;
7713861200Skettenis
7813861200Skettenis __asm volatile (
7913861200Skettenis "1: ldarx %0, 0, %2 \n"
8013861200Skettenis " cmpd 0, %0, %4 \n"
8113861200Skettenis " bne- 2f \n"
8213861200Skettenis " stdcx. %3, 0, %2 \n"
8313861200Skettenis " bne- 1b \n"
8413861200Skettenis "2: \n"
8513861200Skettenis : "=&r" (rv), "+m" (*p)
8613861200Skettenis : "r" (p), "r" (n), "r" (o)
8713861200Skettenis : "cc");
8813861200Skettenis
8913861200Skettenis return (rv);
9013861200Skettenis }
9113861200Skettenis #define atomic_cas_ulong(_p, _o, _n) _atomic_cas_ulong((_p), (_o), (_n))
9213861200Skettenis
9313861200Skettenis static inline void *
_atomic_cas_ptr(volatile void * pp,void * o,void * n)9413861200Skettenis _atomic_cas_ptr(volatile void *pp, void *o, void *n)
9513861200Skettenis {
9613861200Skettenis void * volatile *p = pp;
9713861200Skettenis void *rv;
9813861200Skettenis
9913861200Skettenis __asm volatile (
10013861200Skettenis "1: ldarx %0, 0, %2 \n"
10113861200Skettenis " cmpd 0, %0, %4 \n"
10213861200Skettenis " bne- 2f \n"
10313861200Skettenis " stdcx. %3, 0, %2 \n"
10413861200Skettenis " bne- 1b \n"
10513861200Skettenis "2: \n"
10613861200Skettenis : "=&r" (rv), "+m" (*p)
10713861200Skettenis : "r" (p), "r" (n), "r" (o)
10813861200Skettenis : "cc");
10913861200Skettenis
11013861200Skettenis return (rv);
11113861200Skettenis }
11213861200Skettenis #define atomic_cas_ptr(_p, _o, _n) _atomic_cas_ptr((_p), (_o), (_n))
11313861200Skettenis
11413861200Skettenis static inline unsigned int
_atomic_swap_uint(volatile unsigned int * p,unsigned int v)11513861200Skettenis _atomic_swap_uint(volatile unsigned int *p, unsigned int v)
11613861200Skettenis {
11713861200Skettenis unsigned int rv;
11813861200Skettenis
11913861200Skettenis __asm volatile (
12068e64030Skettenis "1: lwarx %0, 0, %2 \n"
12168e64030Skettenis " stwcx. %3, 0, %2 \n"
12213861200Skettenis " bne- 1b \n"
12313861200Skettenis : "=&r" (rv), "+m" (*p)
12413861200Skettenis : "r" (p), "r" (v)
12513861200Skettenis : "cc");
12613861200Skettenis
12713861200Skettenis return (rv);
12813861200Skettenis }
12913861200Skettenis #define atomic_swap_uint(_p, _v) _atomic_swap_uint((_p), (_v))
13013861200Skettenis
13113861200Skettenis static inline unsigned long
_atomic_swap_ulong(volatile unsigned long * p,unsigned long v)13213861200Skettenis _atomic_swap_ulong(volatile unsigned long *p, unsigned long v)
13313861200Skettenis {
13413861200Skettenis unsigned long rv;
13513861200Skettenis
13613861200Skettenis __asm volatile (
13713861200Skettenis "1: ldarx %0, 0, %2 \n"
13813861200Skettenis " stdcx. %3, 0, %2 \n"
13913861200Skettenis " bne- 1b \n"
14013861200Skettenis : "=&r" (rv), "+m" (*p)
14113861200Skettenis : "r" (p), "r" (v)
14213861200Skettenis : "cc");
14313861200Skettenis
14413861200Skettenis return (rv);
14513861200Skettenis }
14613861200Skettenis #define atomic_swap_ulong(_p, _v) _atomic_swap_ulong((_p), (_v))
14713861200Skettenis
14813861200Skettenis static inline void *
_atomic_swap_ptr(volatile void * pp,void * v)14913861200Skettenis _atomic_swap_ptr(volatile void *pp, void *v)
15013861200Skettenis {
15113861200Skettenis void * volatile *p = pp;
15213861200Skettenis void *rv;
15313861200Skettenis
15413861200Skettenis __asm volatile (
15513861200Skettenis "1: ldarx %0, 0, %2 \n"
15613861200Skettenis " stdcx. %3, 0, %2 \n"
15713861200Skettenis " bne- 1b \n"
15813861200Skettenis : "=&r" (rv), "+m" (*p)
15913861200Skettenis : "r" (p), "r" (v)
16013861200Skettenis : "cc");
16113861200Skettenis
16213861200Skettenis return (rv);
16313861200Skettenis }
16413861200Skettenis #define atomic_swap_ptr(_p, _v) _atomic_swap_ptr((_p), (_v))
16513861200Skettenis
16613861200Skettenis static inline unsigned int
_atomic_add_int_nv(volatile unsigned int * p,unsigned int v)16713861200Skettenis _atomic_add_int_nv(volatile unsigned int *p, unsigned int v)
16813861200Skettenis {
16913861200Skettenis unsigned int rv;
17013861200Skettenis
17113861200Skettenis __asm volatile (
17213861200Skettenis "1: lwarx %0, 0, %2 \n"
17313861200Skettenis " add %0, %3, %0 \n"
17413861200Skettenis " stwcx. %0, 0, %2 \n"
17513861200Skettenis " bne- 1b \n"
17613861200Skettenis : "=&r" (rv), "+m" (*p)
17713861200Skettenis : "r" (p), "r" (v)
17813861200Skettenis : "cc", "xer");
17913861200Skettenis
18013861200Skettenis return (rv);
18113861200Skettenis }
18213861200Skettenis #define atomic_add_int_nv(_p, _v) _atomic_add_int_nv((_p), (_v))
18313861200Skettenis
18413861200Skettenis static inline unsigned long
_atomic_add_long_nv(volatile unsigned long * p,unsigned long v)18513861200Skettenis _atomic_add_long_nv(volatile unsigned long *p, unsigned long v)
18613861200Skettenis {
18713861200Skettenis unsigned long rv;
18813861200Skettenis
18913861200Skettenis __asm volatile (
19013861200Skettenis "1: ldarx %0, 0, %2 \n"
19113861200Skettenis " add %0, %3, %0 \n"
19213861200Skettenis " stdcx. %0, 0, %2 \n"
19313861200Skettenis " bne- 1b \n"
19413861200Skettenis : "=&r" (rv), "+m" (*p)
19513861200Skettenis : "r" (p), "r" (v)
19613861200Skettenis : "cc", "xer");
19713861200Skettenis
19813861200Skettenis return (rv);
19913861200Skettenis }
20013861200Skettenis #define atomic_add_long_nv(_p, _v) _atomic_add_long_nv((_p), (_v))
20113861200Skettenis
20213861200Skettenis static inline unsigned int
_atomic_sub_int_nv(volatile unsigned int * p,unsigned int v)20313861200Skettenis _atomic_sub_int_nv(volatile unsigned int *p, unsigned int v)
20413861200Skettenis {
20513861200Skettenis unsigned int rv;
20613861200Skettenis
20713861200Skettenis __asm volatile (
20813861200Skettenis "1: lwarx %0, 0, %2 \n"
20913861200Skettenis " subf %0, %3, %0 \n"
21013861200Skettenis " stwcx. %0, 0, %2 \n"
21113861200Skettenis " bne- 1b \n"
21213861200Skettenis : "=&r" (rv), "+m" (*p)
21313861200Skettenis : "r" (p), "r" (v)
21413861200Skettenis : "cc", "xer");
21513861200Skettenis
21613861200Skettenis return (rv);
21713861200Skettenis }
21813861200Skettenis #define atomic_sub_int_nv(_p, _v) _atomic_sub_int_nv((_p), (_v))
21913861200Skettenis
22013861200Skettenis static inline unsigned long
_atomic_sub_long_nv(volatile unsigned long * p,unsigned long v)22113861200Skettenis _atomic_sub_long_nv(volatile unsigned long *p, unsigned long v)
22213861200Skettenis {
22313861200Skettenis unsigned long rv;
22413861200Skettenis
22513861200Skettenis __asm volatile (
22613861200Skettenis "1: ldarx %0, 0, %2 \n"
22713861200Skettenis " subf %0, %3, %0 \n"
22813861200Skettenis " stdcx. %0, 0, %2 \n"
22913861200Skettenis " bne- 1b \n"
23013861200Skettenis : "=&r" (rv), "+m" (*p)
23113861200Skettenis : "r" (p), "r" (v)
23213861200Skettenis : "cc", "xer");
23313861200Skettenis
23413861200Skettenis return (rv);
23513861200Skettenis }
23613861200Skettenis #define atomic_sub_long_nv(_p, _v) _atomic_sub_long_nv((_p), (_v))
23713861200Skettenis
23813861200Skettenis static inline unsigned int
_atomic_addic_int_nv(volatile unsigned int * p,unsigned int v)23913861200Skettenis _atomic_addic_int_nv(volatile unsigned int *p, unsigned int v)
24013861200Skettenis {
24113861200Skettenis unsigned int rv;
24213861200Skettenis
24313861200Skettenis __asm volatile (
24413861200Skettenis "1: lwarx %0, 0, %2 \n"
24513861200Skettenis " addic %0, %0, %3 \n"
24613861200Skettenis " stwcx. %0, 0, %2 \n"
24713861200Skettenis " bne- 1b \n"
24813861200Skettenis : "=&r" (rv), "+m" (*p)
24913861200Skettenis : "r" (p), "i" (v)
25013861200Skettenis : "cc", "xer");
25113861200Skettenis
25213861200Skettenis return (rv);
25313861200Skettenis }
25413861200Skettenis #define atomic_inc_int_nv(_p) _atomic_addic_int_nv((_p), 1)
25513861200Skettenis #define atomic_dec_int_nv(_p) _atomic_addic_int_nv((_p), -1)
25613861200Skettenis
25713861200Skettenis static inline unsigned long
_atomic_addic_long_nv(volatile unsigned long * p,unsigned long v)25813861200Skettenis _atomic_addic_long_nv(volatile unsigned long *p, unsigned long v)
25913861200Skettenis {
26013861200Skettenis unsigned long rv;
26113861200Skettenis
26213861200Skettenis __asm volatile (
26313861200Skettenis "1: ldarx %0, 0, %2 \n"
26413861200Skettenis " addic %0, %0, %3 \n"
26513861200Skettenis " stdcx. %0, 0, %2 \n"
26613861200Skettenis " bne- 1b \n"
26713861200Skettenis : "=&r" (rv), "+m" (*p)
26813861200Skettenis : "r" (p), "i" (v)
26913861200Skettenis : "cc", "xer");
27013861200Skettenis
27113861200Skettenis return (rv);
27213861200Skettenis }
27313861200Skettenis #define atomic_inc_long_nv(_p) _atomic_addic_long_nv((_p), 1)
27413861200Skettenis #define atomic_dec_long_nv(_p) _atomic_addic_long_nv((_p), -1)
27513861200Skettenis
276*dd81489dSjsg #define __membar(_f) do { __asm volatile(_f ::: "memory"); } while (0)
27713861200Skettenis
27813861200Skettenis #if defined(MULTIPROCESSOR) || !defined(_KERNEL)
27913861200Skettenis #define membar_enter() __membar("isync")
28013861200Skettenis #define membar_exit() __membar("sync")
28113861200Skettenis #define membar_producer() __membar("sync")
28213861200Skettenis #define membar_consumer() __membar("isync")
28313861200Skettenis #define membar_sync() __membar("sync")
28413861200Skettenis #else
28513861200Skettenis #define membar_enter() __membar("")
28613861200Skettenis #define membar_exit() __membar("")
28713861200Skettenis #define membar_producer() __membar("")
28813861200Skettenis #define membar_consumer() __membar("")
28913861200Skettenis #define membar_sync() __membar("")
29013861200Skettenis #endif
29113861200Skettenis
29213861200Skettenis #endif /* _MACHINE_ATOMIC_H_ */
293