1/* SPDX-License-Identifier: GPL-2.0 */
2/* atomic.S: These things are too big to do inline.
3 *
4 * Copyright (C) 1999, 2007 2012 David S. Miller (davem@davemloft.net)
5 */
6
7#include <linux/linkage.h>
8#include <asm/asi.h>
9#include <asm/backoff.h>
10#include <asm/export.h>
11
12	.text
13
14	/* Three versions of the atomic routines, one that
15	 * does not return a value and does not perform
16	 * memory barriers, and a two which return
17	 * a value, the new and old value resp. and does the
18	 * barriers.
19	 */
20
21#define ATOMIC_OP(op)							\
22ENTRY(atomic_##op) /* %o0 = increment, %o1 = atomic_ptr */		\
23	BACKOFF_SETUP(%o2);						\
241:	lduw	[%o1], %g1;						\
25	op	%g1, %o0, %g7;						\
26	cas	[%o1], %g1, %g7;					\
27	cmp	%g1, %g7;						\
28	bne,pn	%icc, BACKOFF_LABEL(2f, 1b);				\
29	 nop;								\
30	retl;								\
31	 nop;								\
322:	BACKOFF_SPIN(%o2, %o3, 1b);					\
33ENDPROC(atomic_##op);							\
34EXPORT_SYMBOL(atomic_##op);
35
36#define ATOMIC_OP_RETURN(op)						\
37ENTRY(atomic_##op##_return) /* %o0 = increment, %o1 = atomic_ptr */	\
38	BACKOFF_SETUP(%o2);						\
391:	lduw	[%o1], %g1;						\
40	op	%g1, %o0, %g7;						\
41	cas	[%o1], %g1, %g7;					\
42	cmp	%g1, %g7;						\
43	bne,pn	%icc, BACKOFF_LABEL(2f, 1b);				\
44	 op	%g1, %o0, %g1;						\
45	retl;								\
46	 sra	%g1, 0, %o0;						\
472:	BACKOFF_SPIN(%o2, %o3, 1b);					\
48ENDPROC(atomic_##op##_return);						\
49EXPORT_SYMBOL(atomic_##op##_return);
50
51#define ATOMIC_FETCH_OP(op)						\
52ENTRY(atomic_fetch_##op) /* %o0 = increment, %o1 = atomic_ptr */	\
53	BACKOFF_SETUP(%o2);						\
541:	lduw	[%o1], %g1;						\
55	op	%g1, %o0, %g7;						\
56	cas	[%o1], %g1, %g7;					\
57	cmp	%g1, %g7;						\
58	bne,pn	%icc, BACKOFF_LABEL(2f, 1b);				\
59	 nop;								\
60	retl;								\
61	 sra	%g1, 0, %o0;						\
622:	BACKOFF_SPIN(%o2, %o3, 1b);					\
63ENDPROC(atomic_fetch_##op);						\
64EXPORT_SYMBOL(atomic_fetch_##op);
65
66ATOMIC_OP(add)
67ATOMIC_OP_RETURN(add)
68ATOMIC_FETCH_OP(add)
69
70ATOMIC_OP(sub)
71ATOMIC_OP_RETURN(sub)
72ATOMIC_FETCH_OP(sub)
73
74ATOMIC_OP(and)
75ATOMIC_FETCH_OP(and)
76
77ATOMIC_OP(or)
78ATOMIC_FETCH_OP(or)
79
80ATOMIC_OP(xor)
81ATOMIC_FETCH_OP(xor)
82
83#undef ATOMIC_FETCH_OP
84#undef ATOMIC_OP_RETURN
85#undef ATOMIC_OP
86
87#define ATOMIC64_OP(op)							\
88ENTRY(atomic64_##op) /* %o0 = increment, %o1 = atomic_ptr */		\
89	BACKOFF_SETUP(%o2);						\
901:	ldx	[%o1], %g1;						\
91	op	%g1, %o0, %g7;						\
92	casx	[%o1], %g1, %g7;					\
93	cmp	%g1, %g7;						\
94	bne,pn	%xcc, BACKOFF_LABEL(2f, 1b);				\
95	 nop;								\
96	retl;								\
97	 nop;								\
982:	BACKOFF_SPIN(%o2, %o3, 1b);					\
99ENDPROC(atomic64_##op);							\
100EXPORT_SYMBOL(atomic64_##op);
101
102#define ATOMIC64_OP_RETURN(op)						\
103ENTRY(atomic64_##op##_return) /* %o0 = increment, %o1 = atomic_ptr */	\
104	BACKOFF_SETUP(%o2);						\
1051:	ldx	[%o1], %g1;						\
106	op	%g1, %o0, %g7;						\
107	casx	[%o1], %g1, %g7;					\
108	cmp	%g1, %g7;						\
109	bne,pn	%xcc, BACKOFF_LABEL(2f, 1b);				\
110	 nop;								\
111	retl;								\
112	 op	%g1, %o0, %o0;						\
1132:	BACKOFF_SPIN(%o2, %o3, 1b);					\
114ENDPROC(atomic64_##op##_return);					\
115EXPORT_SYMBOL(atomic64_##op##_return);
116
117#define ATOMIC64_FETCH_OP(op)						\
118ENTRY(atomic64_fetch_##op) /* %o0 = increment, %o1 = atomic_ptr */	\
119	BACKOFF_SETUP(%o2);						\
1201:	ldx	[%o1], %g1;						\
121	op	%g1, %o0, %g7;						\
122	casx	[%o1], %g1, %g7;					\
123	cmp	%g1, %g7;						\
124	bne,pn	%xcc, BACKOFF_LABEL(2f, 1b);				\
125	 nop;								\
126	retl;								\
127	 mov	%g1, %o0;						\
1282:	BACKOFF_SPIN(%o2, %o3, 1b);					\
129ENDPROC(atomic64_fetch_##op);						\
130EXPORT_SYMBOL(atomic64_fetch_##op);
131
132ATOMIC64_OP(add)
133ATOMIC64_OP_RETURN(add)
134ATOMIC64_FETCH_OP(add)
135
136ATOMIC64_OP(sub)
137ATOMIC64_OP_RETURN(sub)
138ATOMIC64_FETCH_OP(sub)
139
140ATOMIC64_OP(and)
141ATOMIC64_FETCH_OP(and)
142
143ATOMIC64_OP(or)
144ATOMIC64_FETCH_OP(or)
145
146ATOMIC64_OP(xor)
147ATOMIC64_FETCH_OP(xor)
148
149#undef ATOMIC64_FETCH_OP
150#undef ATOMIC64_OP_RETURN
151#undef ATOMIC64_OP
152
153ENTRY(atomic64_dec_if_positive) /* %o0 = atomic_ptr */
154	BACKOFF_SETUP(%o2)
1551:	ldx	[%o0], %g1
156	brlez,pn %g1, 3f
157	 sub	%g1, 1, %g7
158	casx	[%o0], %g1, %g7
159	cmp	%g1, %g7
160	bne,pn	%xcc, BACKOFF_LABEL(2f, 1b)
161	 nop
1623:	retl
163	 sub	%g1, 1, %o0
1642:	BACKOFF_SPIN(%o2, %o3, 1b)
165ENDPROC(atomic64_dec_if_positive)
166EXPORT_SYMBOL(atomic64_dec_if_positive)
167