1 /* Linux-specific atomic operations for PA Linux.
2 Copyright (C) 2008-2019 Free Software Foundation, Inc.
3 Based on code contributed by CodeSourcery for ARM EABI Linux.
4 Modifications for PA Linux by Helge Deller <deller@gmx.de>
5
6 This file is part of GCC.
7
8 GCC is free software; you can redistribute it and/or modify it under
9 the terms of the GNU General Public License as published by the Free
10 Software Foundation; either version 3, or (at your option) any later
11 version.
12
13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 for more details.
17
18 Under Section 7 of GPL version 3, you are granted additional
19 permissions described in the GCC Runtime Library Exception, version
20 3.1, as published by the Free Software Foundation.
21
22 You should have received a copy of the GNU General Public License and
23 a copy of the GCC Runtime Library Exception along with this program;
24 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
25 <http://www.gnu.org/licenses/>. */
26
27 #define EFAULT 14
28 #define EBUSY 16
29 #define ENOSYS 251
30
31 /* PA-RISC 2.0 supports out-of-order execution for loads and stores.
32 Thus, we need to synchonize memory accesses. For more info, see:
33 "Advanced Performance Features of the 64-bit PA-8000" by Doug Hunt.
34
35 We implement byte, short and int versions of each atomic operation
36 using the kernel helper defined below. There is no support for
37 64-bit operations yet. */
38
39 /* Determine kernel LWS function call (0=32-bit, 1=64-bit userspace). */
40 #define LWS_CAS (sizeof(long) == 4 ? 0 : 1)
41
42 /* Kernel helper for compare-and-exchange a 32-bit value. */
43 static inline long
__kernel_cmpxchg(volatile void * mem,int oldval,int newval)44 __kernel_cmpxchg (volatile void *mem, int oldval, int newval)
45 {
46 register unsigned long lws_mem asm("r26") = (unsigned long) (mem);
47 register int lws_old asm("r25") = oldval;
48 register int lws_new asm("r24") = newval;
49 register long lws_ret asm("r28");
50 register long lws_errno asm("r21");
51 asm volatile ( "ble 0xb0(%%sr2, %%r0) \n\t"
52 "ldi %2, %%r20 \n\t"
53 : "=r" (lws_ret), "=r" (lws_errno)
54 : "i" (LWS_CAS), "r" (lws_mem), "r" (lws_old), "r" (lws_new)
55 : "r1", "r20", "r22", "r23", "r29", "r31", "memory"
56 );
57
58 /* If the kernel LWS call succeeded (lws_errno == 0), lws_ret contains
59 the old value from memory. If this value is equal to OLDVAL, the
60 new value was written to memory. If not, return -EBUSY. */
61 if (!lws_errno && lws_ret != oldval)
62 return -EBUSY;
63
64 return lws_errno;
65 }
66
67 static inline long
__kernel_cmpxchg2(volatile void * mem,const void * oldval,const void * newval,int val_size)68 __kernel_cmpxchg2 (volatile void *mem, const void *oldval, const void *newval,
69 int val_size)
70 {
71 register unsigned long lws_mem asm("r26") = (unsigned long) (mem);
72 register unsigned long lws_old asm("r25") = (unsigned long) oldval;
73 register unsigned long lws_new asm("r24") = (unsigned long) newval;
74 register int lws_size asm("r23") = val_size;
75 register long lws_ret asm("r28");
76 register long lws_errno asm("r21");
77 asm volatile ( "ble 0xb0(%%sr2, %%r0) \n\t"
78 "ldi %6, %%r20 \n\t"
79 : "=r" (lws_ret), "=r" (lws_errno), "+r" (lws_mem),
80 "+r" (lws_old), "+r" (lws_new), "+r" (lws_size)
81 : "i" (2)
82 : "r1", "r20", "r22", "r29", "r31", "fr4", "memory"
83 );
84
85 /* If the kernel LWS call is successful, lws_ret contains 0. */
86 if (__builtin_expect (lws_ret == 0, 1))
87 return 0;
88
89 /* If the kernel LWS call fails with no error, return -EBUSY */
90 if (__builtin_expect (!lws_errno, 0))
91 return -EBUSY;
92
93 return lws_errno;
94 }
95 #define HIDDEN __attribute__ ((visibility ("hidden")))
96
97 /* Big endian masks */
98 #define INVERT_MASK_1 24
99 #define INVERT_MASK_2 16
100
101 #define MASK_1 0xffu
102 #define MASK_2 0xffffu
103
104 #define FETCH_AND_OP_2(OP, PFX_OP, INF_OP, TYPE, WIDTH, INDEX) \
105 TYPE HIDDEN \
106 __sync_fetch_and_##OP##_##WIDTH (volatile void *ptr, TYPE val) \
107 { \
108 TYPE tmp, newval; \
109 long failure; \
110 \
111 do { \
112 tmp = __atomic_load_n ((volatile TYPE *)ptr, __ATOMIC_RELAXED); \
113 newval = PFX_OP (tmp INF_OP val); \
114 failure = __kernel_cmpxchg2 (ptr, &tmp, &newval, INDEX); \
115 } while (failure != 0); \
116 \
117 return tmp; \
118 }
119
120 FETCH_AND_OP_2 (add, , +, long long unsigned int, 8, 3)
121 FETCH_AND_OP_2 (sub, , -, long long unsigned int, 8, 3)
122 FETCH_AND_OP_2 (or, , |, long long unsigned int, 8, 3)
123 FETCH_AND_OP_2 (and, , &, long long unsigned int, 8, 3)
124 FETCH_AND_OP_2 (xor, , ^, long long unsigned int, 8, 3)
125 FETCH_AND_OP_2 (nand, ~, &, long long unsigned int, 8, 3)
126
127 FETCH_AND_OP_2 (add, , +, short unsigned int, 2, 1)
128 FETCH_AND_OP_2 (sub, , -, short unsigned int, 2, 1)
129 FETCH_AND_OP_2 (or, , |, short unsigned int, 2, 1)
130 FETCH_AND_OP_2 (and, , &, short unsigned int, 2, 1)
131 FETCH_AND_OP_2 (xor, , ^, short unsigned int, 2, 1)
132 FETCH_AND_OP_2 (nand, ~, &, short unsigned int, 2, 1)
133
134 FETCH_AND_OP_2 (add, , +, unsigned char, 1, 0)
135 FETCH_AND_OP_2 (sub, , -, unsigned char, 1, 0)
136 FETCH_AND_OP_2 (or, , |, unsigned char, 1, 0)
137 FETCH_AND_OP_2 (and, , &, unsigned char, 1, 0)
138 FETCH_AND_OP_2 (xor, , ^, unsigned char, 1, 0)
139 FETCH_AND_OP_2 (nand, ~, &, unsigned char, 1, 0)
140
141 #define OP_AND_FETCH_2(OP, PFX_OP, INF_OP, TYPE, WIDTH, INDEX) \
142 TYPE HIDDEN \
143 __sync_##OP##_and_fetch_##WIDTH (volatile void *ptr, TYPE val) \
144 { \
145 TYPE tmp, newval; \
146 long failure; \
147 \
148 do { \
149 tmp = __atomic_load_n ((volatile TYPE *)ptr, __ATOMIC_RELAXED); \
150 newval = PFX_OP (tmp INF_OP val); \
151 failure = __kernel_cmpxchg2 (ptr, &tmp, &newval, INDEX); \
152 } while (failure != 0); \
153 \
154 return PFX_OP (tmp INF_OP val); \
155 }
156
157 OP_AND_FETCH_2 (add, , +, long long unsigned int, 8, 3)
158 OP_AND_FETCH_2 (sub, , -, long long unsigned int, 8, 3)
159 OP_AND_FETCH_2 (or, , |, long long unsigned int, 8, 3)
160 OP_AND_FETCH_2 (and, , &, long long unsigned int, 8, 3)
161 OP_AND_FETCH_2 (xor, , ^, long long unsigned int, 8, 3)
162 OP_AND_FETCH_2 (nand, ~, &, long long unsigned int, 8, 3)
163
164 OP_AND_FETCH_2 (add, , +, short unsigned int, 2, 1)
165 OP_AND_FETCH_2 (sub, , -, short unsigned int, 2, 1)
166 OP_AND_FETCH_2 (or, , |, short unsigned int, 2, 1)
167 OP_AND_FETCH_2 (and, , &, short unsigned int, 2, 1)
168 OP_AND_FETCH_2 (xor, , ^, short unsigned int, 2, 1)
169 OP_AND_FETCH_2 (nand, ~, &, short unsigned int, 2, 1)
170
171 OP_AND_FETCH_2 (add, , +, unsigned char, 1, 0)
172 OP_AND_FETCH_2 (sub, , -, unsigned char, 1, 0)
173 OP_AND_FETCH_2 (or, , |, unsigned char, 1, 0)
174 OP_AND_FETCH_2 (and, , &, unsigned char, 1, 0)
175 OP_AND_FETCH_2 (xor, , ^, unsigned char, 1, 0)
176 OP_AND_FETCH_2 (nand, ~, &, unsigned char, 1, 0)
177
178 #define FETCH_AND_OP_WORD(OP, PFX_OP, INF_OP) \
179 unsigned int HIDDEN \
180 __sync_fetch_and_##OP##_4 (volatile void *ptr, unsigned int val) \
181 { \
182 unsigned int tmp; \
183 long failure; \
184 \
185 do { \
186 tmp = __atomic_load_n ((volatile unsigned int *)ptr, \
187 __ATOMIC_RELAXED); \
188 failure = __kernel_cmpxchg (ptr, tmp, PFX_OP (tmp INF_OP val)); \
189 } while (failure != 0); \
190 \
191 return tmp; \
192 }
193
194 FETCH_AND_OP_WORD (add, , +)
195 FETCH_AND_OP_WORD (sub, , -)
196 FETCH_AND_OP_WORD (or, , |)
197 FETCH_AND_OP_WORD (and, , &)
198 FETCH_AND_OP_WORD (xor, , ^)
199 FETCH_AND_OP_WORD (nand, ~, &)
200
201 #define OP_AND_FETCH_WORD(OP, PFX_OP, INF_OP) \
202 unsigned int HIDDEN \
203 __sync_##OP##_and_fetch_4 (volatile void *ptr, unsigned int val) \
204 { \
205 unsigned int tmp; \
206 long failure; \
207 \
208 do { \
209 tmp = __atomic_load_n ((volatile unsigned int *)ptr, \
210 __ATOMIC_RELAXED); \
211 failure = __kernel_cmpxchg (ptr, tmp, PFX_OP (tmp INF_OP val)); \
212 } while (failure != 0); \
213 \
214 return PFX_OP (tmp INF_OP val); \
215 }
216
217 OP_AND_FETCH_WORD (add, , +)
218 OP_AND_FETCH_WORD (sub, , -)
219 OP_AND_FETCH_WORD (or, , |)
220 OP_AND_FETCH_WORD (and, , &)
221 OP_AND_FETCH_WORD (xor, , ^)
222 OP_AND_FETCH_WORD (nand, ~, &)
223
224 typedef unsigned char bool;
225
226 #define COMPARE_AND_SWAP_2(TYPE, WIDTH, INDEX) \
227 TYPE HIDDEN \
228 __sync_val_compare_and_swap_##WIDTH (volatile void *ptr, TYPE oldval, \
229 TYPE newval) \
230 { \
231 TYPE actual_oldval; \
232 long fail; \
233 \
234 while (1) \
235 { \
236 actual_oldval = __atomic_load_n ((volatile TYPE *)ptr, \
237 __ATOMIC_RELAXED); \
238 \
239 if (__builtin_expect (oldval != actual_oldval, 0)) \
240 return actual_oldval; \
241 \
242 fail = __kernel_cmpxchg2 (ptr, &actual_oldval, &newval, INDEX); \
243 \
244 if (__builtin_expect (!fail, 1)) \
245 return actual_oldval; \
246 } \
247 } \
248 \
249 _Bool HIDDEN \
250 __sync_bool_compare_and_swap_##WIDTH (volatile void *ptr, \
251 TYPE oldval, TYPE newval) \
252 { \
253 long failure = __kernel_cmpxchg2 (ptr, &oldval, &newval, INDEX); \
254 return (failure == 0); \
255 }
256
257 COMPARE_AND_SWAP_2 (long long unsigned int, 8, 3)
258 COMPARE_AND_SWAP_2 (short unsigned int, 2, 1)
259 COMPARE_AND_SWAP_2 (unsigned char, 1, 0)
260
261 unsigned int HIDDEN
__sync_val_compare_and_swap_4(volatile void * ptr,unsigned int oldval,unsigned int newval)262 __sync_val_compare_and_swap_4 (volatile void *ptr, unsigned int oldval,
263 unsigned int newval)
264 {
265 long fail;
266 unsigned int actual_oldval;
267
268 while (1)
269 {
270 actual_oldval = __atomic_load_n ((volatile unsigned int *)ptr,
271 __ATOMIC_RELAXED);
272
273 if (__builtin_expect (oldval != actual_oldval, 0))
274 return actual_oldval;
275
276 fail = __kernel_cmpxchg (ptr, actual_oldval, newval);
277
278 if (__builtin_expect (!fail, 1))
279 return actual_oldval;
280 }
281 }
282
283 _Bool HIDDEN
__sync_bool_compare_and_swap_4(volatile void * ptr,unsigned int oldval,unsigned int newval)284 __sync_bool_compare_and_swap_4 (volatile void *ptr, unsigned int oldval,
285 unsigned int newval)
286 {
287 long failure = __kernel_cmpxchg (ptr, oldval, newval);
288 return (failure == 0);
289 }
290
291 #define SYNC_LOCK_TEST_AND_SET_2(TYPE, WIDTH, INDEX) \
292 TYPE HIDDEN \
293 __sync_lock_test_and_set_##WIDTH (volatile void *ptr, TYPE val) \
294 { \
295 TYPE oldval; \
296 long failure; \
297 \
298 do { \
299 oldval = __atomic_load_n ((volatile TYPE *)ptr, \
300 __ATOMIC_RELAXED); \
301 failure = __kernel_cmpxchg2 (ptr, &oldval, &val, INDEX); \
302 } while (failure != 0); \
303 \
304 return oldval; \
305 }
306
307 SYNC_LOCK_TEST_AND_SET_2 (long long unsigned int, 8, 3)
308 SYNC_LOCK_TEST_AND_SET_2 (short unsigned int, 2, 1)
309 SYNC_LOCK_TEST_AND_SET_2 (unsigned char, 1, 0)
310
311 unsigned int HIDDEN
__sync_lock_test_and_set_4(volatile void * ptr,unsigned int val)312 __sync_lock_test_and_set_4 (volatile void *ptr, unsigned int val)
313 {
314 long failure;
315 unsigned int oldval;
316
317 do {
318 oldval = __atomic_load_n ((volatile unsigned int *)ptr, __ATOMIC_RELAXED);
319 failure = __kernel_cmpxchg (ptr, oldval, val);
320 } while (failure != 0);
321
322 return oldval;
323 }
324
325 #define SYNC_LOCK_RELEASE_1(TYPE, WIDTH, INDEX) \
326 void HIDDEN \
327 __sync_lock_release_##WIDTH (volatile void *ptr) \
328 { \
329 TYPE oldval, val = 0; \
330 long failure; \
331 \
332 do { \
333 oldval = __atomic_load_n ((volatile TYPE *)ptr, \
334 __ATOMIC_RELAXED); \
335 failure = __kernel_cmpxchg2 (ptr, &oldval, &val, INDEX); \
336 } while (failure != 0); \
337 }
338
339 SYNC_LOCK_RELEASE_1 (long long unsigned int, 8, 3)
340 SYNC_LOCK_RELEASE_1 (short unsigned int, 2, 1)
341 SYNC_LOCK_RELEASE_1 (unsigned char, 1, 0)
342
343 void HIDDEN
__sync_lock_release_4(volatile void * ptr)344 __sync_lock_release_4 (volatile void *ptr)
345 {
346 long failure;
347 unsigned int oldval;
348
349 do {
350 oldval = __atomic_load_n ((volatile unsigned int *)ptr, __ATOMIC_RELAXED);
351 failure = __kernel_cmpxchg (ptr, oldval, 0);
352 } while (failure != 0);
353 }
354