1 /* atomic.c -- implement atomic routines for Go.
2
3 Copyright 2011 The Go Authors. All rights reserved.
4 Use of this source code is governed by a BSD-style
5 license that can be found in the LICENSE file. */
6
7 #include <stdint.h>
8
9 #include "runtime.h"
10
11 int32_t SwapInt32 (int32_t *, int32_t)
12 __asm__ (GOSYM_PREFIX "sync_atomic.SwapInt32")
13 __attribute__ ((no_split_stack));
14
15 int32_t
SwapInt32(int32_t * addr,int32_t new)16 SwapInt32 (int32_t *addr, int32_t new)
17 {
18 return __atomic_exchange_n (addr, new, __ATOMIC_SEQ_CST);
19 }
20
21 int64_t SwapInt64 (int64_t *, int64_t)
22 __asm__ (GOSYM_PREFIX "sync_atomic.SwapInt64")
23 __attribute__ ((no_split_stack));
24
25 int64_t
SwapInt64(int64_t * addr,int64_t new)26 SwapInt64 (int64_t *addr, int64_t new)
27 {
28 if (((uintptr_t) addr & 7) != 0)
29 panicmem ();
30 return __atomic_exchange_n (addr, new, __ATOMIC_SEQ_CST);
31 }
32
33 uint32_t SwapUint32 (uint32_t *, uint32_t)
34 __asm__ (GOSYM_PREFIX "sync_atomic.SwapUint32")
35 __attribute__ ((no_split_stack));
36
37 uint32_t
SwapUint32(uint32_t * addr,uint32_t new)38 SwapUint32 (uint32_t *addr, uint32_t new)
39 {
40 return __atomic_exchange_n (addr, new, __ATOMIC_SEQ_CST);
41 }
42
43 uint64_t SwapUint64 (uint64_t *, uint64_t)
44 __asm__ (GOSYM_PREFIX "sync_atomic.SwapUint64")
45 __attribute__ ((no_split_stack));
46
47 uint64_t
SwapUint64(uint64_t * addr,uint64_t new)48 SwapUint64 (uint64_t *addr, uint64_t new)
49 {
50 if (((uintptr_t) addr & 7) != 0)
51 panicmem ();
52 return __atomic_exchange_n (addr, new, __ATOMIC_SEQ_CST);
53 }
54
55 uintptr_t SwapUintptr (uintptr_t *, uintptr_t)
56 __asm__ (GOSYM_PREFIX "sync_atomic.SwapUintptr")
57 __attribute__ ((no_split_stack));
58
59 uintptr_t
SwapUintptr(uintptr_t * addr,uintptr_t new)60 SwapUintptr (uintptr_t *addr, uintptr_t new)
61 {
62 return __atomic_exchange_n (addr, new, __ATOMIC_SEQ_CST);
63 }
64
65 _Bool CompareAndSwapInt32 (int32_t *, int32_t, int32_t)
66 __asm__ (GOSYM_PREFIX "sync_atomic.CompareAndSwapInt32")
67 __attribute__ ((no_split_stack));
68
69 _Bool
CompareAndSwapInt32(int32_t * val,int32_t old,int32_t new)70 CompareAndSwapInt32 (int32_t *val, int32_t old, int32_t new)
71 {
72 return __sync_bool_compare_and_swap (val, old, new);
73 }
74
75 _Bool CompareAndSwapInt64 (int64_t *, int64_t, int64_t)
76 __asm__ (GOSYM_PREFIX "sync_atomic.CompareAndSwapInt64")
77 __attribute__ ((no_split_stack));
78
79 _Bool
CompareAndSwapInt64(int64_t * val,int64_t old,int64_t new)80 CompareAndSwapInt64 (int64_t *val, int64_t old, int64_t new)
81 {
82 if (((uintptr_t) val & 7) != 0)
83 val = NULL;
84 return __sync_bool_compare_and_swap (val, old, new);
85 }
86
87 _Bool CompareAndSwapUint32 (uint32_t *, uint32_t, uint32_t)
88 __asm__ (GOSYM_PREFIX "sync_atomic.CompareAndSwapUint32")
89 __attribute__ ((no_split_stack));
90
91 _Bool
CompareAndSwapUint32(uint32_t * val,uint32_t old,uint32_t new)92 CompareAndSwapUint32 (uint32_t *val, uint32_t old, uint32_t new)
93 {
94 return __sync_bool_compare_and_swap (val, old, new);
95 }
96
97 _Bool CompareAndSwapUint64 (uint64_t *, uint64_t, uint64_t)
98 __asm__ (GOSYM_PREFIX "sync_atomic.CompareAndSwapUint64")
99 __attribute__ ((no_split_stack));
100
101 _Bool
CompareAndSwapUint64(uint64_t * val,uint64_t old,uint64_t new)102 CompareAndSwapUint64 (uint64_t *val, uint64_t old, uint64_t new)
103 {
104 if (((uintptr_t) val & 7) != 0)
105 val = NULL;
106 return __sync_bool_compare_and_swap (val, old, new);
107 }
108
109 _Bool CompareAndSwapUintptr (uintptr_t *, uintptr_t, uintptr_t)
110 __asm__ (GOSYM_PREFIX "sync_atomic.CompareAndSwapUintptr")
111 __attribute__ ((no_split_stack));
112
113 _Bool
CompareAndSwapUintptr(uintptr_t * val,uintptr_t old,uintptr_t new)114 CompareAndSwapUintptr (uintptr_t *val, uintptr_t old, uintptr_t new)
115 {
116 return __sync_bool_compare_and_swap (val, old, new);
117 }
118
119 int32_t AddInt32 (int32_t *, int32_t)
120 __asm__ (GOSYM_PREFIX "sync_atomic.AddInt32")
121 __attribute__ ((no_split_stack));
122
123 int32_t
AddInt32(int32_t * val,int32_t delta)124 AddInt32 (int32_t *val, int32_t delta)
125 {
126 return __sync_add_and_fetch (val, delta);
127 }
128
129 uint32_t AddUint32 (uint32_t *, uint32_t)
130 __asm__ (GOSYM_PREFIX "sync_atomic.AddUint32")
131 __attribute__ ((no_split_stack));
132
133 uint32_t
AddUint32(uint32_t * val,uint32_t delta)134 AddUint32 (uint32_t *val, uint32_t delta)
135 {
136 return __sync_add_and_fetch (val, delta);
137 }
138
139 int64_t AddInt64 (int64_t *, int64_t)
140 __asm__ (GOSYM_PREFIX "sync_atomic.AddInt64")
141 __attribute__ ((no_split_stack));
142
143 int64_t
AddInt64(int64_t * val,int64_t delta)144 AddInt64 (int64_t *val, int64_t delta)
145 {
146 if (((uintptr_t) val & 7) != 0)
147 val = NULL;
148 return __sync_add_and_fetch (val, delta);
149 }
150
151 uint64_t AddUint64 (uint64_t *, uint64_t)
152 __asm__ (GOSYM_PREFIX "sync_atomic.AddUint64")
153 __attribute__ ((no_split_stack));
154
155 uint64_t
AddUint64(uint64_t * val,uint64_t delta)156 AddUint64 (uint64_t *val, uint64_t delta)
157 {
158 if (((uintptr_t) val & 7) != 0)
159 val = NULL;
160 return __sync_add_and_fetch (val, delta);
161 }
162
163 uintptr_t AddUintptr (uintptr_t *, uintptr_t)
164 __asm__ (GOSYM_PREFIX "sync_atomic.AddUintptr")
165 __attribute__ ((no_split_stack));
166
167 uintptr_t
AddUintptr(uintptr_t * val,uintptr_t delta)168 AddUintptr (uintptr_t *val, uintptr_t delta)
169 {
170 return __sync_add_and_fetch (val, delta);
171 }
172
173 int32_t LoadInt32 (int32_t *addr)
174 __asm__ (GOSYM_PREFIX "sync_atomic.LoadInt32")
175 __attribute__ ((no_split_stack));
176
177 int32_t
LoadInt32(int32_t * addr)178 LoadInt32 (int32_t *addr)
179 {
180 int32_t v;
181
182 v = *addr;
183 while (! __sync_bool_compare_and_swap (addr, v, v))
184 v = *addr;
185 return v;
186 }
187
188 int64_t LoadInt64 (int64_t *addr)
189 __asm__ (GOSYM_PREFIX "sync_atomic.LoadInt64")
190 __attribute__ ((no_split_stack));
191
192 int64_t
LoadInt64(int64_t * addr)193 LoadInt64 (int64_t *addr)
194 {
195 int64_t v;
196
197 if (((uintptr_t) addr & 7) != 0)
198 panicmem ();
199 v = *addr;
200 while (! __sync_bool_compare_and_swap (addr, v, v))
201 v = *addr;
202 return v;
203 }
204
205 uint32_t LoadUint32 (uint32_t *addr)
206 __asm__ (GOSYM_PREFIX "sync_atomic.LoadUint32")
207 __attribute__ ((no_split_stack));
208
209 uint32_t
LoadUint32(uint32_t * addr)210 LoadUint32 (uint32_t *addr)
211 {
212 uint32_t v;
213
214 v = *addr;
215 while (! __sync_bool_compare_and_swap (addr, v, v))
216 v = *addr;
217 return v;
218 }
219
220 uint64_t LoadUint64 (uint64_t *addr)
221 __asm__ (GOSYM_PREFIX "sync_atomic.LoadUint64")
222 __attribute__ ((no_split_stack));
223
224 uint64_t
LoadUint64(uint64_t * addr)225 LoadUint64 (uint64_t *addr)
226 {
227 uint64_t v;
228
229 if (((uintptr_t) addr & 7) != 0)
230 panicmem ();
231 v = *addr;
232 while (! __sync_bool_compare_and_swap (addr, v, v))
233 v = *addr;
234 return v;
235 }
236
237 uintptr_t LoadUintptr (uintptr_t *addr)
238 __asm__ (GOSYM_PREFIX "sync_atomic.LoadUintptr")
239 __attribute__ ((no_split_stack));
240
241 uintptr_t
LoadUintptr(uintptr_t * addr)242 LoadUintptr (uintptr_t *addr)
243 {
244 uintptr_t v;
245
246 v = *addr;
247 while (! __sync_bool_compare_and_swap (addr, v, v))
248 v = *addr;
249 return v;
250 }
251
252 void *LoadPointer (void **addr)
253 __asm__ (GOSYM_PREFIX "sync_atomic.LoadPointer")
254 __attribute__ ((no_split_stack));
255
256 void *
LoadPointer(void ** addr)257 LoadPointer (void **addr)
258 {
259 void *v;
260
261 v = *addr;
262 while (! __sync_bool_compare_and_swap (addr, v, v))
263 v = *addr;
264 return v;
265 }
266
267 void StoreInt32 (int32_t *addr, int32_t val)
268 __asm__ (GOSYM_PREFIX "sync_atomic.StoreInt32")
269 __attribute__ ((no_split_stack));
270
271 void
StoreInt32(int32_t * addr,int32_t val)272 StoreInt32 (int32_t *addr, int32_t val)
273 {
274 int32_t v;
275
276 v = *addr;
277 while (! __sync_bool_compare_and_swap (addr, v, val))
278 v = *addr;
279 }
280
281 void StoreInt64 (int64_t *addr, int64_t val)
282 __asm__ (GOSYM_PREFIX "sync_atomic.StoreInt64")
283 __attribute__ ((no_split_stack));
284
285 void
StoreInt64(int64_t * addr,int64_t val)286 StoreInt64 (int64_t *addr, int64_t val)
287 {
288 int64_t v;
289
290 if (((uintptr_t) addr & 7) != 0)
291 panicmem ();
292 v = *addr;
293 while (! __sync_bool_compare_and_swap (addr, v, val))
294 v = *addr;
295 }
296
297 void StoreUint32 (uint32_t *addr, uint32_t val)
298 __asm__ (GOSYM_PREFIX "sync_atomic.StoreUint32")
299 __attribute__ ((no_split_stack));
300
301 void
StoreUint32(uint32_t * addr,uint32_t val)302 StoreUint32 (uint32_t *addr, uint32_t val)
303 {
304 uint32_t v;
305
306 v = *addr;
307 while (! __sync_bool_compare_and_swap (addr, v, val))
308 v = *addr;
309 }
310
311 void StoreUint64 (uint64_t *addr, uint64_t val)
312 __asm__ (GOSYM_PREFIX "sync_atomic.StoreUint64")
313 __attribute__ ((no_split_stack));
314
315 void
StoreUint64(uint64_t * addr,uint64_t val)316 StoreUint64 (uint64_t *addr, uint64_t val)
317 {
318 uint64_t v;
319
320 if (((uintptr_t) addr & 7) != 0)
321 panicmem ();
322 v = *addr;
323 while (! __sync_bool_compare_and_swap (addr, v, val))
324 v = *addr;
325 }
326
327 void StoreUintptr (uintptr_t *addr, uintptr_t val)
328 __asm__ (GOSYM_PREFIX "sync_atomic.StoreUintptr")
329 __attribute__ ((no_split_stack));
330
331 void
StoreUintptr(uintptr_t * addr,uintptr_t val)332 StoreUintptr (uintptr_t *addr, uintptr_t val)
333 {
334 uintptr_t v;
335
336 v = *addr;
337 while (! __sync_bool_compare_and_swap (addr, v, val))
338 v = *addr;
339 }
340