1// Copyright 2011 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5// +build !race
6
7#include "textflag.h"
8
9// ARM atomic operations, for use by asm_$(GOOS)_arm.s.
10
11#define DMB_ISHST_7 \
12	MOVB	runtime·goarm(SB), R11; \
13	CMP	$7, R11; \
14	BLT	2(PC); \
15	WORD	$0xf57ff05a	// dmb ishst
16
17#define DMB_ISH_7 \
18	MOVB	runtime·goarm(SB), R11; \
19	CMP	$7, R11; \
20	BLT	2(PC); \
21	WORD	$0xf57ff05b	// dmb ish
22
23TEXT ·armCompareAndSwapUint32(SB),NOSPLIT,$0-13
24	MOVW	addr+0(FP), R1
25	MOVW	old+4(FP), R2
26	MOVW	new+8(FP), R3
27casloop:
28	// LDREX and STREX were introduced in ARMv6.
29	LDREX	(R1), R0
30	CMP	R0, R2
31	BNE	casfail
32	DMB_ISHST_7
33	STREX	R3, (R1), R0
34	CMP	$0, R0
35	BNE	casloop
36	MOVW	$1, R0
37	DMB_ISH_7
38	MOVBU	R0, swapped+12(FP)
39	RET
40casfail:
41	MOVW	$0, R0
42	MOVBU	R0, swapped+12(FP)
43	RET
44
45TEXT ·armCompareAndSwapUint64(SB),NOSPLIT,$0-21
46	BL	fastCheck64<>(SB)
47	MOVW	addr+0(FP), R1
48	// make unaligned atomic access panic
49	AND.S	$7, R1, R2
50	BEQ 	2(PC)
51	MOVW	R2, (R2)
52	MOVW	old_lo+4(FP), R2
53	MOVW	old_hi+8(FP), R3
54	MOVW	new_lo+12(FP), R4
55	MOVW	new_hi+16(FP), R5
56cas64loop:
57	// LDREXD and STREXD were introduced in ARMv6k.
58	LDREXD	(R1), R6  // loads R6 and R7
59	CMP	R2, R6
60	BNE	cas64fail
61	CMP	R3, R7
62	BNE	cas64fail
63	DMB_ISHST_7
64	STREXD	R4, (R1), R0	// stores R4 and R5
65	CMP	$0, R0
66	BNE	cas64loop
67	MOVW	$1, R0
68	DMB_ISH_7
69	MOVBU	R0, swapped+20(FP)
70	RET
71cas64fail:
72	MOVW	$0, R0
73	MOVBU	R0, swapped+20(FP)
74	RET
75
76TEXT ·armAddUint32(SB),NOSPLIT,$0-12
77	MOVW	addr+0(FP), R1
78	MOVW	delta+4(FP), R2
79addloop:
80	// LDREX and STREX were introduced in ARMv6.
81	LDREX	(R1), R3
82	ADD	R2, R3
83	DMB_ISHST_7
84	STREX	R3, (R1), R0
85	CMP	$0, R0
86	BNE	addloop
87	DMB_ISH_7
88	MOVW	R3, new+8(FP)
89	RET
90
91TEXT ·armAddUint64(SB),NOSPLIT,$0-20
92	BL	fastCheck64<>(SB)
93	MOVW	addr+0(FP), R1
94	// make unaligned atomic access panic
95	AND.S	$7, R1, R2
96	BEQ 	2(PC)
97	MOVW	R2, (R2)
98	MOVW	delta_lo+4(FP), R2
99	MOVW	delta_hi+8(FP), R3
100add64loop:
101	// LDREXD and STREXD were introduced in ARMv6k.
102	LDREXD	(R1), R4	// loads R4 and R5
103	ADD.S	R2, R4
104	ADC	R3, R5
105	DMB_ISHST_7
106	STREXD	R4, (R1), R0	// stores R4 and R5
107	CMP	$0, R0
108	BNE	add64loop
109	DMB_ISH_7
110	MOVW	R4, new_lo+12(FP)
111	MOVW	R5, new_hi+16(FP)
112	RET
113
114TEXT ·armSwapUint32(SB),NOSPLIT,$0-12
115	MOVW	addr+0(FP), R1
116	MOVW	new+4(FP), R2
117swaploop:
118	// LDREX and STREX were introduced in ARMv6.
119	LDREX	(R1), R3
120	DMB_ISHST_7
121	STREX	R2, (R1), R0
122	CMP	$0, R0
123	BNE	swaploop
124	DMB_ISH_7
125	MOVW	R3, old+8(FP)
126	RET
127
128TEXT ·armSwapUint64(SB),NOSPLIT,$0-20
129	BL	fastCheck64<>(SB)
130	MOVW	addr+0(FP), R1
131	// make unaligned atomic access panic
132	AND.S	$7, R1, R2
133	BEQ 	2(PC)
134	MOVW	R2, (R2)
135	MOVW	new_lo+4(FP), R2
136	MOVW	new_hi+8(FP), R3
137swap64loop:
138	// LDREXD and STREXD were introduced in ARMv6k.
139	LDREXD	(R1), R4	// loads R4 and R5
140	DMB_ISHST_7
141	STREXD	R2, (R1), R0	// stores R2 and R3
142	CMP	$0, R0
143	BNE	swap64loop
144	DMB_ISH_7
145	MOVW	R4, old_lo+12(FP)
146	MOVW	R5, old_hi+16(FP)
147	RET
148
149TEXT ·armLoadUint64(SB),NOSPLIT,$0-12
150	BL	fastCheck64<>(SB)
151	MOVW	addr+0(FP), R1
152	// make unaligned atomic access panic
153	AND.S	$7, R1, R2
154	BEQ 	2(PC)
155	MOVW	R2, (R2)
156load64loop:
157	LDREXD	(R1), R2	// loads R2 and R3
158	DMB_ISHST_7
159	STREXD	R2, (R1), R0	// stores R2 and R3
160	CMP	$0, R0
161	BNE	load64loop
162	DMB_ISH_7
163	MOVW	R2, val_lo+4(FP)
164	MOVW	R3, val_hi+8(FP)
165	RET
166
167TEXT ·armStoreUint64(SB),NOSPLIT,$0-12
168	BL	fastCheck64<>(SB)
169	MOVW	addr+0(FP), R1
170	// make unaligned atomic access panic
171	AND.S	$7, R1, R2
172	BEQ 	2(PC)
173	MOVW	R2, (R2)
174	MOVW	val_lo+4(FP), R2
175	MOVW	val_hi+8(FP), R3
176store64loop:
177	LDREXD	(R1), R4	// loads R4 and R5
178	DMB_ISHST_7
179	STREXD	R2, (R1), R0	// stores R2 and R3
180	CMP	$0, R0
181	BNE	store64loop
182	DMB_ISH_7
183	RET
184
185// Check for broken 64-bit LDREXD as found in QEMU.
186// LDREXD followed by immediate STREXD should succeed.
187// If it fails, try a few times just to be sure (maybe our thread got
188// rescheduled between the two instructions) and then panic.
189// A bug in some copies of QEMU makes STREXD never succeed,
190// which will make uses of the 64-bit atomic operations loop forever.
191// If things are working, set okLDREXD to avoid future checks.
192// https://bugs.launchpad.net/qemu/+bug/670883.
193TEXT	check64<>(SB),NOSPLIT,$16-0
194	MOVW	$10, R1
195	// 8-aligned stack address scratch space.
196	MOVW	$8(R13), R5
197	AND	$~7, R5
198loop:
199	LDREXD	(R5), R2
200	STREXD	R2, (R5), R0
201	CMP	$0, R0
202	BEQ	ok
203	SUB	$1, R1
204	CMP	$0, R1
205	BNE	loop
206	// Must be buggy QEMU.
207	BL	·panic64(SB)
208ok:
209	RET
210
211// Fast, cached version of check. No frame, just MOVW CMP RET after first time.
212TEXT	fastCheck64<>(SB),NOSPLIT,$-4
213	MOVW	ok64<>(SB), R0
214	CMP	$0, R0	// have we been here before?
215	RET.NE
216	B	slowCheck64<>(SB)
217
218TEXT slowCheck64<>(SB),NOSPLIT,$0-0
219	BL	check64<>(SB)
220	// Still here, must be okay.
221	MOVW	$1, R0
222	MOVW	R0, ok64<>(SB)
223	RET
224
225GLOBL ok64<>(SB), NOPTR, $4
226