1// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
2// See https://llvm.org/LICENSE.txt for license information.
3// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
4
5#include "assembly.h"
6
7// Out-of-line LSE atomics helpers. Ported from libgcc library.
8// N = {1, 2, 4, 8}
9// M = {1, 2, 4, 8, 16}
10// ORDER = {'relax', 'acq', 'rel', 'acq_rel'}
11// Routines implemented:
12//
13//  iM __aarch64_casM_ORDER(iM expected, iM desired, iM *ptr)
14//  iN __aarch64_swpN_ORDER(iN val, iN *ptr)
15//  iN __aarch64_ldaddN_ORDER(iN val, iN *ptr)
16//  iN __aarch64_ldclrN_ORDER(iN val, iN *ptr)
17//  iN __aarch64_ldeorN_ORDER(iN val, iN *ptr)
18//  iN __aarch64_ldsetN_ORDER(iN val, iN *ptr)
19//
20// Routines may modify temporary registers tmp0, tmp1, tmp2,
21// return value x0 and the flags only.
22
23#ifdef __aarch64__
24
25#ifdef HAS_ASM_LSE
26.arch armv8-a+lse
27#else
28.arch armv8-a
29#endif
30
31#if !defined(__APPLE__)
32HIDDEN(__aarch64_have_lse_atomics)
33#else
34HIDDEN(___aarch64_have_lse_atomics)
35#endif
36
37// Generate mnemonics for
38// L_cas:                                 SIZE: 1,2,4,8,16 MODEL: 1,2,3,4
39// L_swp L_ldadd L_ldclr L_ldeor L_ldset: SIZE: 1,2,4,8    MODEL: 1,2,3,4
40
41#if SIZE == 1
42#define S b
43#define UXT uxtb
44#define B 0x00000000
45#elif SIZE == 2
46#define S h
47#define UXT uxth
48#define B 0x40000000
49#elif SIZE == 4 || SIZE == 8 || SIZE == 16
50#define S
51#define UXT mov
52#if SIZE == 4
53#define B 0x80000000
54#elif SIZE == 8
55#define B 0xc0000000
56#endif
57#else
58#error
59#endif // SIZE
60
61#if MODEL == 1
62#define SUFF _relax
63#define A
64#define L
65#define M 0x000000
66#define N 0x000000
67#elif MODEL == 2
68#define SUFF _acq
69#define A a
70#define L
71#define M 0x400000
72#define N 0x800000
73#elif MODEL == 3
74#define SUFF _rel
75#define A
76#define L l
77#define M 0x008000
78#define N 0x400000
79#elif MODEL == 4
80#define SUFF _acq_rel
81#define A a
82#define L l
83#define M 0x408000
84#define N 0xc00000
85#else
86#error
87#endif // MODEL
88
89// Define register size.
90#define x(N) GLUE2(x, N)
91#define w(N) GLUE2(w, N)
92#if SIZE < 8
93#define s(N) w(N)
94#else
95#define s(N) x(N)
96#endif
97
98#define NAME(BASE) GLUE4(__aarch64_, BASE, SIZE, SUFF)
99#define LDXR GLUE4(ld, A, xr, S)
100#define STXR GLUE4(st, L, xr, S)
101
102// Define temporary registers.
103#define tmp0 16
104#define tmp1 17
105#define tmp2 15
106
107// Macro for branch to label if no LSE available
108.macro JUMP_IF_NOT_LSE label
109#if !defined(__APPLE__)
110        adrp    x(tmp0), __aarch64_have_lse_atomics
111        ldrb    w(tmp0), [x(tmp0), :lo12:__aarch64_have_lse_atomics]
112#else
113        adrp    x(tmp0), ___aarch64_have_lse_atomics@page
114        ldrb    w(tmp0), [x(tmp0), ___aarch64_have_lse_atomics@pageoff]
115#endif
116        cbz     w(tmp0), \label
117.endm
118
119#ifdef L_cas
120DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(NAME(cas))
121        JUMP_IF_NOT_LSE 8f
122#if SIZE < 16
123#ifdef HAS_ASM_LSE
124#define CAS GLUE4(cas, A, L, S) s(0), s(1), [x2]
125#else
126#define CAS .inst 0x08a07c41 + B + M
127#endif
128        CAS    // s(0), s(1), [x2]
129        ret
1308:
131        UXT    s(tmp0), s(0)
1320:
133        LDXR   s(0), [x2]
134        cmp    s(0), s(tmp0)
135        bne    1f
136        STXR   w(tmp1), s(1), [x2]
137        cbnz   w(tmp1), 0b
1381:
139        ret
140#else
141#define LDXP GLUE3(ld, A, xp)
142#define STXP GLUE3(st, L, xp)
143#ifdef HAS_ASM_LSE
144#define CASP GLUE3(casp, A, L)  x0, x1, x2, x3, [x4]
145#else
146#define CASP .inst 0x48207c82 + M
147#endif
148
149        CASP   // x0, x1, x2, x3, [x4]
150        ret
1518:
152        mov    x(tmp0), x0
153        mov    x(tmp1), x1
1540:
155        LDXP   x0, x1, [x4]
156        cmp    x0, x(tmp0)
157        ccmp   x1, x(tmp1), #0, eq
158        bne    1f
159        STXP   w(tmp2), x2, x3, [x4]
160        cbnz   w(tmp2), 0b
1611:
162        ret
163#endif
164END_COMPILERRT_OUTLINE_FUNCTION(NAME(cas))
165#endif // L_cas
166
167#ifdef L_swp
168#ifdef HAS_ASM_LSE
169#define SWP GLUE4(swp, A, L, S)  s(0), s(0), [x1]
170#else
171#define SWP .inst 0x38208020 + B + N
172#endif
173DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(NAME(swp))
174        JUMP_IF_NOT_LSE 8f
175        SWP    // s(0), s(0), [x1]
176        ret
1778:
178        mov    s(tmp0), s(0)
1790:
180        LDXR   s(0), [x1]
181        STXR   w(tmp1), s(tmp0), [x1]
182        cbnz   w(tmp1), 0b
183        ret
184END_COMPILERRT_OUTLINE_FUNCTION(NAME(swp))
185#endif // L_swp
186
187#if defined(L_ldadd) || defined(L_ldclr) ||                                    \
188    defined(L_ldeor) || defined(L_ldset)
189
190#ifdef L_ldadd
191#define LDNM ldadd
192#define OP add
193#define OPN 0x0000
194#elif defined(L_ldclr)
195#define LDNM ldclr
196#define OP bic
197#define OPN 0x1000
198#elif defined(L_ldeor)
199#define LDNM ldeor
200#define OP eor
201#define OPN 0x2000
202#elif defined(L_ldset)
203#define LDNM ldset
204#define OP orr
205#define OPN 0x3000
206#else
207#error
208#endif
209
210#ifdef HAS_ASM_LSE
211#define LDOP GLUE4(LDNM, A, L, S) s(0), s(0), [x1]
212#else
213#define LDOP .inst 0x38200020 + OPN + B + N
214#endif
215
216DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(NAME(LDNM))
217        JUMP_IF_NOT_LSE 8f
218        LDOP // s(0), s(0), [x1]
219        ret
2208:
221        mov    s(tmp0), s(0)
2220:
223        LDXR   s(0), [x1]
224        OP     s(tmp1), s(0), s(tmp0)
225        STXR   w(tmp2), s(tmp1), [x1]
226        cbnz   w(tmp2), 0b
227        ret
228END_COMPILERRT_OUTLINE_FUNCTION(NAME(LDNM))
229#endif // L_ldadd L_ldclr L_ldeor L_ldset
230
231NO_EXEC_STACK_DIRECTIVE
232
233// GNU property note for BTI and PAC
234GNU_PROPERTY_BTI_PAC
235
236#endif // __aarch64__
237