1// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
2// See https://llvm.org/LICENSE.txt for license information.
3// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
4
5// This patch implements the support routines for the SME ABI,
6// described here:
7//  https://github.com/ARM-software/abi-aa/blob/main/aapcs64/aapcs64.rst#sme-support-routines
8
9#include "../assembly.h"
10
11
12#if !defined(__APPLE__)
13#define TPIDR2_SYMBOL SYMBOL_NAME(__aarch64_has_sme_and_tpidr2_el0)
14#define TPIDR2_SYMBOL_OFFSET :lo12:SYMBOL_NAME(__aarch64_has_sme_and_tpidr2_el0)
15#else
16// MachO requires @page/@pageoff directives because the global is defined
17// in a different file. Otherwise this file may fail to build.
18#define TPIDR2_SYMBOL SYMBOL_NAME(__aarch64_has_sme_and_tpidr2_el0)@page
19#define TPIDR2_SYMBOL_OFFSET SYMBOL_NAME(__aarch64_has_sme_and_tpidr2_el0)@pageoff
20#endif
21
22.arch armv9-a+sme
23
24// Utility function which calls a system's abort() routine. Because the function
25// is streaming-compatible it should disable streaming-SVE mode before calling
26// abort(). Note that there is no need to preserve any state before the call,
27// because the function does not return.
28DEFINE_COMPILERRT_PRIVATE_FUNCTION(do_abort)
29.cfi_startproc
30	.variant_pcs SYMBOL_NAME(do_abort)
31	stp	x29, x30, [sp, #-32]!
32  cntd x0
33  // Store VG to a stack location that we describe with .cfi_offset
34  str x0, [sp, #16]
35  .cfi_def_cfa_offset 32
36  .cfi_offset w30, -24
37  .cfi_offset w29, -32
38  .cfi_offset 46, -16
39	bl	__arm_sme_state
40	tbz	x0, #0, 2f
411:
42	smstop sm
432:
44  // We can't make this into a tail-call because the unwinder would
45  // need to restore the value of VG.
46	bl	SYMBOL_NAME(abort)
47.cfi_endproc
48END_COMPILERRT_FUNCTION(do_abort)
49
50// __arm_sme_state fills the result registers based on a local
51// that is set as part of the compiler-rt startup code.
52//   __aarch64_has_sme_and_tpidr2_el0
53DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(__arm_sme_state)
54	.variant_pcs __arm_sme_state
55  mov x0, xzr
56  mov x1, xzr
57
58  adrp  x16, TPIDR2_SYMBOL
59  ldrb w16, [x16, TPIDR2_SYMBOL_OFFSET]
60  cbz w16, 1f
610:
62  orr x0, x0, #0xC000000000000000
63  mrs x16, SVCR
64  bfxil x0, x16, #0, #2
65  mrs x1, TPIDR2_EL0
661:
67  ret
68END_COMPILERRT_OUTLINE_FUNCTION(__arm_sme_state)
69
70DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(__arm_tpidr2_restore)
71	.variant_pcs __arm_tpidr2_restore
72  // If TPIDR2_EL0 is nonnull, the subroutine aborts in some platform-specific
73  // manner.
74  mrs x14, TPIDR2_EL0
75  cbnz  x14, 2f
76
77  // If any of the reserved bytes in the first 16 bytes of BLK are nonzero,
78  // the subroutine [..] aborts in some platform-defined manner.
79  ldrh  w14, [x0, #10]
80  cbnz  w14, 2f
81  ldr w14, [x0, #12]
82  cbnz  w14, 2f
83
84  // If BLK.za_save_buffer is NULL, the subroutine does nothing.
85  ldr x16, [x0]
86  cbz x16, 1f
87
88  // If BLK.num_za_save_slices is zero, the subroutine does nothing.
89  ldrh  w14, [x0, #8]
90  cbz x14, 1f
91
92  mov x15, xzr
930:
94  ldr za[w15,0], [x16]
95  addsvl x16, x16, #1
96  add x15, x15, #1
97  cmp x14, x15
98  b.ne  0b
991:
100  ret
1012:
102  b  SYMBOL_NAME(do_abort)
103END_COMPILERRT_OUTLINE_FUNCTION(__arm_tpidr2_restore)
104
105DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(__arm_tpidr2_save)
106	.variant_pcs __arm_tpidr2_restore
107  // If the current thread does not have access to TPIDR2_EL0, the subroutine
108  // does nothing.
109  adrp  x14, TPIDR2_SYMBOL
110  ldrb w14, [x14, TPIDR2_SYMBOL_OFFSET]
111  cbz w14, 1f
112
113  // If TPIDR2_EL0 is null, the subroutine does nothing.
114  mrs x16, TPIDR2_EL0
115  cbz x16, 1f
116
117  // If any of the reserved bytes in the first 16 bytes of the TPIDR2 block are
118  // nonzero, the subroutine [..] aborts in some platform-defined manner.
119  ldrh  w14, [x16, #10]
120  cbnz  w14, 2f
121  ldr w14, [x16, #12]
122  cbnz  w14, 2f
123
124  // If num_za_save_slices is zero, the subroutine does nothing.
125  ldrh  w14, [x16, #8]
126  cbz x14, 1f
127
128  // If za_save_buffer is NULL, the subroutine does nothing.
129  ldr x16, [x16]
130  cbz x16, 1f
131
132  mov x15, xzr
1330:
134  str za[w15,0], [x16]
135  addsvl x16, x16, #1
136  add x15, x15, #1
137  cmp x14, x15
138  b.ne  0b
1391:
140  ret
1412:
142  b  SYMBOL_NAME(do_abort)
143END_COMPILERRT_OUTLINE_FUNCTION(__arm_tpidr2_save)
144
145DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(__arm_za_disable)
146	.variant_pcs __arm_tpidr2_restore
147  // If the current thread does not have access to SME, the subroutine does
148  // nothing.
149  adrp  x14, TPIDR2_SYMBOL
150  ldrb w14, [x14, TPIDR2_SYMBOL_OFFSET]
151  cbz w14, 0f
152
153  // Otherwise, the subroutine behaves as if it did the following:
154  // * Call __arm_tpidr2_save.
155  stp x29, x30, [sp, #-16]!
156  .cfi_def_cfa_offset 16
157  mov x29, sp
158  .cfi_def_cfa w29, 16
159  .cfi_offset w30, -8
160  .cfi_offset w29, -16
161  bl  __arm_tpidr2_save
162
163  // * Set TPIDR2_EL0 to null.
164  msr TPIDR2_EL0, xzr
165
166  // * Set PSTATE.ZA to 0.
167  smstop za
168
169  .cfi_def_cfa wsp, 16
170  ldp x29, x30, [sp], #16
171  .cfi_def_cfa_offset 0
172  .cfi_restore w30
173  .cfi_restore w29
1740:
175  ret
176END_COMPILERRT_OUTLINE_FUNCTION(__arm_za_disable)
177