xref: /freebsd/sys/arm64/arm64/swtch.S (revision 53b70c86) 
1/*-
2 * Copyright (c) 2014 Andrew Turner
3 * Copyright (c) 2014 The FreeBSD Foundation
4 * All rights reserved.
5 *
6 * This software was developed by Andrew Turner under sponsorship from
7 * the FreeBSD Foundation.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 *    notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 *    notice, this list of conditions and the following disclaimer in the
16 *    documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 *
30 */
31
32#include "assym.inc"
33#include "opt_kstack_pages.h"
34#include "opt_sched.h"
35
36#include <machine/asm.h>
37#include <machine/armreg.h>
38
39__FBSDID("$FreeBSD$");
40
41.macro clear_step_flag pcbflags, tmp
42	tbz	\pcbflags, #PCB_SINGLE_STEP_SHIFT, 999f
43	mrs	\tmp, mdscr_el1
44	bic	\tmp, \tmp, #1
45	msr	mdscr_el1, \tmp
46	isb
47999:
48.endm
49
50.macro set_step_flag pcbflags, tmp
51	tbz	\pcbflags, #PCB_SINGLE_STEP_SHIFT, 999f
52	mrs	\tmp, mdscr_el1
53	orr	\tmp, \tmp, #1
54	msr	mdscr_el1, \tmp
55	isb
56999:
57.endm
58
59/*
60 * void cpu_throw(struct thread *old, struct thread *new)
61 */
62ENTRY(cpu_throw)
63	/* Of old == NULL skip disabling stepping */
64	cbz	x0, 1f
65
66	/* If we were single stepping, disable it */
67	ldr	x4, [x0, #TD_PCB]
68	ldr	w5, [x4, #PCB_FLAGS]
69	clear_step_flag w5, x6
701:
71
72#ifdef VFP
73	/* Backup the new thread pointer around a call to C code */
74	mov	x19, x0
75	mov	x20, x1
76	bl	vfp_discard
77	mov	x1, x20
78	mov	x0, x19
79#endif
80
81	bl	pmap_switch
82	mov	x4, x0
83
84	/* If we are single stepping, enable it */
85	ldr	w5, [x4, #PCB_FLAGS]
86	set_step_flag w5, x6
87
88	/* Restore the registers */
89	ldp	x5, x6, [x4, #PCB_SP]
90	mov	sp, x5
91	msr	tpidr_el0, x6
92	ldr	x6, [x4, #PCB_TPIDRRO]
93	msr	tpidrro_el0, x6
94	ldp	x8, x9, [x4, #PCB_REGS + 8 * 8]
95	ldp	x10, x11, [x4, #PCB_REGS + 10 * 8]
96	ldp	x12, x13, [x4, #PCB_REGS + 12 * 8]
97	ldp	x14, x15, [x4, #PCB_REGS + 14 * 8]
98	ldp	x16, x17, [x4, #PCB_REGS + 16 * 8]
99	ldr	     x19, [x4, #PCB_REGS + 19 * 8]
100	ldp	x20, x21, [x4, #PCB_REGS + 20 * 8]
101	ldp	x22, x23, [x4, #PCB_REGS + 22 * 8]
102	ldp	x24, x25, [x4, #PCB_REGS + 24 * 8]
103	ldp	x26, x27, [x4, #PCB_REGS + 26 * 8]
104	ldp	x28, x29, [x4, #PCB_REGS + 28 * 8]
105	ldr	lr, [x4, #PCB_LR]
106
107	ret
108END(cpu_throw)
109
110/*
111 * void cpu_switch(struct thread *old, struct thread *new, struct mtx *mtx)
112 *
113 * x0 = old
114 * x1 = new
115 * x2 = mtx
116 * x3 to x7, x16 and x17 are caller saved
117 */
118ENTRY(cpu_switch)
119	/*
120	 * Save the old context.
121	 */
122	ldr	x4, [x0, #TD_PCB]
123
124	/* Store the callee-saved registers */
125	stp	x8, x9, [x4, #PCB_REGS + 8 * 8]
126	stp	x10, x11, [x4, #PCB_REGS + 10 * 8]
127	stp	x12, x13, [x4, #PCB_REGS + 12 * 8]
128	stp	x14, x15, [x4, #PCB_REGS + 14 * 8]
129	stp	x16, x17, [x4, #PCB_REGS + 16 * 8]
130	stp	x18, x19, [x4, #PCB_REGS + 18 * 8]
131	stp	x20, x21, [x4, #PCB_REGS + 20 * 8]
132	stp	x22, x23, [x4, #PCB_REGS + 22 * 8]
133	stp	x24, x25, [x4, #PCB_REGS + 24 * 8]
134	stp	x26, x27, [x4, #PCB_REGS + 26 * 8]
135	stp	x28, x29, [x4, #PCB_REGS + 28 * 8]
136	str	lr, [x4, #PCB_LR]
137	/* And the old stack pointer */
138	mov	x5, sp
139	mrs	x6, tpidrro_el0
140	str	x6, [x4, #PCB_TPIDRRO]
141	mrs	x6, tpidr_el0
142	stp	x5, x6, [x4, #PCB_SP]
143
144	/* If we were single stepping, disable it */
145	ldr	w5, [x4, #PCB_FLAGS]
146	clear_step_flag w5, x6
147
148	mov	x19, x0
149	mov	x20, x1
150	mov	x21, x2
151
152#ifdef VFP
153	/* Load the pcb address */
154	mov	x1, x4
155	bl	vfp_save_state
156	mov	x1, x20
157	mov	x0, x19
158#endif
159
160	bl	pmap_switch
161	/* Move the new pcb out of the way */
162	mov	x4, x0
163
164	mov	x2, x21
165	mov	x1, x20
166	mov	x0, x19
167
168	/*
169	 * Release the old thread.
170	 */
171	stlr	x2, [x0, #TD_LOCK]
172#if defined(SCHED_ULE) && defined(SMP)
173	/* Spin if TD_LOCK points to a blocked_lock */
174	ldr	x2, =_C_LABEL(blocked_lock)
1751:
176	ldar	x3, [x1, #TD_LOCK]
177	cmp	x3, x2
178	b.eq	1b
179#endif
180
181	/* If we are single stepping, enable it */
182	ldr	w5, [x4, #PCB_FLAGS]
183	set_step_flag w5, x6
184
185	/* Restore the registers */
186	ldp	x5, x6, [x4, #PCB_SP]
187	mov	sp, x5
188	msr	tpidr_el0, x6
189	ldr	x6, [x4, #PCB_TPIDRRO]
190	msr	tpidrro_el0, x6
191	ldp	x8, x9, [x4, #PCB_REGS + 8 * 8]
192	ldp	x10, x11, [x4, #PCB_REGS + 10 * 8]
193	ldp	x12, x13, [x4, #PCB_REGS + 12 * 8]
194	ldp	x14, x15, [x4, #PCB_REGS + 14 * 8]
195	ldp	x16, x17, [x4, #PCB_REGS + 16 * 8]
196	ldr	     x19, [x4, #PCB_REGS + 19 * 8]
197	ldp	x20, x21, [x4, #PCB_REGS + 20 * 8]
198	ldp	x22, x23, [x4, #PCB_REGS + 22 * 8]
199	ldp	x24, x25, [x4, #PCB_REGS + 24 * 8]
200	ldp	x26, x27, [x4, #PCB_REGS + 26 * 8]
201	ldp	x28, x29, [x4, #PCB_REGS + 28 * 8]
202	ldr	lr, [x4, #PCB_LR]
203
204	str	xzr, [x4, #PCB_REGS + 18 * 8]
205	ret
206END(cpu_switch)
207
208ENTRY(fork_trampoline)
209	mov	x0, x8
210	mov	x1, x9
211	mov	x2, sp
212	mov	fp, #0	/* Stack traceback stops here. */
213	bl	_C_LABEL(fork_exit)
214
215	/*
216	 * Disable interrupts to avoid
217	 * overwriting spsr_el1 and sp_el0 by an IRQ exception.
218	 */
219	msr	daifset, #(DAIF_D | DAIF_INTR)
220
221	/* Restore sp, lr, elr, and spsr */
222	ldp	x18, lr, [sp, #TF_SP]
223	ldp	x10, x11, [sp, #TF_ELR]
224	msr	sp_el0, x18
225	msr	spsr_el1, x11
226	msr	elr_el1, x10
227
228	/* Restore the CPU registers */
229	ldp	x0, x1, [sp, #TF_X + 0 * 8]
230	ldp	x2, x3, [sp, #TF_X + 2 * 8]
231	ldp	x4, x5, [sp, #TF_X + 4 * 8]
232	ldp	x6, x7, [sp, #TF_X + 6 * 8]
233	ldp	x8, x9, [sp, #TF_X + 8 * 8]
234	ldp	x10, x11, [sp, #TF_X + 10 * 8]
235	ldp	x12, x13, [sp, #TF_X + 12 * 8]
236	ldp	x14, x15, [sp, #TF_X + 14 * 8]
237	ldp	x16, x17, [sp, #TF_X + 16 * 8]
238	ldp	x18, x19, [sp, #TF_X + 18 * 8]
239	ldp	x20, x21, [sp, #TF_X + 20 * 8]
240	ldp	x22, x23, [sp, #TF_X + 22 * 8]
241	ldp	x24, x25, [sp, #TF_X + 24 * 8]
242	ldp	x26, x27, [sp, #TF_X + 26 * 8]
243	ldp	x28, x29, [sp, #TF_X + 28 * 8]
244
245	/*
246	 * No need for interrupts reenabling since PSR
247	 * will be set to the desired value anyway.
248	 */
249	ERET
250
251END(fork_trampoline)
252
253ENTRY(savectx)
254	/* Store the callee-saved registers */
255	stp	x8,  x9,  [x0, #PCB_REGS + 8 * 8]
256	stp	x10, x11, [x0, #PCB_REGS + 10 * 8]
257	stp	x12, x13, [x0, #PCB_REGS + 12 * 8]
258	stp	x14, x15, [x0, #PCB_REGS + 14 * 8]
259	stp	x16, x17, [x0, #PCB_REGS + 16 * 8]
260	stp	x18, x19, [x0, #PCB_REGS + 18 * 8]
261	stp	x20, x21, [x0, #PCB_REGS + 20 * 8]
262	stp	x22, x23, [x0, #PCB_REGS + 22 * 8]
263	stp	x24, x25, [x0, #PCB_REGS + 24 * 8]
264	stp	x26, x27, [x0, #PCB_REGS + 26 * 8]
265	stp	x28, x29, [x0, #PCB_REGS + 28 * 8]
266	str	lr, [x0, #PCB_LR]
267	/* And the old stack pointer */
268	mov	x5, sp
269	mrs	x6, tpidrro_el0
270	str	x6, [x0, #PCB_TPIDRRO]
271	mrs	x6, tpidr_el0
272	stp	x5, x6, [x0, #PCB_SP]
273
274	/* Store the VFP registers */
275#ifdef VFP
276	mov	x28, lr
277	mov	x1, x0			/* move pcb to the correct register */
278	mov	x0, xzr			/* td = NULL */
279	bl	vfp_save_state
280	mov	lr, x28
281#endif
282
283	ret
284END(savectx)
285
286