xref: /linux/arch/arm/vfp/vfphw.S (revision 84b9b44b) 
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 *  linux/arch/arm/vfp/vfphw.S
4 *
5 *  Copyright (C) 2004 ARM Limited.
6 *  Written by Deep Blue Solutions Limited.
7 *
8 * This code is called from the kernel's undefined instruction trap.
9 * r1 holds the thread_info pointer
10 * r3 holds the return address for successful handling.
11 * lr holds the return address for unrecognised instructions.
12 * sp points to a struct pt_regs (as defined in include/asm/proc/ptrace.h)
13 */
14#include <linux/init.h>
15#include <linux/linkage.h>
16#include <asm/thread_info.h>
17#include <asm/vfpmacros.h>
18#include <linux/kern_levels.h>
19#include <asm/assembler.h>
20#include <asm/asm-offsets.h>
21
22	.macro	DBGSTR, str
23#ifdef DEBUG
24	stmfd	sp!, {r0-r3, ip, lr}
25	ldr	r0, =1f
26	bl	_printk
27	ldmfd	sp!, {r0-r3, ip, lr}
28
29	.pushsection .rodata, "a"
301:	.ascii	KERN_DEBUG "VFP: \str\n"
31	.byte	0
32	.previous
33#endif
34	.endm
35
36	.macro  DBGSTR1, str, arg
37#ifdef DEBUG
38	stmfd	sp!, {r0-r3, ip, lr}
39	mov	r1, \arg
40	ldr	r0, =1f
41	bl	_printk
42	ldmfd	sp!, {r0-r3, ip, lr}
43
44	.pushsection .rodata, "a"
451:	.ascii	KERN_DEBUG "VFP: \str\n"
46	.byte	0
47	.previous
48#endif
49	.endm
50
51	.macro  DBGSTR3, str, arg1, arg2, arg3
52#ifdef DEBUG
53	stmfd	sp!, {r0-r3, ip, lr}
54	mov	r3, \arg3
55	mov	r2, \arg2
56	mov	r1, \arg1
57	ldr	r0, =1f
58	bl	_printk
59	ldmfd	sp!, {r0-r3, ip, lr}
60
61	.pushsection .rodata, "a"
621:	.ascii	KERN_DEBUG "VFP: \str\n"
63	.byte	0
64	.previous
65#endif
66	.endm
67
68
69@ VFP hardware support entry point.
70@
71@  r0  = instruction opcode (32-bit ARM or two 16-bit Thumb)
72@  r1  = thread_info pointer
73@  r2  = PC value to resume execution after successful emulation
74@  r3  = normal "successful" return address
75@  lr  = unrecognised instruction return address
76@  IRQs enabled.
77ENTRY(vfp_support_entry)
78	ldr	r11, [r1, #TI_CPU]	@ CPU number
79	add	r10, r1, #TI_VFPSTATE	@ r10 = workspace
80
81	DBGSTR3	"instr %08x pc %08x state %p", r0, r2, r10
82
83	.fpu	vfpv2
84	VFPFMRX	r1, FPEXC		@ Is the VFP enabled?
85	DBGSTR1	"fpexc %08x", r1
86	tst	r1, #FPEXC_EN
87	bne	look_for_VFP_exceptions	@ VFP is already enabled
88
89	DBGSTR1 "enable %x", r10
90	ldr	r9, vfp_current_hw_state_address
91	orr	r1, r1, #FPEXC_EN	@ user FPEXC has the enable bit set
92	ldr	r4, [r9, r11, lsl #2]	@ vfp_current_hw_state pointer
93	bic	r5, r1, #FPEXC_EX	@ make sure exceptions are disabled
94	cmp	r4, r10			@ this thread owns the hw context?
95#ifndef CONFIG_SMP
96	@ For UP, checking that this thread owns the hw context is
97	@ sufficient to determine that the hardware state is valid.
98	beq	vfp_hw_state_valid
99
100	@ On UP, we lazily save the VFP context.  As a different
101	@ thread wants ownership of the VFP hardware, save the old
102	@ state if there was a previous (valid) owner.
103
104	VFPFMXR	FPEXC, r5		@ enable VFP, disable any pending
105					@ exceptions, so we can get at the
106					@ rest of it
107
108	DBGSTR1	"save old state %p", r4
109	cmp	r4, #0			@ if the vfp_current_hw_state is NULL
110	beq	vfp_reload_hw		@ then the hw state needs reloading
111	VFPFSTMIA r4, r5		@ save the working registers
112	VFPFMRX	r5, FPSCR		@ current status
113#ifndef CONFIG_CPU_FEROCEON
114	tst	r1, #FPEXC_EX		@ is there additional state to save?
115	beq	1f
116	VFPFMRX	r6, FPINST		@ FPINST (only if FPEXC.EX is set)
117	tst	r1, #FPEXC_FP2V		@ is there an FPINST2 to read?
118	beq	1f
119	VFPFMRX	r8, FPINST2		@ FPINST2 if needed (and present)
1201:
121#endif
122	stmia	r4, {r1, r5, r6, r8}	@ save FPEXC, FPSCR, FPINST, FPINST2
123vfp_reload_hw:
124
125#else
126	@ For SMP, if this thread does not own the hw context, then we
127	@ need to reload it.  No need to save the old state as on SMP,
128	@ we always save the state when we switch away from a thread.
129	bne	vfp_reload_hw
130
131	@ This thread has ownership of the current hardware context.
132	@ However, it may have been migrated to another CPU, in which
133	@ case the saved state is newer than the hardware context.
134	@ Check this by looking at the CPU number which the state was
135	@ last loaded onto.
136	ldr	ip, [r10, #VFP_CPU]
137	teq	ip, r11
138	beq	vfp_hw_state_valid
139
140vfp_reload_hw:
141	@ We're loading this threads state into the VFP hardware. Update
142	@ the CPU number which contains the most up to date VFP context.
143	str	r11, [r10, #VFP_CPU]
144
145	VFPFMXR	FPEXC, r5		@ enable VFP, disable any pending
146					@ exceptions, so we can get at the
147					@ rest of it
148#endif
149
150	DBGSTR1	"load state %p", r10
151	str	r10, [r9, r11, lsl #2]	@ update the vfp_current_hw_state pointer
152					@ Load the saved state back into the VFP
153	VFPFLDMIA r10, r5		@ reload the working registers while
154					@ FPEXC is in a safe state
155	ldmia	r10, {r1, r5, r6, r8}	@ load FPEXC, FPSCR, FPINST, FPINST2
156#ifndef CONFIG_CPU_FEROCEON
157	tst	r1, #FPEXC_EX		@ is there additional state to restore?
158	beq	1f
159	VFPFMXR	FPINST, r6		@ restore FPINST (only if FPEXC.EX is set)
160	tst	r1, #FPEXC_FP2V		@ is there an FPINST2 to write?
161	beq	1f
162	VFPFMXR	FPINST2, r8		@ FPINST2 if needed (and present)
1631:
164#endif
165	VFPFMXR	FPSCR, r5		@ restore status
166
167@ The context stored in the VFP hardware is up to date with this thread
168vfp_hw_state_valid:
169	tst	r1, #FPEXC_EX
170	bne	process_exception	@ might as well handle the pending
171					@ exception before retrying branch
172					@ out before setting an FPEXC that
173					@ stops us reading stuff
174	VFPFMXR	FPEXC, r1		@ Restore FPEXC last
175	mov	sp, r3			@ we think we have handled things
176	pop	{lr}
177	sub	r2, r2, #4		@ Retry current instruction - if Thumb
178	str	r2, [sp, #S_PC]		@ mode it's two 16-bit instructions,
179					@ else it's one 32-bit instruction, so
180					@ always subtract 4 from the following
181					@ instruction address.
182
183local_bh_enable_and_ret:
184	adr	r0, .
185	mov	r1, #SOFTIRQ_DISABLE_OFFSET
186	b	__local_bh_enable_ip	@ tail call
187
188look_for_VFP_exceptions:
189	@ Check for synchronous or asynchronous exception
190	tst	r1, #FPEXC_EX | FPEXC_DEX
191	bne	process_exception
192	@ On some implementations of the VFP subarch 1, setting FPSCR.IXE
193	@ causes all the CDP instructions to be bounced synchronously without
194	@ setting the FPEXC.EX bit
195	VFPFMRX	r5, FPSCR
196	tst	r5, #FPSCR_IXE
197	bne	process_exception
198
199	tst	r5, #FPSCR_LENGTH_MASK
200	beq	skip
201	orr	r1, r1, #FPEXC_DEX
202	b	process_exception
203skip:
204
205	@ Fall into hand on to next handler - appropriate coproc instr
206	@ not recognised by VFP
207
208	DBGSTR	"not VFP"
209	b	local_bh_enable_and_ret
210
211process_exception:
212	DBGSTR	"bounce"
213	mov	sp, r3			@ setup for a return to the user code.
214	pop	{lr}
215	mov	r2, sp			@ nothing stacked - regdump is at TOS
216
217	@ Now call the C code to package up the bounce to the support code
218	@   r0 holds the trigger instruction
219	@   r1 holds the FPEXC value
220	@   r2 pointer to register dump
221	b	VFP_bounce		@ we have handled this - the support
222					@ code will raise an exception if
223					@ required. If not, the user code will
224					@ retry the faulted instruction
225ENDPROC(vfp_support_entry)
226
227ENTRY(vfp_save_state)
228	@ Save the current VFP state
229	@ r0 - save location
230	@ r1 - FPEXC
231	DBGSTR1	"save VFP state %p", r0
232	VFPFSTMIA r0, r2		@ save the working registers
233	VFPFMRX	r2, FPSCR		@ current status
234	tst	r1, #FPEXC_EX		@ is there additional state to save?
235	beq	1f
236	VFPFMRX	r3, FPINST		@ FPINST (only if FPEXC.EX is set)
237	tst	r1, #FPEXC_FP2V		@ is there an FPINST2 to read?
238	beq	1f
239	VFPFMRX	r12, FPINST2		@ FPINST2 if needed (and present)
2401:
241	stmia	r0, {r1, r2, r3, r12}	@ save FPEXC, FPSCR, FPINST, FPINST2
242	ret	lr
243ENDPROC(vfp_save_state)
244
245	.align
246vfp_current_hw_state_address:
247	.word	vfp_current_hw_state
248
249	.macro	tbl_branch, base, tmp, shift
250#ifdef CONFIG_THUMB2_KERNEL
251	adr	\tmp, 1f
252	add	\tmp, \tmp, \base, lsl \shift
253	ret	\tmp
254#else
255	add	pc, pc, \base, lsl \shift
256	mov	r0, r0
257#endif
2581:
259	.endm
260
261ENTRY(vfp_get_float)
262	tbl_branch r0, r3, #3
263	.fpu	vfpv2
264	.irp	dr,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15
2651:	vmov	r0, s\dr
266	ret	lr
267	.org	1b + 8
268	.endr
269	.irp	dr,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31
2701:	vmov	r0, s\dr
271	ret	lr
272	.org	1b + 8
273	.endr
274ENDPROC(vfp_get_float)
275
276ENTRY(vfp_put_float)
277	tbl_branch r1, r3, #3
278	.fpu	vfpv2
279	.irp	dr,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15
2801:	vmov	s\dr, r0
281	ret	lr
282	.org	1b + 8
283	.endr
284	.irp	dr,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31
2851:	vmov	s\dr, r0
286	ret	lr
287	.org	1b + 8
288	.endr
289ENDPROC(vfp_put_float)
290
291ENTRY(vfp_get_double)
292	tbl_branch r0, r3, #3
293	.fpu	vfpv2
294	.irp	dr,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15
2951:	vmov	r0, r1, d\dr
296	ret	lr
297	.org	1b + 8
298	.endr
299#ifdef CONFIG_VFPv3
300	@ d16 - d31 registers
301	.fpu	vfpv3
302	.irp	dr,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31
3031:	vmov	r0, r1, d\dr
304	ret	lr
305	.org	1b + 8
306	.endr
307#endif
308
309	@ virtual register 16 (or 32 if VFPv3) for compare with zero
310	mov	r0, #0
311	mov	r1, #0
312	ret	lr
313ENDPROC(vfp_get_double)
314
315ENTRY(vfp_put_double)
316	tbl_branch r2, r3, #3
317	.fpu	vfpv2
318	.irp	dr,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15
3191:	vmov	d\dr, r0, r1
320	ret	lr
321	.org	1b + 8
322	.endr
323#ifdef CONFIG_VFPv3
324	.fpu	vfpv3
325	@ d16 - d31 registers
326	.irp	dr,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31
3271:	vmov	d\dr, r0, r1
328	ret	lr
329	.org	1b + 8
330	.endr
331#endif
332ENDPROC(vfp_put_double)
333