xref: /linux/arch/arm64/kernel/entry-ftrace.S (revision d6fd48ef) 
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * arch/arm64/kernel/entry-ftrace.S
4 *
5 * Copyright (C) 2013 Linaro Limited
6 * Author: AKASHI Takahiro <takahiro.akashi@linaro.org>
7 */
8
9#include <linux/linkage.h>
10#include <linux/cfi_types.h>
11#include <asm/asm-offsets.h>
12#include <asm/assembler.h>
13#include <asm/ftrace.h>
14#include <asm/insn.h>
15
16#ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS
17/*
18 * Due to -fpatchable-function-entry=2, the compiler has placed two NOPs before
19 * the regular function prologue. For an enabled callsite, ftrace_init_nop() and
20 * ftrace_make_call() have patched those NOPs to:
21 *
22 * 	MOV	X9, LR
23 * 	BL	ftrace_caller
24 *
25 * Each instrumented function follows the AAPCS, so here x0-x8 and x18-x30 are
26 * live (x18 holds the Shadow Call Stack pointer), and x9-x17 are safe to
27 * clobber.
28 *
29 * We save the callsite's context into a struct ftrace_regs before invoking any
30 * ftrace callbacks. So that we can get a sensible backtrace, we create frame
31 * records for the callsite and the ftrace entry assembly. This is not
32 * sufficient for reliable stacktrace: until we create the callsite stack
33 * record, its caller is missing from the LR and existing chain of frame
34 * records.
35 */
36SYM_CODE_START(ftrace_caller)
37	bti	c
38
39	/* Save original SP */
40	mov	x10, sp
41
42	/* Make room for ftrace regs, plus two frame records */
43	sub	sp, sp, #(FREGS_SIZE + 32)
44
45	/* Save function arguments */
46	stp	x0, x1, [sp, #FREGS_X0]
47	stp	x2, x3, [sp, #FREGS_X2]
48	stp	x4, x5, [sp, #FREGS_X4]
49	stp	x6, x7, [sp, #FREGS_X6]
50	str	x8,     [sp, #FREGS_X8]
51
52	/* Save the callsite's FP, LR, SP */
53	str	x29, [sp, #FREGS_FP]
54	str	x9,  [sp, #FREGS_LR]
55	str	x10, [sp, #FREGS_SP]
56
57	/* Save the PC after the ftrace callsite */
58	str	x30, [sp, #FREGS_PC]
59
60	/* Create a frame record for the callsite above the ftrace regs */
61	stp	x29, x9, [sp, #FREGS_SIZE + 16]
62	add	x29, sp, #FREGS_SIZE + 16
63
64	/* Create our frame record above the ftrace regs */
65	stp	x29, x30, [sp, #FREGS_SIZE]
66	add	x29, sp, #FREGS_SIZE
67
68	/* Prepare arguments for the the tracer func */
69	sub	x0, x30, #AARCH64_INSN_SIZE		// ip (callsite's BL insn)
70	mov	x1, x9					// parent_ip (callsite's LR)
71	mov	x3, sp					// regs
72
73#ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS
74	/*
75	 * The literal pointer to the ops is at an 8-byte aligned boundary
76	 * which is either 12 or 16 bytes before the BL instruction in the call
77	 * site. See ftrace_call_adjust() for details.
78	 *
79	 * Therefore here the LR points at `literal + 16` or `literal + 20`,
80	 * and we can find the address of the literal in either case by
81	 * aligning to an 8-byte boundary and subtracting 16. We do the
82	 * alignment first as this allows us to fold the subtraction into the
83	 * LDR.
84	 */
85	bic	x2, x30, 0x7
86	ldr	x2, [x2, #-16]				// op
87
88	ldr	x4, [x2, #FTRACE_OPS_FUNC]		// op->func
89	blr	x4					// op->func(ip, parent_ip, op, regs)
90
91#else
92	ldr_l   x2, function_trace_op			// op
93
94SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL)
95	bl      ftrace_stub				// func(ip, parent_ip, op, regs)
96#endif
97
98/*
99 * At the callsite x0-x8 and x19-x30 were live. Any C code will have preserved
100 * x19-x29 per the AAPCS, and we created frame records upon entry, so we need
101 * to restore x0-x8, x29, and x30.
102 */
103	/* Restore function arguments */
104	ldp	x0, x1, [sp, #FREGS_X0]
105	ldp	x2, x3, [sp, #FREGS_X2]
106	ldp	x4, x5, [sp, #FREGS_X4]
107	ldp	x6, x7, [sp, #FREGS_X6]
108	ldr	x8,     [sp, #FREGS_X8]
109
110	/* Restore the callsite's FP, LR, PC */
111	ldr	x29, [sp, #FREGS_FP]
112	ldr	x30, [sp, #FREGS_LR]
113	ldr	x9,  [sp, #FREGS_PC]
114
115	/* Restore the callsite's SP */
116	add	sp, sp, #FREGS_SIZE + 32
117
118	ret	x9
119SYM_CODE_END(ftrace_caller)
120
121#else /* CONFIG_DYNAMIC_FTRACE_WITH_ARGS */
122
123/*
124 * Gcc with -pg will put the following code in the beginning of each function:
125 *      mov x0, x30
126 *      bl _mcount
127 *	[function's body ...]
128 * "bl _mcount" may be replaced to "bl ftrace_caller" or NOP if dynamic
129 * ftrace is enabled.
130 *
131 * Please note that x0 as an argument will not be used here because we can
132 * get lr(x30) of instrumented function at any time by winding up call stack
133 * as long as the kernel is compiled without -fomit-frame-pointer.
134 * (or CONFIG_FRAME_POINTER, this is forced on arm64)
135 *
136 * stack layout after mcount_enter in _mcount():
137 *
138 * current sp/fp =>  0:+-----+
139 * in _mcount()        | x29 | -> instrumented function's fp
140 *                     +-----+
141 *                     | x30 | -> _mcount()'s lr (= instrumented function's pc)
142 * old sp       => +16:+-----+
143 * when instrumented   |     |
144 * function calls      | ... |
145 * _mcount()           |     |
146 *                     |     |
147 * instrumented => +xx:+-----+
148 * function's fp       | x29 | -> parent's fp
149 *                     +-----+
150 *                     | x30 | -> instrumented function's lr (= parent's pc)
151 *                     +-----+
152 *                     | ... |
153 */
154
155	.macro mcount_enter
156	stp	x29, x30, [sp, #-16]!
157	mov	x29, sp
158	.endm
159
160	.macro mcount_exit
161	ldp	x29, x30, [sp], #16
162	ret
163	.endm
164
165	.macro mcount_adjust_addr rd, rn
166	sub	\rd, \rn, #AARCH64_INSN_SIZE
167	.endm
168
169	/* for instrumented function's parent */
170	.macro mcount_get_parent_fp reg
171	ldr	\reg, [x29]
172	ldr	\reg, [\reg]
173	.endm
174
175	/* for instrumented function */
176	.macro mcount_get_pc0 reg
177	mcount_adjust_addr	\reg, x30
178	.endm
179
180	.macro mcount_get_pc reg
181	ldr	\reg, [x29, #8]
182	mcount_adjust_addr	\reg, \reg
183	.endm
184
185	.macro mcount_get_lr reg
186	ldr	\reg, [x29]
187	ldr	\reg, [\reg, #8]
188	.endm
189
190	.macro mcount_get_lr_addr reg
191	ldr	\reg, [x29]
192	add	\reg, \reg, #8
193	.endm
194
195/*
196 * _mcount() is used to build the kernel with -pg option, but all the branch
197 * instructions to _mcount() are replaced to NOP initially at kernel start up,
198 * and later on, NOP to branch to ftrace_caller() when enabled or branch to
199 * NOP when disabled per-function base.
200 */
201SYM_FUNC_START(_mcount)
202	ret
203SYM_FUNC_END(_mcount)
204EXPORT_SYMBOL(_mcount)
205NOKPROBE(_mcount)
206
207/*
208 * void ftrace_caller(unsigned long return_address)
209 * @return_address: return address to instrumented function
210 *
211 * This function is a counterpart of _mcount() in 'static' ftrace, and
212 * makes calls to:
213 *     - tracer function to probe instrumented function's entry,
214 *     - ftrace_graph_caller to set up an exit hook
215 */
216SYM_FUNC_START(ftrace_caller)
217	mcount_enter
218
219	mcount_get_pc0	x0		//     function's pc
220	mcount_get_lr	x1		//     function's lr
221
222SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL)	// tracer(pc, lr);
223	nop				// This will be replaced with "bl xxx"
224					// where xxx can be any kind of tracer.
225
226#ifdef CONFIG_FUNCTION_GRAPH_TRACER
227SYM_INNER_LABEL(ftrace_graph_call, SYM_L_GLOBAL) // ftrace_graph_caller();
228	nop				// If enabled, this will be replaced
229					// "b ftrace_graph_caller"
230#endif
231
232	mcount_exit
233SYM_FUNC_END(ftrace_caller)
234
235#ifdef CONFIG_FUNCTION_GRAPH_TRACER
236/*
237 * void ftrace_graph_caller(void)
238 *
239 * Called from _mcount() or ftrace_caller() when function_graph tracer is
240 * selected.
241 * This function w/ prepare_ftrace_return() fakes link register's value on
242 * the call stack in order to intercept instrumented function's return path
243 * and run return_to_handler() later on its exit.
244 */
245SYM_FUNC_START(ftrace_graph_caller)
246	mcount_get_pc		  x0	//     function's pc
247	mcount_get_lr_addr	  x1	//     pointer to function's saved lr
248	mcount_get_parent_fp	  x2	//     parent's fp
249	bl	prepare_ftrace_return	// prepare_ftrace_return(pc, &lr, fp)
250
251	mcount_exit
252SYM_FUNC_END(ftrace_graph_caller)
253#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
254#endif /* CONFIG_DYNAMIC_FTRACE_WITH_ARGS */
255
256SYM_TYPED_FUNC_START(ftrace_stub)
257	ret
258SYM_FUNC_END(ftrace_stub)
259
260#ifdef CONFIG_FUNCTION_GRAPH_TRACER
261SYM_TYPED_FUNC_START(ftrace_stub_graph)
262	ret
263SYM_FUNC_END(ftrace_stub_graph)
264
265/*
266 * void return_to_handler(void)
267 *
268 * Run ftrace_return_to_handler() before going back to parent.
269 * @fp is checked against the value passed by ftrace_graph_caller().
270 */
271SYM_CODE_START(return_to_handler)
272	/* save return value regs */
273	sub sp, sp, #64
274	stp x0, x1, [sp]
275	stp x2, x3, [sp, #16]
276	stp x4, x5, [sp, #32]
277	stp x6, x7, [sp, #48]
278
279	mov	x0, x29			//     parent's fp
280	bl	ftrace_return_to_handler// addr = ftrace_return_to_hander(fp);
281	mov	x30, x0			// restore the original return address
282
283	/* restore return value regs */
284	ldp x0, x1, [sp]
285	ldp x2, x3, [sp, #16]
286	ldp x4, x5, [sp, #32]
287	ldp x6, x7, [sp, #48]
288	add sp, sp, #64
289
290	ret
291SYM_CODE_END(return_to_handler)
292#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
293