xref: /linux/arch/xtensa/kernel/traps.c (revision dd093fb0) 
1 /*
2  * arch/xtensa/kernel/traps.c
3  *
4  * Exception handling.
5  *
6  * Derived from code with the following copyrights:
7  * Copyright (C) 1994 - 1999 by Ralf Baechle
8  * Modified for R3000 by Paul M. Antoine, 1995, 1996
9  * Complete output from die() by Ulf Carlsson, 1998
10  * Copyright (C) 1999 Silicon Graphics, Inc.
11  *
12  * Essentially rewritten for the Xtensa architecture port.
13  *
14  * Copyright (C) 2001 - 2013 Tensilica Inc.
15  *
16  * Joe Taylor	<joe@tensilica.com, joetylr@yahoo.com>
17  * Chris Zankel	<chris@zankel.net>
18  * Marc Gauthier<marc@tensilica.com, marc@alumni.uwaterloo.ca>
19  * Kevin Chea
20  *
21  * This file is subject to the terms and conditions of the GNU General Public
22  * License.  See the file "COPYING" in the main directory of this archive
23  * for more details.
24  */
25 
26 #include <linux/kernel.h>
27 #include <linux/sched/signal.h>
28 #include <linux/sched/debug.h>
29 #include <linux/sched/task_stack.h>
30 #include <linux/init.h>
31 #include <linux/module.h>
32 #include <linux/stringify.h>
33 #include <linux/kallsyms.h>
34 #include <linux/delay.h>
35 #include <linux/hardirq.h>
36 #include <linux/ratelimit.h>
37 #include <linux/pgtable.h>
38 
39 #include <asm/stacktrace.h>
40 #include <asm/ptrace.h>
41 #include <asm/timex.h>
42 #include <linux/uaccess.h>
43 #include <asm/processor.h>
44 #include <asm/traps.h>
45 #include <asm/hw_breakpoint.h>
46 
47 /*
48  * Machine specific interrupt handlers
49  */
50 
51 static void do_illegal_instruction(struct pt_regs *regs);
52 static void do_div0(struct pt_regs *regs);
53 static void do_interrupt(struct pt_regs *regs);
54 #if XTENSA_FAKE_NMI
55 static void do_nmi(struct pt_regs *regs);
56 #endif
57 #if XCHAL_UNALIGNED_LOAD_EXCEPTION || XCHAL_UNALIGNED_STORE_EXCEPTION
58 static void do_unaligned_user(struct pt_regs *regs);
59 #endif
60 static void do_multihit(struct pt_regs *regs);
61 #if XTENSA_HAVE_COPROCESSORS
62 static void do_coprocessor(struct pt_regs *regs);
63 #endif
64 static void do_debug(struct pt_regs *regs);
65 
66 /*
67  * The vector table must be preceded by a save area (which
68  * implies it must be in RAM, unless one places RAM immediately
69  * before a ROM and puts the vector at the start of the ROM (!))
70  */
71 
72 #define KRNL		0x01
73 #define USER		0x02
74 
75 #define COPROCESSOR(x)							\
76 { EXCCAUSE_COPROCESSOR ## x ## _DISABLED, USER|KRNL, fast_coprocessor },\
77 { EXCCAUSE_COPROCESSOR ## x ## _DISABLED, 0, do_coprocessor }
78 
79 typedef struct {
80 	int cause;
81 	int fast;
82 	void* handler;
83 } dispatch_init_table_t;
84 
85 static dispatch_init_table_t __initdata dispatch_init_table[] = {
86 
87 #ifdef CONFIG_USER_ABI_CALL0_PROBE
88 { EXCCAUSE_ILLEGAL_INSTRUCTION,	USER,	   fast_illegal_instruction_user },
89 #endif
90 { EXCCAUSE_ILLEGAL_INSTRUCTION,	0,	   do_illegal_instruction},
91 { EXCCAUSE_SYSTEM_CALL,		USER,	   fast_syscall_user },
92 { EXCCAUSE_SYSTEM_CALL,		0,	   system_call },
93 /* EXCCAUSE_INSTRUCTION_FETCH unhandled */
94 /* EXCCAUSE_LOAD_STORE_ERROR unhandled*/
95 { EXCCAUSE_LEVEL1_INTERRUPT,	0,	   do_interrupt },
96 #ifdef SUPPORT_WINDOWED
97 { EXCCAUSE_ALLOCA,		USER|KRNL, fast_alloca },
98 #endif
99 { EXCCAUSE_INTEGER_DIVIDE_BY_ZERO, 0,	   do_div0 },
100 /* EXCCAUSE_PRIVILEGED unhandled */
101 #if XCHAL_UNALIGNED_LOAD_EXCEPTION || XCHAL_UNALIGNED_STORE_EXCEPTION
102 #ifdef CONFIG_XTENSA_UNALIGNED_USER
103 { EXCCAUSE_UNALIGNED,		USER,	   fast_unaligned },
104 #endif
105 { EXCCAUSE_UNALIGNED,		0,	   do_unaligned_user },
106 { EXCCAUSE_UNALIGNED,		KRNL,	   fast_unaligned },
107 #endif
108 #ifdef CONFIG_MMU
109 { EXCCAUSE_ITLB_MISS,			0,	   do_page_fault },
110 { EXCCAUSE_ITLB_MISS,			USER|KRNL, fast_second_level_miss},
111 { EXCCAUSE_DTLB_MISS,			USER|KRNL, fast_second_level_miss},
112 { EXCCAUSE_DTLB_MISS,			0,	   do_page_fault },
113 { EXCCAUSE_STORE_CACHE_ATTRIBUTE,	USER|KRNL, fast_store_prohibited },
114 #endif /* CONFIG_MMU */
115 #ifdef CONFIG_PFAULT
116 { EXCCAUSE_ITLB_MULTIHIT,		0,	   do_multihit },
117 { EXCCAUSE_ITLB_PRIVILEGE,		0,	   do_page_fault },
118 { EXCCAUSE_FETCH_CACHE_ATTRIBUTE,	0,	   do_page_fault },
119 { EXCCAUSE_DTLB_MULTIHIT,		0,	   do_multihit },
120 { EXCCAUSE_DTLB_PRIVILEGE,		0,	   do_page_fault },
121 { EXCCAUSE_STORE_CACHE_ATTRIBUTE,	0,	   do_page_fault },
122 { EXCCAUSE_LOAD_CACHE_ATTRIBUTE,	0,	   do_page_fault },
123 #endif
124 /* XCCHAL_EXCCAUSE_FLOATING_POINT unhandled */
125 #if XTENSA_HAVE_COPROCESSOR(0)
126 COPROCESSOR(0),
127 #endif
128 #if XTENSA_HAVE_COPROCESSOR(1)
129 COPROCESSOR(1),
130 #endif
131 #if XTENSA_HAVE_COPROCESSOR(2)
132 COPROCESSOR(2),
133 #endif
134 #if XTENSA_HAVE_COPROCESSOR(3)
135 COPROCESSOR(3),
136 #endif
137 #if XTENSA_HAVE_COPROCESSOR(4)
138 COPROCESSOR(4),
139 #endif
140 #if XTENSA_HAVE_COPROCESSOR(5)
141 COPROCESSOR(5),
142 #endif
143 #if XTENSA_HAVE_COPROCESSOR(6)
144 COPROCESSOR(6),
145 #endif
146 #if XTENSA_HAVE_COPROCESSOR(7)
147 COPROCESSOR(7),
148 #endif
149 #if XTENSA_FAKE_NMI
150 { EXCCAUSE_MAPPED_NMI,			0,		do_nmi },
151 #endif
152 { EXCCAUSE_MAPPED_DEBUG,		0,		do_debug },
153 { -1, -1, 0 }
154 
155 };
156 
157 /* The exception table <exc_table> serves two functions:
158  * 1. it contains three dispatch tables (fast_user, fast_kernel, default-c)
159  * 2. it is a temporary memory buffer for the exception handlers.
160  */
161 
162 DEFINE_PER_CPU(struct exc_table, exc_table);
163 DEFINE_PER_CPU(struct debug_table, debug_table);
164 
165 void die(const char*, struct pt_regs*, long);
166 
167 static inline void
168 __die_if_kernel(const char *str, struct pt_regs *regs, long err)
169 {
170 	if (!user_mode(regs))
171 		die(str, regs, err);
172 }
173 
174 /*
175  * Unhandled Exceptions. Kill user task or panic if in kernel space.
176  */
177 
178 void do_unhandled(struct pt_regs *regs)
179 {
180 	__die_if_kernel("Caught unhandled exception - should not happen",
181 			regs, SIGKILL);
182 
183 	/* If in user mode, send SIGILL signal to current process */
184 	pr_info_ratelimited("Caught unhandled exception in '%s' "
185 			    "(pid = %d, pc = %#010lx) - should not happen\n"
186 			    "\tEXCCAUSE is %ld\n",
187 			    current->comm, task_pid_nr(current), regs->pc,
188 			    regs->exccause);
189 	force_sig(SIGILL);
190 }
191 
192 /*
193  * Multi-hit exception. This if fatal!
194  */
195 
196 static void do_multihit(struct pt_regs *regs)
197 {
198 	die("Caught multihit exception", regs, SIGKILL);
199 }
200 
201 /*
202  * IRQ handler.
203  */
204 
205 #if XTENSA_FAKE_NMI
206 
207 #define IS_POW2(v) (((v) & ((v) - 1)) == 0)
208 
209 #if !(PROFILING_INTLEVEL == XCHAL_EXCM_LEVEL && \
210       IS_POW2(XTENSA_INTLEVEL_MASK(PROFILING_INTLEVEL)))
211 #warning "Fake NMI is requested for PMM, but there are other IRQs at or above its level."
212 #warning "Fake NMI will be used, but there will be a bugcheck if one of those IRQs fire."
213 
214 static inline void check_valid_nmi(void)
215 {
216 	unsigned intread = xtensa_get_sr(interrupt);
217 	unsigned intenable = xtensa_get_sr(intenable);
218 
219 	BUG_ON(intread & intenable &
220 	       ~(XTENSA_INTLEVEL_ANDBELOW_MASK(PROFILING_INTLEVEL) ^
221 		 XTENSA_INTLEVEL_MASK(PROFILING_INTLEVEL) ^
222 		 BIT(XCHAL_PROFILING_INTERRUPT)));
223 }
224 
225 #else
226 
227 static inline void check_valid_nmi(void)
228 {
229 }
230 
231 #endif
232 
233 irqreturn_t xtensa_pmu_irq_handler(int irq, void *dev_id);
234 
235 DEFINE_PER_CPU(unsigned long, nmi_count);
236 
237 static void do_nmi(struct pt_regs *regs)
238 {
239 	struct pt_regs *old_regs = set_irq_regs(regs);
240 
241 	nmi_enter();
242 	++*this_cpu_ptr(&nmi_count);
243 	check_valid_nmi();
244 	xtensa_pmu_irq_handler(0, NULL);
245 	nmi_exit();
246 	set_irq_regs(old_regs);
247 }
248 #endif
249 
250 static void do_interrupt(struct pt_regs *regs)
251 {
252 	static const unsigned int_level_mask[] = {
253 		0,
254 		XCHAL_INTLEVEL1_MASK,
255 		XCHAL_INTLEVEL2_MASK,
256 		XCHAL_INTLEVEL3_MASK,
257 		XCHAL_INTLEVEL4_MASK,
258 		XCHAL_INTLEVEL5_MASK,
259 		XCHAL_INTLEVEL6_MASK,
260 		XCHAL_INTLEVEL7_MASK,
261 	};
262 	struct pt_regs *old_regs = set_irq_regs(regs);
263 	unsigned unhandled = ~0u;
264 
265 	irq_enter();
266 
267 	for (;;) {
268 		unsigned intread = xtensa_get_sr(interrupt);
269 		unsigned intenable = xtensa_get_sr(intenable);
270 		unsigned int_at_level = intread & intenable;
271 		unsigned level;
272 
273 		for (level = LOCKLEVEL; level > 0; --level) {
274 			if (int_at_level & int_level_mask[level]) {
275 				int_at_level &= int_level_mask[level];
276 				if (int_at_level & unhandled)
277 					int_at_level &= unhandled;
278 				else
279 					unhandled |= int_level_mask[level];
280 				break;
281 			}
282 		}
283 
284 		if (level == 0)
285 			break;
286 
287 		/* clear lowest pending irq in the unhandled mask */
288 		unhandled ^= (int_at_level & -int_at_level);
289 		do_IRQ(__ffs(int_at_level), regs);
290 	}
291 
292 	irq_exit();
293 	set_irq_regs(old_regs);
294 }
295 
296 static bool check_div0(struct pt_regs *regs)
297 {
298 	static const u8 pattern[] = {'D', 'I', 'V', '0'};
299 	const u8 *p;
300 	u8 buf[5];
301 
302 	if (user_mode(regs)) {
303 		if (copy_from_user(buf, (void __user *)regs->pc + 2, 5))
304 			return false;
305 		p = buf;
306 	} else {
307 		p = (const u8 *)regs->pc + 2;
308 	}
309 
310 	return memcmp(p, pattern, sizeof(pattern)) == 0 ||
311 		memcmp(p + 1, pattern, sizeof(pattern)) == 0;
312 }
313 
314 /*
315  * Illegal instruction. Fatal if in kernel space.
316  */
317 
318 static void do_illegal_instruction(struct pt_regs *regs)
319 {
320 #ifdef CONFIG_USER_ABI_CALL0_PROBE
321 	/*
322 	 * When call0 application encounters an illegal instruction fast
323 	 * exception handler will attempt to set PS.WOE and retry failing
324 	 * instruction.
325 	 * If we get here we know that that instruction is also illegal
326 	 * with PS.WOE set, so it's not related to the windowed option
327 	 * hence PS.WOE may be cleared.
328 	 */
329 	if (regs->pc == current_thread_info()->ps_woe_fix_addr)
330 		regs->ps &= ~PS_WOE_MASK;
331 #endif
332 	if (check_div0(regs)) {
333 		do_div0(regs);
334 		return;
335 	}
336 
337 	__die_if_kernel("Illegal instruction in kernel", regs, SIGKILL);
338 
339 	/* If in user mode, send SIGILL signal to current process. */
340 
341 	pr_info_ratelimited("Illegal Instruction in '%s' (pid = %d, pc = %#010lx)\n",
342 			    current->comm, task_pid_nr(current), regs->pc);
343 	force_sig(SIGILL);
344 }
345 
346 static void do_div0(struct pt_regs *regs)
347 {
348 	__die_if_kernel("Unhandled division by 0 in kernel", regs, SIGKILL);
349 	force_sig_fault(SIGFPE, FPE_INTDIV, (void __user *)regs->pc);
350 }
351 
352 /*
353  * Handle unaligned memory accesses from user space. Kill task.
354  *
355  * If CONFIG_UNALIGNED_USER is not set, we don't allow unaligned memory
356  * accesses causes from user space.
357  */
358 
359 #if XCHAL_UNALIGNED_LOAD_EXCEPTION || XCHAL_UNALIGNED_STORE_EXCEPTION
360 static void do_unaligned_user(struct pt_regs *regs)
361 {
362 	__die_if_kernel("Unhandled unaligned exception in kernel",
363 			regs, SIGKILL);
364 
365 	pr_info_ratelimited("Unaligned memory access to %08lx in '%s' "
366 			    "(pid = %d, pc = %#010lx)\n",
367 			    regs->excvaddr, current->comm,
368 			    task_pid_nr(current), regs->pc);
369 	force_sig_fault(SIGBUS, BUS_ADRALN, (void *) regs->excvaddr);
370 }
371 #endif
372 
373 #if XTENSA_HAVE_COPROCESSORS
374 static void do_coprocessor(struct pt_regs *regs)
375 {
376 	coprocessor_flush_release_all(current_thread_info());
377 }
378 #endif
379 
380 /* Handle debug events.
381  * When CONFIG_HAVE_HW_BREAKPOINT is on this handler is called with
382  * preemption disabled to avoid rescheduling and keep mapping of hardware
383  * breakpoint structures to debug registers intact, so that
384  * DEBUGCAUSE.DBNUM could be used in case of data breakpoint hit.
385  */
386 static void do_debug(struct pt_regs *regs)
387 {
388 #ifdef CONFIG_HAVE_HW_BREAKPOINT
389 	int ret = check_hw_breakpoint(regs);
390 
391 	preempt_enable();
392 	if (ret == 0)
393 		return;
394 #endif
395 	__die_if_kernel("Breakpoint in kernel", regs, SIGKILL);
396 
397 	/* If in user mode, send SIGTRAP signal to current process */
398 
399 	force_sig(SIGTRAP);
400 }
401 
402 
403 #define set_handler(type, cause, handler)				\
404 	do {								\
405 		unsigned int cpu;					\
406 									\
407 		for_each_possible_cpu(cpu)				\
408 			per_cpu(exc_table, cpu).type[cause] = (handler);\
409 	} while (0)
410 
411 /* Set exception C handler - for temporary use when probing exceptions */
412 
413 xtensa_exception_handler *
414 __init trap_set_handler(int cause, xtensa_exception_handler *handler)
415 {
416 	void *previous = per_cpu(exc_table, 0).default_handler[cause];
417 
418 	set_handler(default_handler, cause, handler);
419 	return previous;
420 }
421 
422 
423 static void trap_init_excsave(void)
424 {
425 	xtensa_set_sr(this_cpu_ptr(&exc_table), excsave1);
426 }
427 
428 static void trap_init_debug(void)
429 {
430 	unsigned long debugsave = (unsigned long)this_cpu_ptr(&debug_table);
431 
432 	this_cpu_ptr(&debug_table)->debug_exception = debug_exception;
433 	__asm__ __volatile__("wsr %0, excsave" __stringify(XCHAL_DEBUGLEVEL)
434 			     :: "a"(debugsave));
435 }
436 
437 /*
438  * Initialize dispatch tables.
439  *
440  * The exception vectors are stored compressed the __init section in the
441  * dispatch_init_table. This function initializes the following three tables
442  * from that compressed table:
443  * - fast user		first dispatch table for user exceptions
444  * - fast kernel	first dispatch table for kernel exceptions
445  * - default C-handler	C-handler called by the default fast handler.
446  *
447  * See vectors.S for more details.
448  */
449 
450 void __init trap_init(void)
451 {
452 	int i;
453 
454 	/* Setup default vectors. */
455 
456 	for (i = 0; i < EXCCAUSE_N; i++) {
457 		set_handler(fast_user_handler, i, user_exception);
458 		set_handler(fast_kernel_handler, i, kernel_exception);
459 		set_handler(default_handler, i, do_unhandled);
460 	}
461 
462 	/* Setup specific handlers. */
463 
464 	for(i = 0; dispatch_init_table[i].cause >= 0; i++) {
465 		int fast = dispatch_init_table[i].fast;
466 		int cause = dispatch_init_table[i].cause;
467 		void *handler = dispatch_init_table[i].handler;
468 
469 		if (fast == 0)
470 			set_handler(default_handler, cause, handler);
471 		if ((fast & USER) != 0)
472 			set_handler(fast_user_handler, cause, handler);
473 		if ((fast & KRNL) != 0)
474 			set_handler(fast_kernel_handler, cause, handler);
475 	}
476 
477 	/* Initialize EXCSAVE_1 to hold the address of the exception table. */
478 	trap_init_excsave();
479 	trap_init_debug();
480 }
481 
482 #ifdef CONFIG_SMP
483 void secondary_trap_init(void)
484 {
485 	trap_init_excsave();
486 	trap_init_debug();
487 }
488 #endif
489 
490 /*
491  * This function dumps the current valid window frame and other base registers.
492  */
493 
494 void show_regs(struct pt_regs * regs)
495 {
496 	int i;
497 
498 	show_regs_print_info(KERN_DEFAULT);
499 
500 	for (i = 0; i < 16; i++) {
501 		if ((i % 8) == 0)
502 			pr_info("a%02d:", i);
503 		pr_cont(" %08lx", regs->areg[i]);
504 	}
505 	pr_cont("\n");
506 	pr_info("pc: %08lx, ps: %08lx, depc: %08lx, excvaddr: %08lx\n",
507 		regs->pc, regs->ps, regs->depc, regs->excvaddr);
508 	pr_info("lbeg: %08lx, lend: %08lx lcount: %08lx, sar: %08lx\n",
509 		regs->lbeg, regs->lend, regs->lcount, regs->sar);
510 	if (user_mode(regs))
511 		pr_cont("wb: %08lx, ws: %08lx, wmask: %08lx, syscall: %ld\n",
512 			regs->windowbase, regs->windowstart, regs->wmask,
513 			regs->syscall);
514 }
515 
516 static int show_trace_cb(struct stackframe *frame, void *data)
517 {
518 	const char *loglvl = data;
519 
520 	if (kernel_text_address(frame->pc))
521 		printk("%s [<%08lx>] %pB\n",
522 			loglvl, frame->pc, (void *)frame->pc);
523 	return 0;
524 }
525 
526 static void show_trace(struct task_struct *task, unsigned long *sp,
527 		       const char *loglvl)
528 {
529 	if (!sp)
530 		sp = stack_pointer(task);
531 
532 	printk("%sCall Trace:\n", loglvl);
533 	walk_stackframe(sp, show_trace_cb, (void *)loglvl);
534 }
535 
536 #define STACK_DUMP_ENTRY_SIZE 4
537 #define STACK_DUMP_LINE_SIZE 32
538 static size_t kstack_depth_to_print = CONFIG_PRINT_STACK_DEPTH;
539 
540 void show_stack(struct task_struct *task, unsigned long *sp, const char *loglvl)
541 {
542 	size_t len;
543 
544 	if (!sp)
545 		sp = stack_pointer(task);
546 
547 	len = min((-(size_t)sp) & (THREAD_SIZE - STACK_DUMP_ENTRY_SIZE),
548 		  kstack_depth_to_print * STACK_DUMP_ENTRY_SIZE);
549 
550 	printk("%sStack:\n", loglvl);
551 	print_hex_dump(loglvl, " ", DUMP_PREFIX_NONE,
552 		       STACK_DUMP_LINE_SIZE, STACK_DUMP_ENTRY_SIZE,
553 		       sp, len, false);
554 	show_trace(task, sp, loglvl);
555 }
556 
557 DEFINE_SPINLOCK(die_lock);
558 
559 void __noreturn die(const char * str, struct pt_regs * regs, long err)
560 {
561 	static int die_counter;
562 	const char *pr = "";
563 
564 	if (IS_ENABLED(CONFIG_PREEMPTION))
565 		pr = IS_ENABLED(CONFIG_PREEMPT_RT) ? " PREEMPT_RT" : " PREEMPT";
566 
567 	console_verbose();
568 	spin_lock_irq(&die_lock);
569 
570 	pr_info("%s: sig: %ld [#%d]%s\n", str, err, ++die_counter, pr);
571 	show_regs(regs);
572 	if (!user_mode(regs))
573 		show_stack(NULL, (unsigned long *)regs->areg[1], KERN_INFO);
574 
575 	add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE);
576 	spin_unlock_irq(&die_lock);
577 
578 	if (in_interrupt())
579 		panic("Fatal exception in interrupt");
580 
581 	if (panic_on_oops)
582 		panic("Fatal exception");
583 
584 	make_task_dead(err);
585 }
586