1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * preemptoff and irqoff tracepoints 4 * 5 * Copyright (C) Joel Fernandes (Google) <joel@joelfernandes.org> 6 */ 7 8 #include <linux/kallsyms.h> 9 #include <linux/uaccess.h> 10 #include <linux/module.h> 11 #include <linux/ftrace.h> 12 #include <linux/kprobes.h> 13 #include "trace.h" 14 15 #define CREATE_TRACE_POINTS 16 #include <trace/events/preemptirq.h> 17 18 #ifdef CONFIG_TRACE_IRQFLAGS 19 /* Per-cpu variable to prevent redundant calls when IRQs already off */ 20 static DEFINE_PER_CPU(int, tracing_irq_cpu); 21 22 void trace_hardirqs_on(void) 23 { 24 if (this_cpu_read(tracing_irq_cpu)) { 25 if (!in_nmi()) 26 trace_irq_enable_rcuidle(CALLER_ADDR0, CALLER_ADDR1); 27 tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1); 28 this_cpu_write(tracing_irq_cpu, 0); 29 } 30 31 lockdep_hardirqs_on(CALLER_ADDR0); 32 } 33 EXPORT_SYMBOL(trace_hardirqs_on); 34 NOKPROBE_SYMBOL(trace_hardirqs_on); 35 36 void trace_hardirqs_off(void) 37 { 38 if (!this_cpu_read(tracing_irq_cpu)) { 39 this_cpu_write(tracing_irq_cpu, 1); 40 tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1); 41 if (!in_nmi()) 42 trace_irq_disable_rcuidle(CALLER_ADDR0, CALLER_ADDR1); 43 } 44 45 lockdep_hardirqs_off(CALLER_ADDR0); 46 } 47 EXPORT_SYMBOL(trace_hardirqs_off); 48 NOKPROBE_SYMBOL(trace_hardirqs_off); 49 50 __visible void trace_hardirqs_on_caller(unsigned long caller_addr) 51 { 52 if (this_cpu_read(tracing_irq_cpu)) { 53 if (!in_nmi()) 54 trace_irq_enable_rcuidle(CALLER_ADDR0, caller_addr); 55 tracer_hardirqs_on(CALLER_ADDR0, caller_addr); 56 this_cpu_write(tracing_irq_cpu, 0); 57 } 58 59 lockdep_hardirqs_on(CALLER_ADDR0); 60 } 61 EXPORT_SYMBOL(trace_hardirqs_on_caller); 62 NOKPROBE_SYMBOL(trace_hardirqs_on_caller); 63 64 __visible void trace_hardirqs_off_caller(unsigned long caller_addr) 65 { 66 if (!this_cpu_read(tracing_irq_cpu)) { 67 this_cpu_write(tracing_irq_cpu, 1); 68 tracer_hardirqs_off(CALLER_ADDR0, caller_addr); 69 if (!in_nmi()) 70 trace_irq_disable_rcuidle(CALLER_ADDR0, caller_addr); 71 } 72 73 lockdep_hardirqs_off(CALLER_ADDR0); 74 } 75 EXPORT_SYMBOL(trace_hardirqs_off_caller); 76 NOKPROBE_SYMBOL(trace_hardirqs_off_caller); 77 #endif /* CONFIG_TRACE_IRQFLAGS */ 78 79 #ifdef CONFIG_TRACE_PREEMPT_TOGGLE 80 81 void trace_preempt_on(unsigned long a0, unsigned long a1) 82 { 83 if (!in_nmi()) 84 trace_preempt_enable_rcuidle(a0, a1); 85 tracer_preempt_on(a0, a1); 86 } 87 88 void trace_preempt_off(unsigned long a0, unsigned long a1) 89 { 90 if (!in_nmi()) 91 trace_preempt_disable_rcuidle(a0, a1); 92 tracer_preempt_off(a0, a1); 93 } 94 #endif 95