1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _LINUX_TRACE_RECURSION_H 3 #define _LINUX_TRACE_RECURSION_H 4 5 #include <linux/interrupt.h> 6 #include <linux/sched.h> 7 8 #ifdef CONFIG_TRACING 9 10 /* Only current can touch trace_recursion */ 11 12 /* 13 * For function tracing recursion: 14 * The order of these bits are important. 15 * 16 * When function tracing occurs, the following steps are made: 17 * If arch does not support a ftrace feature: 18 * call internal function (uses INTERNAL bits) which calls... 19 * The function callback, which can use the FTRACE bits to 20 * check for recursion. 21 */ 22 enum { 23 /* Function recursion bits */ 24 TRACE_FTRACE_BIT, 25 TRACE_FTRACE_NMI_BIT, 26 TRACE_FTRACE_IRQ_BIT, 27 TRACE_FTRACE_SIRQ_BIT, 28 TRACE_FTRACE_TRANSITION_BIT, 29 30 /* Internal use recursion bits */ 31 TRACE_INTERNAL_BIT, 32 TRACE_INTERNAL_NMI_BIT, 33 TRACE_INTERNAL_IRQ_BIT, 34 TRACE_INTERNAL_SIRQ_BIT, 35 TRACE_INTERNAL_TRANSITION_BIT, 36 37 TRACE_BRANCH_BIT, 38 /* 39 * Abuse of the trace_recursion. 40 * As we need a way to maintain state if we are tracing the function 41 * graph in irq because we want to trace a particular function that 42 * was called in irq context but we have irq tracing off. Since this 43 * can only be modified by current, we can reuse trace_recursion. 44 */ 45 TRACE_IRQ_BIT, 46 47 /* Set if the function is in the set_graph_function file */ 48 TRACE_GRAPH_BIT, 49 50 /* 51 * In the very unlikely case that an interrupt came in 52 * at a start of graph tracing, and we want to trace 53 * the function in that interrupt, the depth can be greater 54 * than zero, because of the preempted start of a previous 55 * trace. In an even more unlikely case, depth could be 2 56 * if a softirq interrupted the start of graph tracing, 57 * followed by an interrupt preempting a start of graph 58 * tracing in the softirq, and depth can even be 3 59 * if an NMI came in at the start of an interrupt function 60 * that preempted a softirq start of a function that 61 * preempted normal context!!!! Luckily, it can't be 62 * greater than 3, so the next two bits are a mask 63 * of what the depth is when we set TRACE_GRAPH_BIT 64 */ 65 66 TRACE_GRAPH_DEPTH_START_BIT, 67 TRACE_GRAPH_DEPTH_END_BIT, 68 69 /* 70 * To implement set_graph_notrace, if this bit is set, we ignore 71 * function graph tracing of called functions, until the return 72 * function is called to clear it. 73 */ 74 TRACE_GRAPH_NOTRACE_BIT, 75 76 /* Used to prevent recursion recording from recursing. */ 77 TRACE_RECORD_RECURSION_BIT, 78 }; 79 80 #define trace_recursion_set(bit) do { (current)->trace_recursion |= (1<<(bit)); } while (0) 81 #define trace_recursion_clear(bit) do { (current)->trace_recursion &= ~(1<<(bit)); } while (0) 82 #define trace_recursion_test(bit) ((current)->trace_recursion & (1<<(bit))) 83 84 #define trace_recursion_depth() \ 85 (((current)->trace_recursion >> TRACE_GRAPH_DEPTH_START_BIT) & 3) 86 #define trace_recursion_set_depth(depth) \ 87 do { \ 88 current->trace_recursion &= \ 89 ~(3 << TRACE_GRAPH_DEPTH_START_BIT); \ 90 current->trace_recursion |= \ 91 ((depth) & 3) << TRACE_GRAPH_DEPTH_START_BIT; \ 92 } while (0) 93 94 #define TRACE_CONTEXT_BITS 4 95 96 #define TRACE_FTRACE_START TRACE_FTRACE_BIT 97 98 #define TRACE_LIST_START TRACE_INTERNAL_BIT 99 100 #define TRACE_CONTEXT_MASK ((1 << (TRACE_LIST_START + TRACE_CONTEXT_BITS)) - 1) 101 102 /* 103 * Used for setting context 104 * NMI = 0 105 * IRQ = 1 106 * SOFTIRQ = 2 107 * NORMAL = 3 108 */ 109 enum { 110 TRACE_CTX_NMI, 111 TRACE_CTX_IRQ, 112 TRACE_CTX_SOFTIRQ, 113 TRACE_CTX_NORMAL, 114 TRACE_CTX_TRANSITION, 115 }; 116 117 static __always_inline int trace_get_context_bit(void) 118 { 119 unsigned char bit = interrupt_context_level(); 120 121 return TRACE_CTX_NORMAL - bit; 122 } 123 124 #ifdef CONFIG_FTRACE_RECORD_RECURSION 125 extern void ftrace_record_recursion(unsigned long ip, unsigned long parent_ip); 126 # define do_ftrace_record_recursion(ip, pip) \ 127 do { \ 128 if (!trace_recursion_test(TRACE_RECORD_RECURSION_BIT)) { \ 129 trace_recursion_set(TRACE_RECORD_RECURSION_BIT); \ 130 ftrace_record_recursion(ip, pip); \ 131 trace_recursion_clear(TRACE_RECORD_RECURSION_BIT); \ 132 } \ 133 } while (0) 134 #else 135 # define do_ftrace_record_recursion(ip, pip) do { } while (0) 136 #endif 137 138 /* 139 * Preemption is promised to be disabled when return bit >= 0. 140 */ 141 static __always_inline int trace_test_and_set_recursion(unsigned long ip, unsigned long pip, 142 int start) 143 { 144 unsigned int val = READ_ONCE(current->trace_recursion); 145 int bit; 146 147 bit = trace_get_context_bit() + start; 148 if (unlikely(val & (1 << bit))) { 149 /* 150 * If an interrupt occurs during a trace, and another trace 151 * happens in that interrupt but before the preempt_count is 152 * updated to reflect the new interrupt context, then this 153 * will think a recursion occurred, and the event will be dropped. 154 * Let a single instance happen via the TRANSITION_BIT to 155 * not drop those events. 156 */ 157 bit = TRACE_CTX_TRANSITION + start; 158 if (val & (1 << bit)) { 159 do_ftrace_record_recursion(ip, pip); 160 return -1; 161 } 162 } 163 164 val |= 1 << bit; 165 current->trace_recursion = val; 166 barrier(); 167 168 preempt_disable_notrace(); 169 170 return bit; 171 } 172 173 /* 174 * Preemption will be enabled (if it was previously enabled). 175 */ 176 static __always_inline void trace_clear_recursion(int bit) 177 { 178 preempt_enable_notrace(); 179 barrier(); 180 trace_recursion_clear(bit); 181 } 182 183 /** 184 * ftrace_test_recursion_trylock - tests for recursion in same context 185 * 186 * Use this for ftrace callbacks. This will detect if the function 187 * tracing recursed in the same context (normal vs interrupt), 188 * 189 * Returns: -1 if a recursion happened. 190 * >= 0 if no recursion. 191 */ 192 static __always_inline int ftrace_test_recursion_trylock(unsigned long ip, 193 unsigned long parent_ip) 194 { 195 return trace_test_and_set_recursion(ip, parent_ip, TRACE_FTRACE_START); 196 } 197 198 /** 199 * ftrace_test_recursion_unlock - called when function callback is complete 200 * @bit: The return of a successful ftrace_test_recursion_trylock() 201 * 202 * This is used at the end of a ftrace callback. 203 */ 204 static __always_inline void ftrace_test_recursion_unlock(int bit) 205 { 206 trace_clear_recursion(bit); 207 } 208 209 #endif /* CONFIG_TRACING */ 210 #endif /* _LINUX_TRACE_RECURSION_H */ 211