1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _LINUX_STATIC_CALL_H 3 #define _LINUX_STATIC_CALL_H 4 5 /* 6 * Static call support 7 * 8 * Static calls use code patching to hard-code function pointers into direct 9 * branch instructions. They give the flexibility of function pointers, but 10 * with improved performance. This is especially important for cases where 11 * retpolines would otherwise be used, as retpolines can significantly impact 12 * performance. 13 * 14 * 15 * API overview: 16 * 17 * DECLARE_STATIC_CALL(name, func); 18 * DEFINE_STATIC_CALL(name, func); 19 * DEFINE_STATIC_CALL_NULL(name, typename); 20 * DEFINE_STATIC_CALL_RET0(name, typename); 21 * 22 * __static_call_return0; 23 * 24 * static_call(name)(args...); 25 * static_call_cond(name)(args...); 26 * static_call_update(name, func); 27 * static_call_query(name); 28 * 29 * EXPORT_STATIC_CALL{,_TRAMP}{,_GPL}() 30 * 31 * Usage example: 32 * 33 * # Start with the following functions (with identical prototypes): 34 * int func_a(int arg1, int arg2); 35 * int func_b(int arg1, int arg2); 36 * 37 * # Define a 'my_name' reference, associated with func_a() by default 38 * DEFINE_STATIC_CALL(my_name, func_a); 39 * 40 * # Call func_a() 41 * static_call(my_name)(arg1, arg2); 42 * 43 * # Update 'my_name' to point to func_b() 44 * static_call_update(my_name, &func_b); 45 * 46 * # Call func_b() 47 * static_call(my_name)(arg1, arg2); 48 * 49 * 50 * Implementation details: 51 * 52 * This requires some arch-specific code (CONFIG_HAVE_STATIC_CALL). 53 * Otherwise basic indirect calls are used (with function pointers). 54 * 55 * Each static_call() site calls into a trampoline associated with the name. 56 * The trampoline has a direct branch to the default function. Updates to a 57 * name will modify the trampoline's branch destination. 58 * 59 * If the arch has CONFIG_HAVE_STATIC_CALL_INLINE, then the call sites 60 * themselves will be patched at runtime to call the functions directly, 61 * rather than calling through the trampoline. This requires objtool or a 62 * compiler plugin to detect all the static_call() sites and annotate them 63 * in the .static_call_sites section. 64 * 65 * 66 * Notes on NULL function pointers: 67 * 68 * Static_call()s support NULL functions, with many of the caveats that 69 * regular function pointers have. 70 * 71 * Clearly calling a NULL function pointer is 'BAD', so too for 72 * static_call()s (although when HAVE_STATIC_CALL it might not be immediately 73 * fatal). A NULL static_call can be the result of: 74 * 75 * DECLARE_STATIC_CALL_NULL(my_static_call, void (*)(int)); 76 * 77 * which is equivalent to declaring a NULL function pointer with just a 78 * typename: 79 * 80 * void (*my_func_ptr)(int arg1) = NULL; 81 * 82 * or using static_call_update() with a NULL function. In both cases the 83 * HAVE_STATIC_CALL implementation will patch the trampoline with a RET 84 * instruction, instead of an immediate tail-call JMP. HAVE_STATIC_CALL_INLINE 85 * architectures can patch the trampoline call to a NOP. 86 * 87 * In all cases, any argument evaluation is unconditional. Unlike a regular 88 * conditional function pointer call: 89 * 90 * if (my_func_ptr) 91 * my_func_ptr(arg1) 92 * 93 * where the argument evaludation also depends on the pointer value. 94 * 95 * When calling a static_call that can be NULL, use: 96 * 97 * static_call_cond(name)(arg1); 98 * 99 * which will include the required value tests to avoid NULL-pointer 100 * dereferences. 101 * 102 * To query which function is currently set to be called, use: 103 * 104 * func = static_call_query(name); 105 * 106 * 107 * DEFINE_STATIC_CALL_RET0 / __static_call_return0: 108 * 109 * Just like how DEFINE_STATIC_CALL_NULL() / static_call_cond() optimize the 110 * conditional void function call, DEFINE_STATIC_CALL_RET0 / 111 * __static_call_return0 optimize the do nothing return 0 function. 112 * 113 * This feature is strictly UB per the C standard (since it casts a function 114 * pointer to a different signature) and relies on the architecture ABI to 115 * make things work. In particular it relies on Caller Stack-cleanup and the 116 * whole return register being clobbered for short return values. All normal 117 * CDECL style ABIs conform. 118 * 119 * In particular the x86_64 implementation replaces the 5 byte CALL 120 * instruction at the callsite with a 5 byte clear of the RAX register, 121 * completely eliding any function call overhead. 122 * 123 * Notably argument setup is unconditional. 124 * 125 * 126 * EXPORT_STATIC_CALL() vs EXPORT_STATIC_CALL_TRAMP(): 127 * 128 * The difference is that the _TRAMP variant tries to only export the 129 * trampoline with the result that a module can use static_call{,_cond}() but 130 * not static_call_update(). 131 * 132 */ 133 134 #include <linux/types.h> 135 #include <linux/cpu.h> 136 #include <linux/static_call_types.h> 137 138 #ifdef CONFIG_HAVE_STATIC_CALL 139 #include <asm/static_call.h> 140 141 /* 142 * Either @site or @tramp can be NULL. 143 */ 144 extern void arch_static_call_transform(void *site, void *tramp, void *func, bool tail); 145 146 #define STATIC_CALL_TRAMP_ADDR(name) &STATIC_CALL_TRAMP(name) 147 148 #else 149 #define STATIC_CALL_TRAMP_ADDR(name) NULL 150 #endif 151 152 #define static_call_update(name, func) \ 153 ({ \ 154 typeof(&STATIC_CALL_TRAMP(name)) __F = (func); \ 155 __static_call_update(&STATIC_CALL_KEY(name), \ 156 STATIC_CALL_TRAMP_ADDR(name), __F); \ 157 }) 158 159 #define static_call_query(name) (READ_ONCE(STATIC_CALL_KEY(name).func)) 160 161 #ifdef CONFIG_HAVE_STATIC_CALL_INLINE 162 163 extern int __init static_call_init(void); 164 165 struct static_call_mod { 166 struct static_call_mod *next; 167 struct module *mod; /* for vmlinux, mod == NULL */ 168 struct static_call_site *sites; 169 }; 170 171 /* For finding the key associated with a trampoline */ 172 struct static_call_tramp_key { 173 s32 tramp; 174 s32 key; 175 }; 176 177 extern void __static_call_update(struct static_call_key *key, void *tramp, void *func); 178 extern int static_call_mod_init(struct module *mod); 179 extern int static_call_text_reserved(void *start, void *end); 180 181 extern long __static_call_return0(void); 182 183 #define DEFINE_STATIC_CALL(name, _func) \ 184 DECLARE_STATIC_CALL(name, _func); \ 185 struct static_call_key STATIC_CALL_KEY(name) = { \ 186 .func = _func, \ 187 .type = 1, \ 188 }; \ 189 ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func) 190 191 #define DEFINE_STATIC_CALL_NULL(name, _func) \ 192 DECLARE_STATIC_CALL(name, _func); \ 193 struct static_call_key STATIC_CALL_KEY(name) = { \ 194 .func = NULL, \ 195 .type = 1, \ 196 }; \ 197 ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name) 198 199 #define DEFINE_STATIC_CALL_RET0(name, _func) \ 200 DECLARE_STATIC_CALL(name, _func); \ 201 struct static_call_key STATIC_CALL_KEY(name) = { \ 202 .func = __static_call_return0, \ 203 .type = 1, \ 204 }; \ 205 ARCH_DEFINE_STATIC_CALL_RET0_TRAMP(name) 206 207 #define static_call_cond(name) (void)__static_call(name) 208 209 #define EXPORT_STATIC_CALL(name) \ 210 EXPORT_SYMBOL(STATIC_CALL_KEY(name)); \ 211 EXPORT_SYMBOL(STATIC_CALL_TRAMP(name)) 212 #define EXPORT_STATIC_CALL_GPL(name) \ 213 EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name)); \ 214 EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name)) 215 216 /* Leave the key unexported, so modules can't change static call targets: */ 217 #define EXPORT_STATIC_CALL_TRAMP(name) \ 218 EXPORT_SYMBOL(STATIC_CALL_TRAMP(name)); \ 219 ARCH_ADD_TRAMP_KEY(name) 220 #define EXPORT_STATIC_CALL_TRAMP_GPL(name) \ 221 EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name)); \ 222 ARCH_ADD_TRAMP_KEY(name) 223 224 #elif defined(CONFIG_HAVE_STATIC_CALL) 225 226 static inline int static_call_init(void) { return 0; } 227 228 #define DEFINE_STATIC_CALL(name, _func) \ 229 DECLARE_STATIC_CALL(name, _func); \ 230 struct static_call_key STATIC_CALL_KEY(name) = { \ 231 .func = _func, \ 232 }; \ 233 ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func) 234 235 #define DEFINE_STATIC_CALL_NULL(name, _func) \ 236 DECLARE_STATIC_CALL(name, _func); \ 237 struct static_call_key STATIC_CALL_KEY(name) = { \ 238 .func = NULL, \ 239 }; \ 240 ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name) 241 242 #define DEFINE_STATIC_CALL_RET0(name, _func) \ 243 DECLARE_STATIC_CALL(name, _func); \ 244 struct static_call_key STATIC_CALL_KEY(name) = { \ 245 .func = __static_call_return0, \ 246 }; \ 247 ARCH_DEFINE_STATIC_CALL_RET0_TRAMP(name) 248 249 #define static_call_cond(name) (void)__static_call(name) 250 251 static inline 252 void __static_call_update(struct static_call_key *key, void *tramp, void *func) 253 { 254 cpus_read_lock(); 255 WRITE_ONCE(key->func, func); 256 arch_static_call_transform(NULL, tramp, func, false); 257 cpus_read_unlock(); 258 } 259 260 static inline int static_call_text_reserved(void *start, void *end) 261 { 262 return 0; 263 } 264 265 extern long __static_call_return0(void); 266 267 #define EXPORT_STATIC_CALL(name) \ 268 EXPORT_SYMBOL(STATIC_CALL_KEY(name)); \ 269 EXPORT_SYMBOL(STATIC_CALL_TRAMP(name)) 270 #define EXPORT_STATIC_CALL_GPL(name) \ 271 EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name)); \ 272 EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name)) 273 274 /* Leave the key unexported, so modules can't change static call targets: */ 275 #define EXPORT_STATIC_CALL_TRAMP(name) \ 276 EXPORT_SYMBOL(STATIC_CALL_TRAMP(name)) 277 #define EXPORT_STATIC_CALL_TRAMP_GPL(name) \ 278 EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name)) 279 280 #else /* Generic implementation */ 281 282 static inline int static_call_init(void) { return 0; } 283 284 static inline long __static_call_return0(void) 285 { 286 return 0; 287 } 288 289 #define __DEFINE_STATIC_CALL(name, _func, _func_init) \ 290 DECLARE_STATIC_CALL(name, _func); \ 291 struct static_call_key STATIC_CALL_KEY(name) = { \ 292 .func = _func_init, \ 293 } 294 295 #define DEFINE_STATIC_CALL(name, _func) \ 296 __DEFINE_STATIC_CALL(name, _func, _func) 297 298 #define DEFINE_STATIC_CALL_NULL(name, _func) \ 299 __DEFINE_STATIC_CALL(name, _func, NULL) 300 301 #define DEFINE_STATIC_CALL_RET0(name, _func) \ 302 __DEFINE_STATIC_CALL(name, _func, __static_call_return0) 303 304 static inline void __static_call_nop(void) { } 305 306 /* 307 * This horrific hack takes care of two things: 308 * 309 * - it ensures the compiler will only load the function pointer ONCE, 310 * which avoids a reload race. 311 * 312 * - it ensures the argument evaluation is unconditional, similar 313 * to the HAVE_STATIC_CALL variant. 314 * 315 * Sadly current GCC/Clang (10 for both) do not optimize this properly 316 * and will emit an indirect call for the NULL case :-( 317 */ 318 #define __static_call_cond(name) \ 319 ({ \ 320 void *func = READ_ONCE(STATIC_CALL_KEY(name).func); \ 321 if (!func) \ 322 func = &__static_call_nop; \ 323 (typeof(STATIC_CALL_TRAMP(name))*)func; \ 324 }) 325 326 #define static_call_cond(name) (void)__static_call_cond(name) 327 328 static inline 329 void __static_call_update(struct static_call_key *key, void *tramp, void *func) 330 { 331 WRITE_ONCE(key->func, func); 332 } 333 334 static inline int static_call_text_reserved(void *start, void *end) 335 { 336 return 0; 337 } 338 339 #define EXPORT_STATIC_CALL(name) EXPORT_SYMBOL(STATIC_CALL_KEY(name)) 340 #define EXPORT_STATIC_CALL_GPL(name) EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name)) 341 342 #endif /* CONFIG_HAVE_STATIC_CALL */ 343 344 #endif /* _LINUX_STATIC_CALL_H */ 345