1 // SPDX-License-Identifier: GPL-2.0-only 2 3 #define pr_fmt(fmt) "callthunks: " fmt 4 5 #include <linux/debugfs.h> 6 #include <linux/kallsyms.h> 7 #include <linux/memory.h> 8 #include <linux/moduleloader.h> 9 #include <linux/static_call.h> 10 11 #include <asm/alternative.h> 12 #include <asm/asm-offsets.h> 13 #include <asm/cpu.h> 14 #include <asm/ftrace.h> 15 #include <asm/insn.h> 16 #include <asm/kexec.h> 17 #include <asm/nospec-branch.h> 18 #include <asm/paravirt.h> 19 #include <asm/sections.h> 20 #include <asm/switch_to.h> 21 #include <asm/sync_core.h> 22 #include <asm/text-patching.h> 23 #include <asm/xen/hypercall.h> 24 25 static int __initdata_or_module debug_callthunks; 26 27 #define prdbg(fmt, args...) \ 28 do { \ 29 if (debug_callthunks) \ 30 printk(KERN_DEBUG pr_fmt(fmt), ##args); \ 31 } while(0) 32 33 static int __init debug_thunks(char *str) 34 { 35 debug_callthunks = 1; 36 return 1; 37 } 38 __setup("debug-callthunks", debug_thunks); 39 40 #ifdef CONFIG_CALL_THUNKS_DEBUG 41 DEFINE_PER_CPU(u64, __x86_call_count); 42 DEFINE_PER_CPU(u64, __x86_ret_count); 43 DEFINE_PER_CPU(u64, __x86_stuffs_count); 44 DEFINE_PER_CPU(u64, __x86_ctxsw_count); 45 EXPORT_SYMBOL_GPL(__x86_ctxsw_count); 46 EXPORT_SYMBOL_GPL(__x86_call_count); 47 #endif 48 49 extern s32 __call_sites[], __call_sites_end[]; 50 51 struct core_text { 52 unsigned long base; 53 unsigned long end; 54 const char *name; 55 }; 56 57 static bool thunks_initialized __ro_after_init; 58 59 static const struct core_text builtin_coretext = { 60 .base = (unsigned long)_text, 61 .end = (unsigned long)_etext, 62 .name = "builtin", 63 }; 64 65 asm ( 66 ".pushsection .rodata \n" 67 ".global skl_call_thunk_template \n" 68 "skl_call_thunk_template: \n" 69 __stringify(INCREMENT_CALL_DEPTH)" \n" 70 ".global skl_call_thunk_tail \n" 71 "skl_call_thunk_tail: \n" 72 ".popsection \n" 73 ); 74 75 extern u8 skl_call_thunk_template[]; 76 extern u8 skl_call_thunk_tail[]; 77 78 #define SKL_TMPL_SIZE \ 79 ((unsigned int)(skl_call_thunk_tail - skl_call_thunk_template)) 80 81 extern void error_entry(void); 82 extern void xen_error_entry(void); 83 extern void paranoid_entry(void); 84 85 static inline bool within_coretext(const struct core_text *ct, void *addr) 86 { 87 unsigned long p = (unsigned long)addr; 88 89 return ct->base <= p && p < ct->end; 90 } 91 92 static inline bool within_module_coretext(void *addr) 93 { 94 bool ret = false; 95 96 #ifdef CONFIG_MODULES 97 struct module *mod; 98 99 preempt_disable(); 100 mod = __module_address((unsigned long)addr); 101 if (mod && within_module_core((unsigned long)addr, mod)) 102 ret = true; 103 preempt_enable(); 104 #endif 105 return ret; 106 } 107 108 static bool is_coretext(const struct core_text *ct, void *addr) 109 { 110 if (ct && within_coretext(ct, addr)) 111 return true; 112 if (within_coretext(&builtin_coretext, addr)) 113 return true; 114 return within_module_coretext(addr); 115 } 116 117 static bool skip_addr(void *dest) 118 { 119 if (dest == error_entry) 120 return true; 121 if (dest == paranoid_entry) 122 return true; 123 if (dest == xen_error_entry) 124 return true; 125 /* Does FILL_RSB... */ 126 if (dest == __switch_to_asm) 127 return true; 128 /* Accounts directly */ 129 if (dest == ret_from_fork) 130 return true; 131 #if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_AMD_MEM_ENCRYPT) 132 if (dest == soft_restart_cpu) 133 return true; 134 #endif 135 #ifdef CONFIG_FUNCTION_TRACER 136 if (dest == __fentry__) 137 return true; 138 #endif 139 #ifdef CONFIG_KEXEC_CORE 140 if (dest >= (void *)relocate_kernel && 141 dest < (void*)relocate_kernel + KEXEC_CONTROL_CODE_MAX_SIZE) 142 return true; 143 #endif 144 #ifdef CONFIG_XEN 145 if (dest >= (void *)hypercall_page && 146 dest < (void*)hypercall_page + PAGE_SIZE) 147 return true; 148 #endif 149 return false; 150 } 151 152 static __init_or_module void *call_get_dest(void *addr) 153 { 154 struct insn insn; 155 void *dest; 156 int ret; 157 158 ret = insn_decode_kernel(&insn, addr); 159 if (ret) 160 return ERR_PTR(ret); 161 162 /* Patched out call? */ 163 if (insn.opcode.bytes[0] != CALL_INSN_OPCODE) 164 return NULL; 165 166 dest = addr + insn.length + insn.immediate.value; 167 if (skip_addr(dest)) 168 return NULL; 169 return dest; 170 } 171 172 static const u8 nops[] = { 173 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 174 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 175 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 176 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 177 }; 178 179 static void *patch_dest(void *dest, bool direct) 180 { 181 unsigned int tsize = SKL_TMPL_SIZE; 182 u8 *pad = dest - tsize; 183 184 /* Already patched? */ 185 if (!bcmp(pad, skl_call_thunk_template, tsize)) 186 return pad; 187 188 /* Ensure there are nops */ 189 if (bcmp(pad, nops, tsize)) { 190 pr_warn_once("Invalid padding area for %pS\n", dest); 191 return NULL; 192 } 193 194 if (direct) 195 memcpy(pad, skl_call_thunk_template, tsize); 196 else 197 text_poke_copy_locked(pad, skl_call_thunk_template, tsize, true); 198 return pad; 199 } 200 201 static __init_or_module void patch_call(void *addr, const struct core_text *ct) 202 { 203 void *pad, *dest; 204 u8 bytes[8]; 205 206 if (!within_coretext(ct, addr)) 207 return; 208 209 dest = call_get_dest(addr); 210 if (!dest || WARN_ON_ONCE(IS_ERR(dest))) 211 return; 212 213 if (!is_coretext(ct, dest)) 214 return; 215 216 pad = patch_dest(dest, within_coretext(ct, dest)); 217 if (!pad) 218 return; 219 220 prdbg("Patch call at: %pS %px to %pS %px -> %px \n", addr, addr, 221 dest, dest, pad); 222 __text_gen_insn(bytes, CALL_INSN_OPCODE, addr, pad, CALL_INSN_SIZE); 223 text_poke_early(addr, bytes, CALL_INSN_SIZE); 224 } 225 226 static __init_or_module void 227 patch_call_sites(s32 *start, s32 *end, const struct core_text *ct) 228 { 229 s32 *s; 230 231 for (s = start; s < end; s++) 232 patch_call((void *)s + *s, ct); 233 } 234 235 static __init_or_module void 236 patch_alt_call_sites(struct alt_instr *start, struct alt_instr *end, 237 const struct core_text *ct) 238 { 239 struct alt_instr *a; 240 241 for (a = start; a < end; a++) 242 patch_call((void *)&a->instr_offset + a->instr_offset, ct); 243 } 244 245 static __init_or_module void 246 callthunks_setup(struct callthunk_sites *cs, const struct core_text *ct) 247 { 248 prdbg("Patching call sites %s\n", ct->name); 249 patch_call_sites(cs->call_start, cs->call_end, ct); 250 patch_alt_call_sites(cs->alt_start, cs->alt_end, ct); 251 prdbg("Patching call sites done%s\n", ct->name); 252 } 253 254 void __init callthunks_patch_builtin_calls(void) 255 { 256 struct callthunk_sites cs = { 257 .call_start = __call_sites, 258 .call_end = __call_sites_end, 259 .alt_start = __alt_instructions, 260 .alt_end = __alt_instructions_end 261 }; 262 263 if (!cpu_feature_enabled(X86_FEATURE_CALL_DEPTH)) 264 return; 265 266 pr_info("Setting up call depth tracking\n"); 267 mutex_lock(&text_mutex); 268 callthunks_setup(&cs, &builtin_coretext); 269 thunks_initialized = true; 270 mutex_unlock(&text_mutex); 271 } 272 273 void *callthunks_translate_call_dest(void *dest) 274 { 275 void *target; 276 277 lockdep_assert_held(&text_mutex); 278 279 if (!thunks_initialized || skip_addr(dest)) 280 return dest; 281 282 if (!is_coretext(NULL, dest)) 283 return dest; 284 285 target = patch_dest(dest, false); 286 return target ? : dest; 287 } 288 289 #ifdef CONFIG_BPF_JIT 290 static bool is_callthunk(void *addr) 291 { 292 unsigned int tmpl_size = SKL_TMPL_SIZE; 293 void *tmpl = skl_call_thunk_template; 294 unsigned long dest; 295 296 dest = roundup((unsigned long)addr, CONFIG_FUNCTION_ALIGNMENT); 297 if (!thunks_initialized || skip_addr((void *)dest)) 298 return false; 299 300 return !bcmp((void *)(dest - tmpl_size), tmpl, tmpl_size); 301 } 302 303 int x86_call_depth_emit_accounting(u8 **pprog, void *func) 304 { 305 unsigned int tmpl_size = SKL_TMPL_SIZE; 306 void *tmpl = skl_call_thunk_template; 307 308 if (!thunks_initialized) 309 return 0; 310 311 /* Is function call target a thunk? */ 312 if (func && is_callthunk(func)) 313 return 0; 314 315 memcpy(*pprog, tmpl, tmpl_size); 316 *pprog += tmpl_size; 317 return tmpl_size; 318 } 319 #endif 320 321 #ifdef CONFIG_MODULES 322 void noinline callthunks_patch_module_calls(struct callthunk_sites *cs, 323 struct module *mod) 324 { 325 struct core_text ct = { 326 .base = (unsigned long)mod->mem[MOD_TEXT].base, 327 .end = (unsigned long)mod->mem[MOD_TEXT].base + mod->mem[MOD_TEXT].size, 328 .name = mod->name, 329 }; 330 331 if (!thunks_initialized) 332 return; 333 334 mutex_lock(&text_mutex); 335 callthunks_setup(cs, &ct); 336 mutex_unlock(&text_mutex); 337 } 338 #endif /* CONFIG_MODULES */ 339 340 #if defined(CONFIG_CALL_THUNKS_DEBUG) && defined(CONFIG_DEBUG_FS) 341 static int callthunks_debug_show(struct seq_file *m, void *p) 342 { 343 unsigned long cpu = (unsigned long)m->private; 344 345 seq_printf(m, "C: %16llu R: %16llu S: %16llu X: %16llu\n,", 346 per_cpu(__x86_call_count, cpu), 347 per_cpu(__x86_ret_count, cpu), 348 per_cpu(__x86_stuffs_count, cpu), 349 per_cpu(__x86_ctxsw_count, cpu)); 350 return 0; 351 } 352 353 static int callthunks_debug_open(struct inode *inode, struct file *file) 354 { 355 return single_open(file, callthunks_debug_show, inode->i_private); 356 } 357 358 static const struct file_operations dfs_ops = { 359 .open = callthunks_debug_open, 360 .read = seq_read, 361 .llseek = seq_lseek, 362 .release = single_release, 363 }; 364 365 static int __init callthunks_debugfs_init(void) 366 { 367 struct dentry *dir; 368 unsigned long cpu; 369 370 dir = debugfs_create_dir("callthunks", NULL); 371 for_each_possible_cpu(cpu) { 372 void *arg = (void *)cpu; 373 char name [10]; 374 375 sprintf(name, "cpu%lu", cpu); 376 debugfs_create_file(name, 0644, dir, arg, &dfs_ops); 377 } 378 return 0; 379 } 380 __initcall(callthunks_debugfs_init); 381 #endif 382