11a59d1b8SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later
22d5bf28fSAmerigo Wang /* Kernel module help for x86.
32d5bf28fSAmerigo Wang Copyright (C) 2001 Rusty Russell.
42d5bf28fSAmerigo Wang
52d5bf28fSAmerigo Wang */
6c767a54bSJoe Perches
7c767a54bSJoe Perches #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
8c767a54bSJoe Perches
92d5bf28fSAmerigo Wang #include <linux/moduleloader.h>
102d5bf28fSAmerigo Wang #include <linux/elf.h>
112d5bf28fSAmerigo Wang #include <linux/vmalloc.h>
122d5bf28fSAmerigo Wang #include <linux/fs.h>
132d5bf28fSAmerigo Wang #include <linux/string.h>
142d5bf28fSAmerigo Wang #include <linux/kernel.h>
15bebf56a1SAndrey Ryabinin #include <linux/kasan.h>
162d5bf28fSAmerigo Wang #include <linux/bug.h>
172d5bf28fSAmerigo Wang #include <linux/mm.h>
185a0e3ad6STejun Heo #include <linux/gfp.h>
19d430d3d7SJason Baron #include <linux/jump_label.h>
20e2b32e67SKees Cook #include <linux/random.h>
215b384f93SJosh Poimboeuf #include <linux/memory.h>
222d5bf28fSAmerigo Wang
2335de5b06SAndy Lutomirski #include <asm/text-patching.h>
242d5bf28fSAmerigo Wang #include <asm/page.h>
2578cac48cSBorislav Petkov #include <asm/setup.h>
26ee9f8fceSJosh Poimboeuf #include <asm/unwind.h>
272d5bf28fSAmerigo Wang
282d5bf28fSAmerigo Wang #if 0
29c767a54bSJoe Perches #define DEBUGP(fmt, ...) \
30c767a54bSJoe Perches printk(KERN_DEBUG fmt, ##__VA_ARGS__)
312d5bf28fSAmerigo Wang #else
32c767a54bSJoe Perches #define DEBUGP(fmt, ...) \
33c767a54bSJoe Perches do { \
34c767a54bSJoe Perches if (0) \
35c767a54bSJoe Perches printk(KERN_DEBUG fmt, ##__VA_ARGS__); \
36c767a54bSJoe Perches } while (0)
372d5bf28fSAmerigo Wang #endif
382d5bf28fSAmerigo Wang
390fdc83b9SAmerigo Wang #ifdef CONFIG_X86_32
apply_relocate(Elf32_Shdr * sechdrs,const char * strtab,unsigned int symindex,unsigned int relsec,struct module * me)400fdc83b9SAmerigo Wang int apply_relocate(Elf32_Shdr *sechdrs,
410fdc83b9SAmerigo Wang const char *strtab,
420fdc83b9SAmerigo Wang unsigned int symindex,
430fdc83b9SAmerigo Wang unsigned int relsec,
440fdc83b9SAmerigo Wang struct module *me)
450fdc83b9SAmerigo Wang {
460fdc83b9SAmerigo Wang unsigned int i;
470fdc83b9SAmerigo Wang Elf32_Rel *rel = (void *)sechdrs[relsec].sh_addr;
480fdc83b9SAmerigo Wang Elf32_Sym *sym;
490fdc83b9SAmerigo Wang uint32_t *location;
500fdc83b9SAmerigo Wang
51c767a54bSJoe Perches DEBUGP("Applying relocate section %u to %u\n",
52c767a54bSJoe Perches relsec, sechdrs[relsec].sh_info);
530fdc83b9SAmerigo Wang for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) {
540fdc83b9SAmerigo Wang /* This is where to make the change */
550fdc83b9SAmerigo Wang location = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr
560fdc83b9SAmerigo Wang + rel[i].r_offset;
570fdc83b9SAmerigo Wang /* This is the symbol it is referring to. Note that all
580fdc83b9SAmerigo Wang undefined symbols have been resolved. */
590fdc83b9SAmerigo Wang sym = (Elf32_Sym *)sechdrs[symindex].sh_addr
600fdc83b9SAmerigo Wang + ELF32_R_SYM(rel[i].r_info);
610fdc83b9SAmerigo Wang
620fdc83b9SAmerigo Wang switch (ELF32_R_TYPE(rel[i].r_info)) {
630fdc83b9SAmerigo Wang case R_386_32:
640fdc83b9SAmerigo Wang /* We add the value into the location given */
650fdc83b9SAmerigo Wang *location += sym->st_value;
660fdc83b9SAmerigo Wang break;
670fdc83b9SAmerigo Wang case R_386_PC32:
68bb73d071SFangrui Song case R_386_PLT32:
692e76c283SGeert Uytterhoeven /* Add the value, subtract its position */
700fdc83b9SAmerigo Wang *location += sym->st_value - (uint32_t)location;
710fdc83b9SAmerigo Wang break;
720fdc83b9SAmerigo Wang default:
73c767a54bSJoe Perches pr_err("%s: Unknown relocation: %u\n",
740fdc83b9SAmerigo Wang me->name, ELF32_R_TYPE(rel[i].r_info));
750fdc83b9SAmerigo Wang return -ENOEXEC;
760fdc83b9SAmerigo Wang }
770fdc83b9SAmerigo Wang }
780fdc83b9SAmerigo Wang return 0;
790fdc83b9SAmerigo Wang }
800fdc83b9SAmerigo Wang #else /*X86_64*/
__write_relocate_add(Elf64_Shdr * sechdrs,const char * strtab,unsigned int symindex,unsigned int relsec,struct module * me,void * (* write)(void * dest,const void * src,size_t len),bool apply)810c05e7bdSSong Liu static int __write_relocate_add(Elf64_Shdr *sechdrs,
820fdc83b9SAmerigo Wang const char *strtab,
830fdc83b9SAmerigo Wang unsigned int symindex,
840fdc83b9SAmerigo Wang unsigned int relsec,
8588fc078aSPeter Zijlstra struct module *me,
860c05e7bdSSong Liu void *(*write)(void *dest, const void *src, size_t len),
870c05e7bdSSong Liu bool apply)
880fdc83b9SAmerigo Wang {
890fdc83b9SAmerigo Wang unsigned int i;
900fdc83b9SAmerigo Wang Elf64_Rela *rel = (void *)sechdrs[relsec].sh_addr;
910fdc83b9SAmerigo Wang Elf64_Sym *sym;
920fdc83b9SAmerigo Wang void *loc;
930fdc83b9SAmerigo Wang u64 val;
940c05e7bdSSong Liu u64 zero = 0ULL;
950fdc83b9SAmerigo Wang
960c05e7bdSSong Liu DEBUGP("%s relocate section %u to %u\n",
970c05e7bdSSong Liu apply ? "Applying" : "Clearing",
98c767a54bSJoe Perches relsec, sechdrs[relsec].sh_info);
990fdc83b9SAmerigo Wang for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) {
1000c05e7bdSSong Liu size_t size;
1010c05e7bdSSong Liu
1020fdc83b9SAmerigo Wang /* This is where to make the change */
1030fdc83b9SAmerigo Wang loc = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr
1040fdc83b9SAmerigo Wang + rel[i].r_offset;
1050fdc83b9SAmerigo Wang
1060fdc83b9SAmerigo Wang /* This is the symbol it is referring to. Note that all
1070fdc83b9SAmerigo Wang undefined symbols have been resolved. */
1080fdc83b9SAmerigo Wang sym = (Elf64_Sym *)sechdrs[symindex].sh_addr
1090fdc83b9SAmerigo Wang + ELF64_R_SYM(rel[i].r_info);
1100fdc83b9SAmerigo Wang
1110fdc83b9SAmerigo Wang DEBUGP("type %d st_value %Lx r_addend %Lx loc %Lx\n",
1120fdc83b9SAmerigo Wang (int)ELF64_R_TYPE(rel[i].r_info),
1130fdc83b9SAmerigo Wang sym->st_value, rel[i].r_addend, (u64)loc);
1140fdc83b9SAmerigo Wang
1150fdc83b9SAmerigo Wang val = sym->st_value + rel[i].r_addend;
1160fdc83b9SAmerigo Wang
1170fdc83b9SAmerigo Wang switch (ELF64_R_TYPE(rel[i].r_info)) {
1180fdc83b9SAmerigo Wang case R_X86_64_NONE:
1190c05e7bdSSong Liu continue; /* nothing to write */
1200fdc83b9SAmerigo Wang case R_X86_64_64:
1210c05e7bdSSong Liu size = 8;
1220fdc83b9SAmerigo Wang break;
1230fdc83b9SAmerigo Wang case R_X86_64_32:
1240c05e7bdSSong Liu if (val != *(u32 *)&val)
1250fdc83b9SAmerigo Wang goto overflow;
1260c05e7bdSSong Liu size = 4;
1270fdc83b9SAmerigo Wang break;
1280fdc83b9SAmerigo Wang case R_X86_64_32S:
1290c05e7bdSSong Liu if ((s64)val != *(s32 *)&val)
1300fdc83b9SAmerigo Wang goto overflow;
1310c05e7bdSSong Liu size = 4;
1320fdc83b9SAmerigo Wang break;
1330fdc83b9SAmerigo Wang case R_X86_64_PC32:
134b21ebf2fSH.J. Lu case R_X86_64_PLT32:
1350fdc83b9SAmerigo Wang val -= (u64)loc;
1360c05e7bdSSong Liu size = 4;
1370fdc83b9SAmerigo Wang break;
138b40a142bSArd Biesheuvel case R_X86_64_PC64:
139b40a142bSArd Biesheuvel val -= (u64)loc;
1400c05e7bdSSong Liu size = 8;
141b40a142bSArd Biesheuvel break;
1420fdc83b9SAmerigo Wang default:
143c767a54bSJoe Perches pr_err("%s: Unknown rela relocation: %llu\n",
1440fdc83b9SAmerigo Wang me->name, ELF64_R_TYPE(rel[i].r_info));
1450fdc83b9SAmerigo Wang return -ENOEXEC;
1460fdc83b9SAmerigo Wang }
1470fdc83b9SAmerigo Wang
1480c05e7bdSSong Liu if (apply) {
1490c05e7bdSSong Liu if (memcmp(loc, &zero, size)) {
1500c05e7bdSSong Liu pr_err("x86/modules: Invalid relocation target, existing value is nonzero for type %d, loc %p, val %Lx\n",
151eda9cec4SJosh Poimboeuf (int)ELF64_R_TYPE(rel[i].r_info), loc, val);
152eda9cec4SJosh Poimboeuf return -ENOEXEC;
1530c05e7bdSSong Liu }
1540c05e7bdSSong Liu write(loc, &val, size);
1550c05e7bdSSong Liu } else {
1560c05e7bdSSong Liu if (memcmp(loc, &val, size)) {
1570c05e7bdSSong Liu pr_warn("x86/modules: Invalid relocation target, existing value does not match expected value for type %d, loc %p, val %Lx\n",
1580c05e7bdSSong Liu (int)ELF64_R_TYPE(rel[i].r_info), loc, val);
1590c05e7bdSSong Liu return -ENOEXEC;
1600c05e7bdSSong Liu }
1610c05e7bdSSong Liu write(loc, &zero, size);
1620c05e7bdSSong Liu }
1630c05e7bdSSong Liu }
1640c05e7bdSSong Liu return 0;
165eda9cec4SJosh Poimboeuf
1660fdc83b9SAmerigo Wang overflow:
167c767a54bSJoe Perches pr_err("overflow in relocation type %d val %Lx\n",
1680fdc83b9SAmerigo Wang (int)ELF64_R_TYPE(rel[i].r_info), val);
169c767a54bSJoe Perches pr_err("`%s' likely not compiled with -mcmodel=kernel\n",
1700fdc83b9SAmerigo Wang me->name);
1710fdc83b9SAmerigo Wang return -ENOEXEC;
1720fdc83b9SAmerigo Wang }
17388fc078aSPeter Zijlstra
write_relocate_add(Elf64_Shdr * sechdrs,const char * strtab,unsigned int symindex,unsigned int relsec,struct module * me,bool apply)1740c05e7bdSSong Liu static int write_relocate_add(Elf64_Shdr *sechdrs,
17588fc078aSPeter Zijlstra const char *strtab,
17688fc078aSPeter Zijlstra unsigned int symindex,
17788fc078aSPeter Zijlstra unsigned int relsec,
1780c05e7bdSSong Liu struct module *me,
1790c05e7bdSSong Liu bool apply)
18088fc078aSPeter Zijlstra {
18188fc078aSPeter Zijlstra int ret;
18288fc078aSPeter Zijlstra bool early = me->state == MODULE_STATE_UNFORMED;
18388fc078aSPeter Zijlstra void *(*write)(void *, const void *, size_t) = memcpy;
18488fc078aSPeter Zijlstra
1855b384f93SJosh Poimboeuf if (!early) {
18688fc078aSPeter Zijlstra write = text_poke;
1875b384f93SJosh Poimboeuf mutex_lock(&text_mutex);
1885b384f93SJosh Poimboeuf }
18988fc078aSPeter Zijlstra
1900c05e7bdSSong Liu ret = __write_relocate_add(sechdrs, strtab, symindex, relsec, me,
1910c05e7bdSSong Liu write, apply);
19288fc078aSPeter Zijlstra
1935b384f93SJosh Poimboeuf if (!early) {
19488fc078aSPeter Zijlstra text_poke_sync();
1955b384f93SJosh Poimboeuf mutex_unlock(&text_mutex);
1965b384f93SJosh Poimboeuf }
19788fc078aSPeter Zijlstra
19888fc078aSPeter Zijlstra return ret;
19988fc078aSPeter Zijlstra }
20088fc078aSPeter Zijlstra
apply_relocate_add(Elf64_Shdr * sechdrs,const char * strtab,unsigned int symindex,unsigned int relsec,struct module * me)2010c05e7bdSSong Liu int apply_relocate_add(Elf64_Shdr *sechdrs,
2020c05e7bdSSong Liu const char *strtab,
2030c05e7bdSSong Liu unsigned int symindex,
2040c05e7bdSSong Liu unsigned int relsec,
2050c05e7bdSSong Liu struct module *me)
2060c05e7bdSSong Liu {
2070c05e7bdSSong Liu return write_relocate_add(sechdrs, strtab, symindex, relsec, me, true);
2080c05e7bdSSong Liu }
2090c05e7bdSSong Liu
2100c05e7bdSSong Liu #ifdef CONFIG_LIVEPATCH
clear_relocate_add(Elf64_Shdr * sechdrs,const char * strtab,unsigned int symindex,unsigned int relsec,struct module * me)2110c05e7bdSSong Liu void clear_relocate_add(Elf64_Shdr *sechdrs,
2120c05e7bdSSong Liu const char *strtab,
2130c05e7bdSSong Liu unsigned int symindex,
2140c05e7bdSSong Liu unsigned int relsec,
2150c05e7bdSSong Liu struct module *me)
2160c05e7bdSSong Liu {
2170c05e7bdSSong Liu write_relocate_add(sechdrs, strtab, symindex, relsec, me, false);
2180c05e7bdSSong Liu }
2190c05e7bdSSong Liu #endif
2200c05e7bdSSong Liu
2210fdc83b9SAmerigo Wang #endif
2220fdc83b9SAmerigo Wang
module_finalize(const Elf_Ehdr * hdr,const Elf_Shdr * sechdrs,struct module * me)2232d5bf28fSAmerigo Wang int module_finalize(const Elf_Ehdr *hdr,
2242d5bf28fSAmerigo Wang const Elf_Shdr *sechdrs,
2252d5bf28fSAmerigo Wang struct module *me)
2262d5bf28fSAmerigo Wang {
227be84d8edSJulian Pidancet const Elf_Shdr *s, *alt = NULL, *locks = NULL,
228*60bc276bSJuergen Gross *orc = NULL, *orc_ip = NULL,
229eaf44c81SThomas Gleixner *retpolines = NULL, *returns = NULL, *ibt_endbr = NULL,
230931ab636SPeter Zijlstra *calls = NULL, *cfi = NULL;
2312d5bf28fSAmerigo Wang char *secstrings = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;
2322d5bf28fSAmerigo Wang
2332d5bf28fSAmerigo Wang for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) {
2342d5bf28fSAmerigo Wang if (!strcmp(".altinstructions", secstrings + s->sh_name))
2352d5bf28fSAmerigo Wang alt = s;
2362d5bf28fSAmerigo Wang if (!strcmp(".smp_locks", secstrings + s->sh_name))
2372d5bf28fSAmerigo Wang locks = s;
238ee9f8fceSJosh Poimboeuf if (!strcmp(".orc_unwind", secstrings + s->sh_name))
239ee9f8fceSJosh Poimboeuf orc = s;
240ee9f8fceSJosh Poimboeuf if (!strcmp(".orc_unwind_ip", secstrings + s->sh_name))
241ee9f8fceSJosh Poimboeuf orc_ip = s;
24275085009SPeter Zijlstra if (!strcmp(".retpoline_sites", secstrings + s->sh_name))
24375085009SPeter Zijlstra retpolines = s;
24415e67227SPeter Zijlstra if (!strcmp(".return_sites", secstrings + s->sh_name))
24515e67227SPeter Zijlstra returns = s;
246eaf44c81SThomas Gleixner if (!strcmp(".call_sites", secstrings + s->sh_name))
247eaf44c81SThomas Gleixner calls = s;
248931ab636SPeter Zijlstra if (!strcmp(".cfi_sites", secstrings + s->sh_name))
249931ab636SPeter Zijlstra cfi = s;
250ed53a0d9SPeter Zijlstra if (!strcmp(".ibt_endbr_seal", secstrings + s->sh_name))
251ed53a0d9SPeter Zijlstra ibt_endbr = s;
2522d5bf28fSAmerigo Wang }
2532d5bf28fSAmerigo Wang
254931ab636SPeter Zijlstra if (retpolines || cfi) {
255931ab636SPeter Zijlstra void *rseg = NULL, *cseg = NULL;
256931ab636SPeter Zijlstra unsigned int rsize = 0, csize = 0;
257931ab636SPeter Zijlstra
258931ab636SPeter Zijlstra if (retpolines) {
259931ab636SPeter Zijlstra rseg = (void *)retpolines->sh_addr;
260931ab636SPeter Zijlstra rsize = retpolines->sh_size;
261931ab636SPeter Zijlstra }
262931ab636SPeter Zijlstra
263931ab636SPeter Zijlstra if (cfi) {
264931ab636SPeter Zijlstra cseg = (void *)cfi->sh_addr;
265931ab636SPeter Zijlstra csize = cfi->sh_size;
266931ab636SPeter Zijlstra }
267931ab636SPeter Zijlstra
268931ab636SPeter Zijlstra apply_fineibt(rseg, rseg + rsize, cseg, cseg + csize);
269931ab636SPeter Zijlstra }
27075085009SPeter Zijlstra if (retpolines) {
27175085009SPeter Zijlstra void *rseg = (void *)retpolines->sh_addr;
27275085009SPeter Zijlstra apply_retpolines(rseg, rseg + retpolines->sh_size);
27375085009SPeter Zijlstra }
27415e67227SPeter Zijlstra if (returns) {
27515e67227SPeter Zijlstra void *rseg = (void *)returns->sh_addr;
27615e67227SPeter Zijlstra apply_returns(rseg, rseg + returns->sh_size);
27715e67227SPeter Zijlstra }
2782d5bf28fSAmerigo Wang if (alt) {
2792d5bf28fSAmerigo Wang /* patch .altinstructions */
2802d5bf28fSAmerigo Wang void *aseg = (void *)alt->sh_addr;
2812d5bf28fSAmerigo Wang apply_alternatives(aseg, aseg + alt->sh_size);
2822d5bf28fSAmerigo Wang }
283*60bc276bSJuergen Gross if (calls || alt) {
284eaf44c81SThomas Gleixner struct callthunk_sites cs = {};
285eaf44c81SThomas Gleixner
286eaf44c81SThomas Gleixner if (calls) {
287eaf44c81SThomas Gleixner cs.call_start = (void *)calls->sh_addr;
288eaf44c81SThomas Gleixner cs.call_end = (void *)calls->sh_addr + calls->sh_size;
289eaf44c81SThomas Gleixner }
290eaf44c81SThomas Gleixner
291*60bc276bSJuergen Gross if (alt) {
292*60bc276bSJuergen Gross cs.alt_start = (void *)alt->sh_addr;
293*60bc276bSJuergen Gross cs.alt_end = (void *)alt->sh_addr + alt->sh_size;
294eaf44c81SThomas Gleixner }
295eaf44c81SThomas Gleixner
296eaf44c81SThomas Gleixner callthunks_patch_module_calls(&cs, me);
297eaf44c81SThomas Gleixner }
298ed53a0d9SPeter Zijlstra if (ibt_endbr) {
299ed53a0d9SPeter Zijlstra void *iseg = (void *)ibt_endbr->sh_addr;
300be0fffa5SPeter Zijlstra apply_seal_endbr(iseg, iseg + ibt_endbr->sh_size);
301ed53a0d9SPeter Zijlstra }
302be84d8edSJulian Pidancet if (locks) {
3032d5bf28fSAmerigo Wang void *lseg = (void *)locks->sh_addr;
304ac3b4328SSong Liu void *text = me->mem[MOD_TEXT].base;
305ac3b4328SSong Liu void *text_end = text + me->mem[MOD_TEXT].size;
3062d5bf28fSAmerigo Wang alternatives_smp_module_add(me, me->name,
3072d5bf28fSAmerigo Wang lseg, lseg + locks->sh_size,
308be84d8edSJulian Pidancet text, text_end);
3092d5bf28fSAmerigo Wang }
3102d5bf28fSAmerigo Wang
311ee9f8fceSJosh Poimboeuf if (orc && orc_ip)
312ee9f8fceSJosh Poimboeuf unwind_module_init(me, (void *)orc_ip->sh_addr, orc_ip->sh_size,
313ee9f8fceSJosh Poimboeuf (void *)orc->sh_addr, orc->sh_size);
314ee9f8fceSJosh Poimboeuf
3155336377dSLinus Torvalds return 0;
3162d5bf28fSAmerigo Wang }
3172d5bf28fSAmerigo Wang
module_arch_cleanup(struct module * mod)3182d5bf28fSAmerigo Wang void module_arch_cleanup(struct module *mod)
3192d5bf28fSAmerigo Wang {
3202d5bf28fSAmerigo Wang alternatives_smp_module_del(mod);
3212d5bf28fSAmerigo Wang }
322