1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * handle transition of Linux booting another kernel 4 * Copyright (C) 2002-2005 Eric Biederman <ebiederm@xmission.com> 5 */ 6 7 #include <linux/mm.h> 8 #include <linux/kexec.h> 9 #include <linux/delay.h> 10 #include <linux/numa.h> 11 #include <linux/ftrace.h> 12 #include <linux/suspend.h> 13 #include <linux/gfp.h> 14 #include <linux/io.h> 15 16 #include <asm/pgtable.h> 17 #include <asm/pgalloc.h> 18 #include <asm/tlbflush.h> 19 #include <asm/mmu_context.h> 20 #include <asm/apic.h> 21 #include <asm/io_apic.h> 22 #include <asm/cpufeature.h> 23 #include <asm/desc.h> 24 #include <asm/set_memory.h> 25 #include <asm/debugreg.h> 26 27 static void set_gdt(void *newgdt, __u16 limit) 28 { 29 struct desc_ptr curgdt; 30 31 /* ia32 supports unaligned loads & stores */ 32 curgdt.size = limit; 33 curgdt.address = (unsigned long)newgdt; 34 35 load_gdt(&curgdt); 36 } 37 38 static void load_segments(void) 39 { 40 #define __STR(X) #X 41 #define STR(X) __STR(X) 42 43 __asm__ __volatile__ ( 44 "\tljmp $"STR(__KERNEL_CS)",$1f\n" 45 "\t1:\n" 46 "\tmovl $"STR(__KERNEL_DS)",%%eax\n" 47 "\tmovl %%eax,%%ds\n" 48 "\tmovl %%eax,%%es\n" 49 "\tmovl %%eax,%%ss\n" 50 : : : "eax", "memory"); 51 #undef STR 52 #undef __STR 53 } 54 55 static void machine_kexec_free_page_tables(struct kimage *image) 56 { 57 free_pages((unsigned long)image->arch.pgd, PGD_ALLOCATION_ORDER); 58 image->arch.pgd = NULL; 59 #ifdef CONFIG_X86_PAE 60 free_page((unsigned long)image->arch.pmd0); 61 image->arch.pmd0 = NULL; 62 free_page((unsigned long)image->arch.pmd1); 63 image->arch.pmd1 = NULL; 64 #endif 65 free_page((unsigned long)image->arch.pte0); 66 image->arch.pte0 = NULL; 67 free_page((unsigned long)image->arch.pte1); 68 image->arch.pte1 = NULL; 69 } 70 71 static int machine_kexec_alloc_page_tables(struct kimage *image) 72 { 73 image->arch.pgd = (pgd_t *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 74 PGD_ALLOCATION_ORDER); 75 #ifdef CONFIG_X86_PAE 76 image->arch.pmd0 = (pmd_t *)get_zeroed_page(GFP_KERNEL); 77 image->arch.pmd1 = (pmd_t *)get_zeroed_page(GFP_KERNEL); 78 #endif 79 image->arch.pte0 = (pte_t *)get_zeroed_page(GFP_KERNEL); 80 image->arch.pte1 = (pte_t *)get_zeroed_page(GFP_KERNEL); 81 if (!image->arch.pgd || 82 #ifdef CONFIG_X86_PAE 83 !image->arch.pmd0 || !image->arch.pmd1 || 84 #endif 85 !image->arch.pte0 || !image->arch.pte1) { 86 return -ENOMEM; 87 } 88 return 0; 89 } 90 91 static void machine_kexec_page_table_set_one( 92 pgd_t *pgd, pmd_t *pmd, pte_t *pte, 93 unsigned long vaddr, unsigned long paddr) 94 { 95 p4d_t *p4d; 96 pud_t *pud; 97 98 pgd += pgd_index(vaddr); 99 #ifdef CONFIG_X86_PAE 100 if (!(pgd_val(*pgd) & _PAGE_PRESENT)) 101 set_pgd(pgd, __pgd(__pa(pmd) | _PAGE_PRESENT)); 102 #endif 103 p4d = p4d_offset(pgd, vaddr); 104 pud = pud_offset(p4d, vaddr); 105 pmd = pmd_offset(pud, vaddr); 106 if (!(pmd_val(*pmd) & _PAGE_PRESENT)) 107 set_pmd(pmd, __pmd(__pa(pte) | _PAGE_TABLE)); 108 pte = pte_offset_kernel(pmd, vaddr); 109 set_pte(pte, pfn_pte(paddr >> PAGE_SHIFT, PAGE_KERNEL_EXEC)); 110 } 111 112 static void machine_kexec_prepare_page_tables(struct kimage *image) 113 { 114 void *control_page; 115 pmd_t *pmd = NULL; 116 117 control_page = page_address(image->control_code_page); 118 #ifdef CONFIG_X86_PAE 119 pmd = image->arch.pmd0; 120 #endif 121 machine_kexec_page_table_set_one( 122 image->arch.pgd, pmd, image->arch.pte0, 123 (unsigned long)control_page, __pa(control_page)); 124 #ifdef CONFIG_X86_PAE 125 pmd = image->arch.pmd1; 126 #endif 127 machine_kexec_page_table_set_one( 128 image->arch.pgd, pmd, image->arch.pte1, 129 __pa(control_page), __pa(control_page)); 130 } 131 132 /* 133 * A architecture hook called to validate the 134 * proposed image and prepare the control pages 135 * as needed. The pages for KEXEC_CONTROL_PAGE_SIZE 136 * have been allocated, but the segments have yet 137 * been copied into the kernel. 138 * 139 * Do what every setup is needed on image and the 140 * reboot code buffer to allow us to avoid allocations 141 * later. 142 * 143 * - Make control page executable. 144 * - Allocate page tables 145 * - Setup page tables 146 */ 147 int machine_kexec_prepare(struct kimage *image) 148 { 149 int error; 150 151 set_memory_x((unsigned long)page_address(image->control_code_page), 1); 152 error = machine_kexec_alloc_page_tables(image); 153 if (error) 154 return error; 155 machine_kexec_prepare_page_tables(image); 156 return 0; 157 } 158 159 /* 160 * Undo anything leftover by machine_kexec_prepare 161 * when an image is freed. 162 */ 163 void machine_kexec_cleanup(struct kimage *image) 164 { 165 set_memory_nx((unsigned long)page_address(image->control_code_page), 1); 166 machine_kexec_free_page_tables(image); 167 } 168 169 /* 170 * Do not allocate memory (or fail in any way) in machine_kexec(). 171 * We are past the point of no return, committed to rebooting now. 172 */ 173 void machine_kexec(struct kimage *image) 174 { 175 unsigned long page_list[PAGES_NR]; 176 void *control_page; 177 int save_ftrace_enabled; 178 asmlinkage unsigned long 179 (*relocate_kernel_ptr)(unsigned long indirection_page, 180 unsigned long control_page, 181 unsigned long start_address, 182 unsigned int has_pae, 183 unsigned int preserve_context); 184 185 #ifdef CONFIG_KEXEC_JUMP 186 if (image->preserve_context) 187 save_processor_state(); 188 #endif 189 190 save_ftrace_enabled = __ftrace_enabled_save(); 191 192 /* Interrupts aren't acceptable while we reboot */ 193 local_irq_disable(); 194 hw_breakpoint_disable(); 195 196 if (image->preserve_context) { 197 #ifdef CONFIG_X86_IO_APIC 198 /* 199 * We need to put APICs in legacy mode so that we can 200 * get timer interrupts in second kernel. kexec/kdump 201 * paths already have calls to restore_boot_irq_mode() 202 * in one form or other. kexec jump path also need one. 203 */ 204 clear_IO_APIC(); 205 restore_boot_irq_mode(); 206 #endif 207 } 208 209 control_page = page_address(image->control_code_page); 210 memcpy(control_page, relocate_kernel, KEXEC_CONTROL_CODE_MAX_SIZE); 211 212 relocate_kernel_ptr = control_page; 213 page_list[PA_CONTROL_PAGE] = __pa(control_page); 214 page_list[VA_CONTROL_PAGE] = (unsigned long)control_page; 215 page_list[PA_PGD] = __pa(image->arch.pgd); 216 217 if (image->type == KEXEC_TYPE_DEFAULT) 218 page_list[PA_SWAP_PAGE] = (page_to_pfn(image->swap_page) 219 << PAGE_SHIFT); 220 221 /* 222 * The segment registers are funny things, they have both a 223 * visible and an invisible part. Whenever the visible part is 224 * set to a specific selector, the invisible part is loaded 225 * with from a table in memory. At no other time is the 226 * descriptor table in memory accessed. 227 * 228 * I take advantage of this here by force loading the 229 * segments, before I zap the gdt with an invalid value. 230 */ 231 load_segments(); 232 /* 233 * The gdt & idt are now invalid. 234 * If you want to load them you must set up your own idt & gdt. 235 */ 236 idt_invalidate(phys_to_virt(0)); 237 set_gdt(phys_to_virt(0), 0); 238 239 /* now call it */ 240 image->start = relocate_kernel_ptr((unsigned long)image->head, 241 (unsigned long)page_list, 242 image->start, 243 boot_cpu_has(X86_FEATURE_PAE), 244 image->preserve_context); 245 246 #ifdef CONFIG_KEXEC_JUMP 247 if (image->preserve_context) 248 restore_processor_state(); 249 #endif 250 251 __ftrace_enabled_restore(save_ftrace_enabled); 252 } 253 254 void arch_crash_save_vmcoreinfo(void) 255 { 256 #ifdef CONFIG_NUMA 257 VMCOREINFO_SYMBOL(node_data); 258 VMCOREINFO_LENGTH(node_data, MAX_NUMNODES); 259 #endif 260 #ifdef CONFIG_X86_PAE 261 VMCOREINFO_CONFIG(X86_PAE); 262 #endif 263 } 264 265