1/*- 2 * Copyright (c) 2003 Peter Wemm 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27#include <machine/asmacros.h> /* miscellaneous asm macros */ 28#include <machine/specialreg.h> 29 30#include "assym.inc" 31 32 .data /* So we can modify it */ 33 34 .p2align 4,0 35 .globl mptramp_start 36mptramp_start: 37 .code16 38 /* 39 * The AP enters here in response to the startup IPI. 40 * We are in real mode. %cs is the only segment register set. 41 */ 42 cli /* make sure no interrupts */ 43 mov %cs, %ax /* copy %cs to %ds. Remember these */ 44 mov %ax, %ds /* are offsets rather than selectors */ 45 mov %ax, %ss 46 47 /* 48 * Find relocation base and patch the gdt descriptor and ljmp targets 49 */ 50 xorl %ebx,%ebx 51 mov %cs, %bx 52 sall $4, %ebx /* %ebx is now our relocation base */ 53 orl %ebx, lgdt_desc-mptramp_start+2 54 orl %ebx, jmp_32-mptramp_start+2 55 orl %ebx, jmp_64-mptramp_start+1 56 57 /* 58 * Load the descriptor table pointer. We'll need it when running 59 * in 16 bit protected mode. 60 */ 61 lgdt lgdt_desc-mptramp_start 62 63 /* Enable protected mode */ 64 movl $CR0_PE, %eax 65 mov %eax, %cr0 66 67 /* 68 * Now execute a far jump to turn on protected mode. This 69 * causes the segment registers to turn into selectors and causes 70 * %cs to be loaded from the gdt. 71 * 72 * The following instruction is: 73 * ljmpl $bootcode-gdt, $protmode-mptramp_start 74 * but gas cannot assemble that. And besides, we patch the targets 75 * in early startup and its a little clearer what we are patching. 76 */ 77jmp_32: 78 .byte 0x66 /* size override to 32 bits */ 79 .byte 0xea /* opcode for far jump */ 80 .long protmode-mptramp_start /* offset in segment */ 81 .word bootcode-gdt /* index in gdt for 32 bit code */ 82 83 /* 84 * At this point, we are running in 32 bit legacy protected mode. 85 */ 86 .code32 87protmode: 88 mov $bootdata-gdt, %eax 89 mov %ax, %ds 90 91 /* 92 * Turn on the PAE bit and optionally the LA57 bit for when paging 93 * is later enabled. 94 */ 95 mov %cr4, %eax 96 orl $(CR4_PAE | CR4_PGE), %eax 97 cmpb $0, mptramp_la57-mptramp_start(%ebx) 98 je 1f 99 orl $CR4_LA57, %eax 1001: mov %eax, %cr4 101 102 /* 103 * If the BSP reported NXE support, enable EFER.NXE for all APs 104 * prior to loading %cr3. This avoids page faults if the AP 105 * encounters memory marked with the NX bit prior to detecting and 106 * enabling NXE support. 107 */ 108 cmpb $0,mptramp_nx-mptramp_start(%ebx) 109 je 2f 110 movl $MSR_EFER, %ecx 111 rdmsr 112 orl $EFER_NXE, %eax 113 wrmsr 1142: 115 /* 116 * Enable EFER.LME so that we get long mode when all the prereqs are 117 * in place. In this case, it turns on when CR0_PG is finally enabled. 118 * Pick up a few other EFER bits that we'll use need we're here. 119 */ 120 movl $MSR_EFER, %ecx 121 rdmsr 122 orl $EFER_LME | EFER_SCE, %eax 123 wrmsr 124 125 /* 126 * Load kernel page table pointer into %cr3. 127 * %ebx is still our relocation base. 128 * 129 * Note that this only gets accessed after we're actually in 64 bit 130 * mode, however we can only set the bottom 32 bits of %cr3 in this 131 * state. This means we depend on the kernel page table being 132 * allocated from the low 4G. 133 */ 134 leal mptramp_pagetables-mptramp_start(%ebx),%eax 135 movl (%eax), %eax 136 mov %eax, %cr3 137 138 /* 139 * Finally, switch to long bit mode by enabling paging. We have 140 * to be very careful here because all the segmentation disappears 141 * out from underneath us. The spec says we can depend on the 142 * subsequent pipelined branch to execute, but *only if* everything 143 * is still identity mapped. If any mappings change, the pipeline 144 * will flush. 145 */ 146 mov %cr0, %eax 147 orl $CR0_PG, %eax 148 mov %eax, %cr0 149 150 /* 151 * At this point paging is enabled, and we are in "compatibility" mode. 152 * We do another far jump to reload %cs with the 64 bit selector. 153 * %cr3 points to a 4- or 5-level page table. 154 * We cannot yet jump all the way to the kernel because we can only 155 * specify a 32 bit linear address. So, we use yet another trampoline. 156 * 157 * The following instruction is: 158 * ljmp $kernelcode-gdt, $tramp_64-mptramp_start 159 * but gas cannot assemble that. And besides, we patch the targets 160 * in early startup and its a little clearer what we are patching. 161 */ 162jmp_64: 163 .byte 0xea /* opcode for far jump */ 164 .long tramp_64-mptramp_start /* offset in segment */ 165 .word kernelcode-gdt /* index in gdt for 64 bit code */ 166 167 /* 168 * Yeehar! We're running in 64 bit mode! We can mostly ignore our 169 * segment registers, and get on with it. 170 * We are running at the correct virtual address space. 171 * Note that the jmp is relative and that we've been relocated, 172 * so use an indirect jump. 173 */ 174 .code64 175tramp_64: 176 movabsq $entry_64,%rax /* 64 bit immediate load */ 177 jmp *%rax 178 179 .p2align 4,0 180gdt: 181 /* 182 * All segment descriptor tables start with a null descriptor 183 */ 184 .long 0x00000000 185 .long 0x00000000 186 187 /* 188 * This is the 64 bit long mode code descriptor. There is no 189 * 64 bit data descriptor. 190 */ 191kernelcode: 192 .long 0x00000000 193 .long 0x00209800 194 195 /* 196 * This is the descriptor for the 32 bit boot code. 197 * %cs: +A, +R, -C, DPL=0, +P, +D, +G 198 * Accessed, Readable, Present, 32 bit, 4G granularity 199 */ 200bootcode: 201 .long 0x0000ffff 202 .long 0x00cf9b00 203 204 /* 205 * This is the descriptor for the 32 bit boot data. 206 * We load it into %ds and %ss. The bits for each selector 207 * are interpreted slightly differently. 208 * %ds: +A, +W, -E, DPL=0, +P, +D, +G 209 * %ss: +A, +W, -E, DPL=0, +P, +B, +G 210 * Accessed, Writeable, Expand up, Present, 32 bit, 4GB 211 * For %ds, +D means 'default operand size is 32 bit'. 212 * For %ss, +B means the stack register is %esp rather than %sp. 213 */ 214bootdata: 215 .long 0x0000ffff 216 .long 0x00cf9300 217 218gdtend: 219 220 /* 221 * The address of our page table pages that the boot code 222 * uses to trampoline up to kernel address space. 223 */ 224 .globl mptramp_pagetables 225mptramp_pagetables: 226 .long 0 227 228 /* 5-level paging ? */ 229 .globl mptramp_la57 230mptramp_la57: 231 .long 0 232 233 .globl mptramp_nx 234mptramp_nx: 235 .long 0 236 237 /* 238 * The pseudo descriptor for lgdt to use. 239 */ 240lgdt_desc: 241 .word gdtend-gdt /* Length */ 242 .long gdt-mptramp_start /* Offset plus %ds << 4 */ 243 244mptramp_end: 245 /* 246 * The size of the trampoline code that needs to be relocated 247 * below the 1MiB boundary. 248 */ 249 .globl bootMP_size 250bootMP_size: 251 .long mptramp_end - mptramp_start 252 253 /* 254 * From here on down is executed in the kernel .text section. 255 */ 256 .text 257 .code64 258 .p2align 4,0 259entry_64: 260 movq bootSTK, %rsp 261 262 /* 263 * Initialize the segment register used for the PCPU area. The PCPU 264 * area will be initialized by init_secondary(), but it should be 265 * accessible before that to support sanitizer instrumentation which 266 * accesses per-CPU variables. 267 * 268 * Note that GS.base is loaded again in init_secondary(). This is not 269 * redundant: lgdt() loads a selector into %gs and this has the side 270 * effect of clearing GS.base. 271 */ 272 movl $MSR_GSBASE, %ecx 273 movq bootpcpu, %rax 274 movq %rax, %rdx 275 shrq $32, %rdx 276 wrmsr 277 278 jmp init_secondary 279