1/* 2 * Copyright (c) 1998, 2000, 2007, 2008, 2016, 2017 The NetBSD Foundation, Inc. 3 * All rights reserved. 4 * 5 * This code is derived from software contributed to The NetBSD Foundation 6 * by Charles M. Hannum and by Maxime Villard. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 18 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 19 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 21 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 * POSSIBILITY OF SUCH DAMAGE. 28 */ 29 30#define _LOCORE 31 32/* Override user-land alignment before including asm.h */ 33#define ALIGN_DATA .align 8 34#define ALIGN_TEXT .align 16,0x90 35#define _ALIGN_TEXT ALIGN_TEXT 36 37#include <machine/asm.h> 38#include <machine/param.h> 39#include <machine/pte.h> 40#include <machine/psl.h> 41#include <machine/segments.h> 42#include <machine/specialreg.h> 43#include <machine/trap.h> 44 45#define _KERNEL 46#include <machine/bootinfo.h> 47#undef _KERNEL 48 49#include "pdir.h" 50 51/* 32bit version of PG_NX */ 52#define PG_NX32 0x80000000 53 54#define TABLE_L2_ENTRIES (NKL2_KIMG_ENTRIES + 1) 55#define TABLE_L3_ENTRIES NKL3_KIMG_ENTRIES 56 57#define PROC0_PML4_OFF 0 58#define PROC0_STK_OFF (PROC0_PML4_OFF + 1 * PAGE_SIZE) 59#define PROC0_PTP3_OFF (PROC0_STK_OFF + UPAGES * PAGE_SIZE) 60#define PROC0_PTP2_OFF (PROC0_PTP3_OFF + NKL4_KIMG_ENTRIES * PAGE_SIZE) 61#define PROC0_PTP1_OFF (PROC0_PTP2_OFF + TABLE_L3_ENTRIES * PAGE_SIZE) 62#define TABLESIZE \ 63 ((NKL4_KIMG_ENTRIES + TABLE_L3_ENTRIES + TABLE_L2_ENTRIES + 1 + UPAGES) \ 64 * PAGE_SIZE) 65 66/* 67 * fillkpt - Fill in a kernel page table 68 * eax = pte (page frame | control | status) 69 * ebx = page table address 70 * ecx = number of pages to map 71 * 72 * Each entry is 8 (PDE_SIZE) bytes long: we must set the 4 upper bytes to 0. 73 */ 74#define fillkpt \ 75 cmpl $0,%ecx ; /* zero-sized? */ \ 76 je 2f ; \ 771: movl $0,(PDE_SIZE-4)(%ebx) ; /* upper 32 bits: 0 */ \ 78 movl %eax,(%ebx) ; /* store phys addr */ \ 79 addl $PDE_SIZE,%ebx ; /* next PTE/PDE */ \ 80 addl $PAGE_SIZE,%eax ; /* next phys page */ \ 81 loop 1b ; \ 822: ; 83 84/* 85 * fillkpt_nox - Same as fillkpt, but sets the NX/XD bit. 86 */ 87#define fillkpt_nox \ 88 cmpl $0,%ecx ; /* zero-sized? */ \ 89 je 2f ; \ 90 pushl %ebp ; \ 91 movl _C_LABEL(nox_flag),%ebp ; \ 921: movl %ebp,(PDE_SIZE-4)(%ebx) ; /* upper 32 bits: NX */ \ 93 movl %eax,(%ebx) ; /* store phys addr */ \ 94 addl $PDE_SIZE,%ebx ; /* next PTE/PDE */ \ 95 addl $PAGE_SIZE,%eax ; /* next phys page */ \ 96 loop 1b ; \ 97 popl %ebp ; \ 982: ; 99 100/* 101 * fillkpt_blank - Fill in a kernel page table with blank entries 102 * ebx = page table address 103 * ecx = number of pages to map 104 */ 105#define fillkpt_blank \ 106 cmpl $0,%ecx ; /* zero-sized? */ \ 107 je 2f ; \ 1081: movl $0,(PDE_SIZE-4)(%ebx) ; /* upper 32 bits: 0 */ \ 109 movl $0,(%ebx) ; /* lower 32 bits: 0 */ \ 110 addl $PDE_SIZE,%ebx ; /* next PTE/PDE */ \ 111 loop 1b ; \ 1122: ; 113 114/* 115 * Initialization 116 */ 117 .data 118 119 .globl _C_LABEL(tablesize) 120 .globl _C_LABEL(nox_flag) 121 .globl _C_LABEL(cpuid_level) 122 .globl _C_LABEL(PDPpaddr) 123 .globl _C_LABEL(boothowto) 124 .globl _C_LABEL(bootinfo) 125 .globl _C_LABEL(biosbasemem) 126 .globl _C_LABEL(biosextmem) 127 .globl _C_LABEL(atdevbase) 128 .globl _C_LABEL(stkpa) 129 .globl _C_LABEL(stkva) 130 131 .type _C_LABEL(tablesize), @object 132_C_LABEL(tablesize): .long TABLESIZE 133END(tablesize) 134 .type _C_LABEL(nox_flag), @object 135LABEL(nox_flag) .long 0 /* 32bit NOX flag, set if supported */ 136END(nox_flag) 137 .type _C_LABEL(cpuid_level), @object 138LABEL(cpuid_level) .long -1 /* max. level accepted by cpuid instr */ 139END(cpuid_level) 140 .type _C_LABEL(PDPpaddr), @object 141LABEL(PDPpaddr) .quad 0 142END(PDPpaddr) 143 .type _C_LABEL(atdevbase), @object 144LABEL(atdevbase) .quad 0 /* location of start of iomem in virt */ 145END(atdevbase) 146 .type _C_LABEL(lapicbase), @object 147LABEL(lapicbase) .quad 0 /* location of start of lapic in virt */ 148END(lapicbase) 149 .type _C_LABEL(stkpa), @object 150LABEL(stkpa) .quad 0 151END(stkpa) 152 .type _C_LABEL(stkva), @object 153LABEL(stkva) .quad 0 154END(stkva) 155 156 .globl gdt64_lo 157 .globl gdt64_start 158 159#define GDT64_LIMIT gdt64_end-gdt64_start-1 160/* Temporary gdt64, with base address in low memory */ 161 .type _C_LABEL(gdt64_lo), @object 162LABEL(gdt64_lo) 163 .word GDT64_LIMIT 164 .quad gdt64_start 165END(gdt64_lo) 166.align 64 167#undef GDT64_LIMIT 168 169 .type _C_LABEL(gdt64_start), @object 170LABEL(gdt64_start) 171 .quad 0x0000000000000000 /* always empty */ 172 .quad 0x00af9a000000ffff /* kernel CS */ 173 .quad 0x00cf92000000ffff /* kernel DS */ 174 .quad 0x0000000000000000 /* kernel TSS [1/2] */ 175 .quad 0x0000000000000000 /* kernel TSS [2/2] */ 176END(gdt64_start) 177gdt64_end: 178 179 .type _C_LABEL(farjmp64), @object 180_C_LABEL(farjmp64): 181 .long longmode 182 .word GSEL(GCODE_SEL, SEL_KPL) 183END(farjmp64) 184 185 /* Space for the temporary stack */ 186 .size tmpstk, tmpstk - . 187 .space 512 188tmpstk: 189 190 .text 191 192ENTRY(start) 193 .code32 194 195 xorl %eax,%eax 196 197 /* Switch to new stack now. */ 198 movl $_C_LABEL(tmpstk),%esp 199 200 /* First, reset the PSL. */ 201 pushl $PSL_MBO 202 popfl 203 204 xorl %eax,%eax 205 cpuid 206 movl %eax,_C_LABEL(cpuid_level) 207 208 /* 209 * Retrieve the NX/XD flag. We use the 32bit version of PG_NX. 210 */ 211 movl $0x80000001,%eax 212 cpuid 213 andl $CPUID_NOX,%edx 214 jz .Lno_NOX 215 movl $PG_NX32,_C_LABEL(nox_flag) 216.Lno_NOX: 217 218/* 219 * There are four levels of pages in amd64: PML4 -> PDP -> PD -> PT. They will 220 * be referred to as: L4 -> L3 -> L2 -> L1. 221 * 222 * Physical address space: 223 * +-----------------+------------------+ 224 * | SMALLKERN IMAGE | BOOTSTRAP TABLES | 225 * +-----------------+------------------+ 226 * (1) 227 * 228 * Virtual address space of the smallkern: 229 * +-----------------+------------------+ 230 * | SMALLKERN IMAGE | BOOTSTRAP TABLES | 231 * +-----------------+------------------+ 232 * 233 * PROC0 STK is obviously not linked as a page level. It just happens to be 234 * caught between L4 and L3. 235 * 236 * (PROC0 STK + L4 + L3 + L2 + L1) is later referred to as BOOTSTRAP TABLES. 237 * 238 * Important note: the kernel segments are properly 4k-aligned 239 * (see kern.ldscript), so there's no need to enforce alignment. 240 */ 241 242 /* Find end of the kernel image; brings us on (1). */ 243 movl $_C_LABEL(__smallkern_end),%edi 244 245 /* We are on (1). Align up for LAPIC PAGE. */ 246 movl %edi,%esi 247 addl $PGOFSET,%esi 248 andl $~PGOFSET,%esi 249 250 /* We are on the BOOTSTRAP TABLES. Save L4's physical address. */ 251 movl $_C_LABEL(PDPpaddr),%ebp 252 movl %esi,(%ebp) 253 movl $0,4(%ebp) 254 255 /* Now, zero out the BOOTSTRAP TABLES (before filling them in). */ 256 movl %esi,%edi 257 xorl %eax,%eax 258 cld 259 movl $TABLESIZE,%ecx 260 shrl $2,%ecx 261 rep 262 stosl /* copy eax -> edi */ 263 264/* 265 * Build the page tables and levels. We go from L1 to L4, and link the levels 266 * together. 267 */ 268 /* 269 * Build L1. 270 */ 271 leal (PROC0_PTP1_OFF)(%esi),%ebx 272 273 /* Skip the area below the smallkern text. */ 274 movl $(SMALLKERNTEXTOFF - SMALLKERNBASE),%ecx 275 shrl $PGSHIFT,%ecx 276 fillkpt_blank 277 278 /* Map the smallkern text RX. */ 279 movl $(SMALLKERNTEXTOFF - SMALLKERNBASE),%eax /* start of TEXT */ 280 movl $_C_LABEL(__rodata_start),%ecx 281 subl %eax,%ecx 282 shrl $PGSHIFT,%ecx 283 orl $(PTE_P),%eax 284 fillkpt 285 286 /* Map the smallkern rodata R. */ 287 movl $_C_LABEL(__rodata_start),%eax 288 movl $_C_LABEL(__data_start),%ecx 289 subl %eax,%ecx 290 shrl $PGSHIFT,%ecx 291 orl $(PTE_P),%eax 292 fillkpt_nox 293 294 /* Map the smallkern data+bss RW. */ 295 movl $_C_LABEL(__data_start),%eax 296 movl $_C_LABEL(__smallkern_end),%ecx 297 subl %eax,%ecx 298 shrl $PGSHIFT,%ecx 299 orl $(PTE_P|PTE_W),%eax 300 fillkpt_nox 301 302 /* Map some blank space, to keep pa = va. */ 303 movl $_C_LABEL(__smallkern_end),%eax 304 movl %esi,%ecx /* start of BOOTSTRAP TABLES */ 305 subl %eax,%ecx 306 shrl $PGSHIFT,%ecx 307 fillkpt_blank 308 309 /* Map the BOOTSTRAP TABLES RW. */ 310 movl %esi,%eax /* start of BOOTSTRAP TABLES */ 311 movl $TABLESIZE,%ecx /* length of BOOTSTRAP TABLES */ 312 shrl $PGSHIFT,%ecx 313 orl $(PTE_P|PTE_W),%eax 314 fillkpt_nox 315 316 /* Map the LAPIC PAGE RW. */ 317 movl $0xfee00000,%eax 318 movl $PAGE_SIZE,%ecx /* size of the LAPIC PAGE */ 319 shrl $PGSHIFT,%ecx 320 orl $(PTE_P|PTE_W/*|PTE_PCD*/),%eax 321 fillkpt_nox 322 323 /* Map the ISA I/O MEM RW. */ 324 movl $IOM_BEGIN,%eax 325 movl $IOM_SIZE,%ecx /* size of ISA I/O MEM */ 326 shrl $PGSHIFT,%ecx 327 orl $(PTE_P|PTE_W/*|PTE_PCD*/),%eax 328 fillkpt_nox 329 330 /* 331 * Build L2. Linked to L1. 332 */ 333 leal (PROC0_PTP2_OFF)(%esi),%ebx 334 leal (PROC0_PTP1_OFF)(%esi),%eax 335 orl $(PTE_P|PTE_W),%eax 336 movl $(NKL2_KIMG_ENTRIES+1),%ecx 337 fillkpt 338 339 /* 340 * Build L3. Linked to L2. 341 */ 342 leal (PROC0_PTP3_OFF)(%esi),%ebx 343 leal (PROC0_PTP2_OFF)(%esi),%eax 344 orl $(PTE_P|PTE_W),%eax 345 movl $NKL3_KIMG_ENTRIES,%ecx 346 fillkpt 347 348 /* 349 * Build L4. Linked to L3. 350 */ 351 leal (PROC0_PML4_OFF)(%esi),%ebx 352 leal (PROC0_PTP3_OFF)(%esi),%eax 353 orl $(PTE_P|PTE_W),%eax 354 movl $NKL4_KIMG_ENTRIES,%ecx 355 fillkpt 356 357 /* Install recursive top level PDE (one entry) */ 358 leal (PROC0_PML4_OFF + PDIR_SLOT_PTE * PDE_SIZE)(%esi),%ebx 359 leal (PROC0_PML4_OFF)(%esi),%eax 360 orl $(PTE_P|PTE_W),%eax 361 movl $1,%ecx 362 fillkpt_nox 363 364 /* 365 * Startup checklist: 366 * 1. Enable PAE (and SSE while here). 367 */ 368 movl %cr4,%eax 369 orl $(CR4_PAE|CR4_OSFXSR|CR4_OSXMMEXCPT),%eax 370 movl %eax,%cr4 371 372 /* 373 * 2. Set Long Mode Enable in EFER. Also enable the syscall extensions, 374 * and NOX if available. 375 */ 376 movl $MSR_EFER,%ecx 377 rdmsr 378 xorl %eax,%eax 379 orl $(EFER_LME|EFER_SCE),%eax 380 movl _C_LABEL(nox_flag),%ebx 381 cmpl $0,%ebx 382 je .Lskip_NOX 383 orl $(EFER_NXE),%eax 384.Lskip_NOX: 385 wrmsr 386 387 /* 388 * 3. Load %cr3 with pointer to PML4. 389 */ 390 movl %esi,%eax 391 movl %eax,%cr3 392 393 /* 394 * 4. Enable paging and the rest of it. 395 */ 396 movl %cr0,%eax 397 orl $(CR0_PE|CR0_PG|CR0_NE|CR0_TS|CR0_MP|CR0_WP|CR0_AM),%eax 398 movl %eax,%cr0 399 jmp compat 400compat: 401 402 /* 403 * 5. Not quite done yet, we're now in a compatibility segment, in 404 * legacy mode. We must jump to a long mode segment. Need to set up 405 * a GDT with a long mode segment in it to do that. 406 */ 407 movl $_C_LABEL(gdt64_lo),%eax 408 lgdt (%eax) 409 movl $_C_LABEL(farjmp64),%eax 410 ljmp *(%eax) 411 412 .code64 413longmode: 414 415 /* 416 * We have arrived. Everything is identity-mapped. 417 */ 418 419 /* Store lapicbase. */ 420 movq $TABLESIZE,%rdx 421 addq %rsi,%rdx 422 movq %rdx,_C_LABEL(lapicbase)(%rip) 423 424 /* Store atdevbase. */ 425 addq $PAGE_SIZE,%rdx 426 movq %rdx,_C_LABEL(atdevbase)(%rip) 427 428 /* Set up bootstrap stack. */ 429 leaq (PROC0_STK_OFF)(%rsi),%rax 430 movq %rax,_C_LABEL(stkpa)(%rip) 431 leaq (USPACE-FRAMESIZE)(%rax),%rsp 432 xorq %rbp,%rbp /* mark end of frames */ 433 434 xorw %ax,%ax 435 movw %ax,%gs 436 movw %ax,%fs 437 438 /* The first physical page available. */ 439 leaq (TABLESIZE)(%rsi),%rdi 440 441 /* 442 * Continue execution in C. 443 */ 444 call _C_LABEL(main) 445 446 ret 447END(start) 448 449/* -------------------------------------------------------------------------- */ 450 451ENTRY(lidt) 452 lidt (%rdi) 453 ret 454END(lidt) 455 456ENTRY(rdtsc) 457 xorq %rax,%rax 458 rdtsc 459 shlq $32,%rdx 460 orq %rdx,%rax 461 ret 462END(rdtsc) 463 464ENTRY(vmmcall) 465 vmmcall 466END(vmmcall) 467 468ENTRY(outsb) 469 movl %edx,%ecx 470 movl %edi,%edx 471 rep 472 outsb 473 ret 474END(outsb) 475 476ENTRY(clts) 477 clts 478 ret 479END(clts) 480 481ENTRY(sti) 482 sti 483 ret 484END(sti) 485 486ENTRY(lcr8) 487 movq %rdi, %cr8 488 ret 489END(lcr8) 490 491ENTRY(rdmsr) 492 movq %rdi, %rcx 493 xorq %rax, %rax 494 rdmsr 495 shlq $32, %rdx 496 orq %rdx, %rax 497 ret 498END(rdmsr) 499 500ENTRY(cpuid) 501 movq %rbx,%r8 502 movq %rdi,%rax 503 movq %rsi,%rcx 504 movq %rdx,%rsi 505 cpuid 506 movl %eax,0(%rsi) 507 movl %ebx,4(%rsi) 508 movl %ecx,8(%rsi) 509 movl %edx,12(%rsi) 510 movq %r8,%rbx 511 ret 512END(cpuid) 513