1/* 2 * Copyright (c) 1998, 2000, 2007, 2008, 2016, 2017 The NetBSD Foundation, Inc. 3 * All rights reserved. 4 * 5 * This code is derived from software contributed to The NetBSD Foundation 6 * by Charles M. Hannum and by Maxime Villard. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 18 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 19 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 21 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 * POSSIBILITY OF SUCH DAMAGE. 28 */ 29 30#define _LOCORE 31#define LOCORE /* DragonFly */ 32 33#include "asm.h" 34#include "pdir.h" 35 36#include <machine/param.h> 37#include <machine/segments.h> 38#include <machine/specialreg.h> 39 40#define PSL_MBO 0x00000002 /* must be zero bits */ 41 42#define PTE_P 0x001 /* P: Valid */ 43#define PTE_W 0x002 /* R/W: Read/Write */ 44#define PTE_PCD 0x010 /* PCD: Cache disable */ 45 46/* 32bit version of PG_NX */ 47#define PG_NX32 0x80000000 48 49#define TABLE_L2_ENTRIES (NKL2_KIMG_ENTRIES + 1) 50#define TABLE_L3_ENTRIES NKL3_KIMG_ENTRIES 51 52#define PROC0_PML4_OFF 0 53#define PROC0_STK_OFF (PROC0_PML4_OFF + 1 * PAGE_SIZE) 54#define PROC0_PTP3_OFF (PROC0_STK_OFF + UPAGES * PAGE_SIZE) 55#define PROC0_PTP2_OFF (PROC0_PTP3_OFF + NKL4_KIMG_ENTRIES * PAGE_SIZE) 56#define PROC0_PTP1_OFF (PROC0_PTP2_OFF + TABLE_L3_ENTRIES * PAGE_SIZE) 57#define TABLESIZE \ 58 ((NKL4_KIMG_ENTRIES + TABLE_L3_ENTRIES + TABLE_L2_ENTRIES + 1 + UPAGES) \ 59 * PAGE_SIZE) 60 61/* 62 * fillkpt - Fill in a kernel page table 63 * eax = pte (page frame | control | status) 64 * ebx = page table address 65 * ecx = number of pages to map 66 * 67 * Each entry is 8 (PDE_SIZE) bytes long: we must set the 4 upper bytes to 0. 68 */ 69#define fillkpt \ 70 cmpl $0,%ecx ; /* zero-sized? */ \ 71 je 2f ; \ 721: movl $0,(PDE_SIZE-4)(%ebx) ; /* upper 32 bits: 0 */ \ 73 movl %eax,(%ebx) ; /* store phys addr */ \ 74 addl $PDE_SIZE,%ebx ; /* next PTE/PDE */ \ 75 addl $PAGE_SIZE,%eax ; /* next phys page */ \ 76 loop 1b ; \ 772: ; 78 79/* 80 * fillkpt_nox - Same as fillkpt, but sets the NX/XD bit. 81 */ 82#define fillkpt_nox \ 83 cmpl $0,%ecx ; /* zero-sized? */ \ 84 je 2f ; \ 85 pushl %ebp ; \ 86 movl _C_LABEL(nox_flag),%ebp ; \ 871: movl %ebp,(PDE_SIZE-4)(%ebx) ; /* upper 32 bits: NX */ \ 88 movl %eax,(%ebx) ; /* store phys addr */ \ 89 addl $PDE_SIZE,%ebx ; /* next PTE/PDE */ \ 90 addl $PAGE_SIZE,%eax ; /* next phys page */ \ 91 loop 1b ; \ 92 popl %ebp ; \ 932: ; 94 95/* 96 * fillkpt_blank - Fill in a kernel page table with blank entries 97 * ebx = page table address 98 * ecx = number of pages to map 99 */ 100#define fillkpt_blank \ 101 cmpl $0,%ecx ; /* zero-sized? */ \ 102 je 2f ; \ 1031: movl $0,(PDE_SIZE-4)(%ebx) ; /* upper 32 bits: 0 */ \ 104 movl $0,(%ebx) ; /* lower 32 bits: 0 */ \ 105 addl $PDE_SIZE,%ebx ; /* next PTE/PDE */ \ 106 loop 1b ; \ 1072: ; 108 109/* 110 * Initialization 111 */ 112 .data 113 114 .globl _C_LABEL(tablesize) 115 .globl _C_LABEL(nox_flag) 116 .globl _C_LABEL(cpuid_level) 117 .globl _C_LABEL(PDPpaddr) 118 .globl _C_LABEL(atdevbase) 119 .globl _C_LABEL(lapicbase) 120 .globl _C_LABEL(stkpa) 121 .globl _C_LABEL(stkva) 122 123 .type _C_LABEL(tablesize), @object 124_C_LABEL(tablesize): .long TABLESIZE 125END(tablesize) 126 .type _C_LABEL(nox_flag), @object 127LABEL(nox_flag) .long 0 /* 32bit NOX flag, set if supported */ 128END(nox_flag) 129 .type _C_LABEL(cpuid_level), @object 130LABEL(cpuid_level) .long -1 /* max. level accepted by cpuid instr */ 131END(cpuid_level) 132 .type _C_LABEL(PDPpaddr), @object 133LABEL(PDPpaddr) .quad 0 134END(PDPpaddr) 135 .type _C_LABEL(atdevbase), @object 136LABEL(atdevbase) .quad 0 /* location of start of iomem in virt */ 137END(atdevbase) 138 .type _C_LABEL(lapicbase), @object 139LABEL(lapicbase) .quad 0 /* location of start of lapic in virt */ 140END(lapicbase) 141 .type _C_LABEL(stkpa), @object 142LABEL(stkpa) .quad 0 143END(stkpa) 144 .type _C_LABEL(stkva), @object 145LABEL(stkva) .quad 0 146END(stkva) 147 148 .globl gdt64_lo 149 .globl gdt64_start 150 151#define GDT64_LIMIT gdt64_end-gdt64_start-1 152/* Temporary gdt64, with base address in low memory */ 153 .type _C_LABEL(gdt64_lo), @object 154LABEL(gdt64_lo) 155 .word GDT64_LIMIT 156 .quad gdt64_start 157END(gdt64_lo) 158.align 64 159#undef GDT64_LIMIT 160 161 .type _C_LABEL(gdt64_start), @object 162LABEL(gdt64_start) 163 .quad 0x0000000000000000 /* always empty */ 164 .quad 0x00af9a000000ffff /* kernel CS */ 165 .quad 0x00cf92000000ffff /* kernel DS */ 166 .quad 0x0000000000000000 /* kernel TSS [1/2] */ 167 .quad 0x0000000000000000 /* kernel TSS [2/2] */ 168END(gdt64_start) 169gdt64_end: 170 171 .type _C_LABEL(farjmp64), @object 172_C_LABEL(farjmp64): 173 .long longmode 174 .word GSEL(GCODE_SEL, SEL_KPL) 175END(farjmp64) 176 177 /* Space for the temporary stack */ 178 .size tmpstk, tmpstk - . 179 .space 512 180tmpstk: 181 182 .text 183 184ENTRY(start) 185 .code32 186 187 xorl %eax,%eax 188 189 /* Switch to new stack now. */ 190 movl $_C_LABEL(tmpstk),%esp 191 192 /* First, reset the PSL. */ 193 pushl $PSL_MBO 194 popfl 195 196 xorl %eax,%eax 197 cpuid 198 movl %eax,_C_LABEL(cpuid_level) 199 200 /* 201 * Retrieve the NX/XD flag. We use the 32bit version of PG_NX. 202 */ 203 movl $0x80000001,%eax 204 cpuid 205 andl $CPUID_NOX,%edx 206 jz .Lno_NOX 207 movl $PG_NX32,_C_LABEL(nox_flag) 208.Lno_NOX: 209 210/* 211 * There are four levels of pages in amd64: PML4 -> PDP -> PD -> PT. They will 212 * be referred to as: L4 -> L3 -> L2 -> L1. 213 * 214 * Physical address space: 215 * +-----------------+------------------+ 216 * | SMALLKERN IMAGE | BOOTSTRAP TABLES | 217 * +-----------------+------------------+ 218 * (1) 219 * 220 * Virtual address space of the smallkern: 221 * +-----------------+------------------+ 222 * | SMALLKERN IMAGE | BOOTSTRAP TABLES | 223 * +-----------------+------------------+ 224 * 225 * PROC0 STK is obviously not linked as a page level. It just happens to be 226 * caught between L4 and L3. 227 * 228 * (PROC0 STK + L4 + L3 + L2 + L1) is later referred to as BOOTSTRAP TABLES. 229 * 230 * Important note: the kernel segments are properly 4k-aligned 231 * (see kern.ldscript), so there's no need to enforce alignment. 232 */ 233 234 /* Find end of the kernel image; brings us on (1). */ 235 movl $_C_LABEL(__smallkern_end),%edi 236 237 /* We are on (1). Align up for LAPIC PAGE. */ 238 movl %edi,%esi 239 addl $PGOFSET,%esi 240 andl $~PGOFSET,%esi 241 242 /* We are on the BOOTSTRAP TABLES. Save L4's physical address. */ 243 movl $_C_LABEL(PDPpaddr),%ebp 244 movl %esi,(%ebp) 245 movl $0,4(%ebp) 246 247 /* Now, zero out the BOOTSTRAP TABLES (before filling them in). */ 248 movl %esi,%edi 249 xorl %eax,%eax 250 cld 251 movl $TABLESIZE,%ecx 252 shrl $2,%ecx 253 rep 254 stosl /* copy eax -> edi */ 255 256/* 257 * Build the page tables and levels. We go from L1 to L4, and link the levels 258 * together. 259 */ 260 /* 261 * Build L1. 262 */ 263 leal (PROC0_PTP1_OFF)(%esi),%ebx 264 265 /* Skip the area below the smallkern text. */ 266 movl $(SMALLKERNTEXTOFF - SMALLKERNBASE),%ecx 267 shrl $PGSHIFT,%ecx 268 fillkpt_blank 269 270 /* Map the smallkern text RX. */ 271 movl $(SMALLKERNTEXTOFF - SMALLKERNBASE),%eax /* start of TEXT */ 272 movl $_C_LABEL(__rodata_start),%ecx 273 subl %eax,%ecx 274 shrl $PGSHIFT,%ecx 275 orl $(PTE_P),%eax 276 fillkpt 277 278 /* Map the smallkern rodata R. */ 279 movl $_C_LABEL(__rodata_start),%eax 280 movl $_C_LABEL(__data_start),%ecx 281 subl %eax,%ecx 282 shrl $PGSHIFT,%ecx 283 orl $(PTE_P),%eax 284 fillkpt_nox 285 286 /* Map the smallkern data+bss RW. */ 287 movl $_C_LABEL(__data_start),%eax 288 movl $_C_LABEL(__smallkern_end),%ecx 289 subl %eax,%ecx 290 shrl $PGSHIFT,%ecx 291 orl $(PTE_P|PTE_W),%eax 292 fillkpt_nox 293 294 /* Map some blank space, to keep pa = va. */ 295 movl $_C_LABEL(__smallkern_end),%eax 296 movl %esi,%ecx /* start of BOOTSTRAP TABLES */ 297 subl %eax,%ecx 298 shrl $PGSHIFT,%ecx 299 fillkpt_blank 300 301 /* Map the BOOTSTRAP TABLES RW. */ 302 movl %esi,%eax /* start of BOOTSTRAP TABLES */ 303 movl $TABLESIZE,%ecx /* length of BOOTSTRAP TABLES */ 304 shrl $PGSHIFT,%ecx 305 orl $(PTE_P|PTE_W),%eax 306 fillkpt_nox 307 308 /* Map the LAPIC PAGE RW. */ 309 movl $0xfee00000,%eax 310 movl $PAGE_SIZE,%ecx /* size of the LAPIC PAGE */ 311 shrl $PGSHIFT,%ecx 312 orl $(PTE_P|PTE_W/*|PTE_PCD*/),%eax 313 fillkpt_nox 314 315 /* Map the ISA I/O MEM RW. */ 316 movl $IOM_BEGIN,%eax 317 movl $IOM_SIZE,%ecx /* size of ISA I/O MEM */ 318 shrl $PGSHIFT,%ecx 319 orl $(PTE_P|PTE_W/*|PTE_PCD*/),%eax 320 fillkpt_nox 321 322 /* 323 * Build L2. Linked to L1. 324 */ 325 leal (PROC0_PTP2_OFF)(%esi),%ebx 326 leal (PROC0_PTP1_OFF)(%esi),%eax 327 orl $(PTE_P|PTE_W),%eax 328 movl $(NKL2_KIMG_ENTRIES+1),%ecx 329 fillkpt 330 331 /* 332 * Build L3. Linked to L2. 333 */ 334 leal (PROC0_PTP3_OFF)(%esi),%ebx 335 leal (PROC0_PTP2_OFF)(%esi),%eax 336 orl $(PTE_P|PTE_W),%eax 337 movl $NKL3_KIMG_ENTRIES,%ecx 338 fillkpt 339 340 /* 341 * Build L4. Linked to L3. 342 */ 343 leal (PROC0_PML4_OFF)(%esi),%ebx 344 leal (PROC0_PTP3_OFF)(%esi),%eax 345 orl $(PTE_P|PTE_W),%eax 346 movl $NKL4_KIMG_ENTRIES,%ecx 347 fillkpt 348 349 /* Install recursive top level PDE (one entry) */ 350 leal (PROC0_PML4_OFF + PDIR_SLOT_PTE * PDE_SIZE)(%esi),%ebx 351 leal (PROC0_PML4_OFF)(%esi),%eax 352 orl $(PTE_P|PTE_W),%eax 353 movl $1,%ecx 354 fillkpt_nox 355 356 /* 357 * Startup checklist: 358 * 1. Enable PAE (and SSE while here). 359 */ 360 movl %cr4,%eax 361 orl $(CR4_PAE|CR4_OSFXSR|CR4_OSXMMEXCPT),%eax 362 movl %eax,%cr4 363 364 /* 365 * 2. Set Long Mode Enable in EFER. Also enable the syscall extensions, 366 * and NOX if available. 367 */ 368 movl $MSR_EFER,%ecx 369 rdmsr 370 xorl %eax,%eax 371 orl $(EFER_LME|EFER_SCE),%eax 372 movl _C_LABEL(nox_flag),%ebx 373 cmpl $0,%ebx 374 je .Lskip_NOX 375 orl $(EFER_NXE),%eax 376.Lskip_NOX: 377 wrmsr 378 379 /* 380 * 3. Load %cr3 with pointer to PML4. 381 */ 382 movl %esi,%eax 383 movl %eax,%cr3 384 385 /* 386 * 4. Enable paging and the rest of it. 387 */ 388 movl %cr0,%eax 389 orl $(CR0_PE|CR0_PG|CR0_NE|CR0_TS|CR0_MP|CR0_WP|CR0_AM),%eax 390 movl %eax,%cr0 391 jmp compat 392compat: 393 394 /* 395 * 5. Not quite done yet, we're now in a compatibility segment, in 396 * legacy mode. We must jump to a long mode segment. Need to set up 397 * a GDT with a long mode segment in it to do that. 398 */ 399 movl $_C_LABEL(gdt64_lo),%eax 400 lgdt (%eax) 401 movl $_C_LABEL(farjmp64),%eax 402 ljmp *(%eax) 403 404 .code64 405longmode: 406 407 /* 408 * We have arrived. Everything is identity-mapped. 409 */ 410 411 /* Store lapicbase. */ 412 movq $TABLESIZE,%rdx 413 addq %rsi,%rdx 414 movq %rdx,_C_LABEL(lapicbase)(%rip) 415 416 /* Store atdevbase. */ 417 addq $PAGE_SIZE,%rdx 418 movq %rdx,_C_LABEL(atdevbase)(%rip) 419 420 /* Set up bootstrap stack. */ 421 leaq (PROC0_STK_OFF)(%rsi),%rax 422 movq %rax,_C_LABEL(stkpa)(%rip) 423 leaq (USPACE-FRAMESIZE)(%rax),%rsp 424 xorq %rbp,%rbp /* mark end of frames */ 425 426 xorw %ax,%ax 427 movw %ax,%gs 428 movw %ax,%fs 429 430 /* The first physical page available. */ 431 leaq (TABLESIZE)(%rsi),%rdi 432 433 /* 434 * Continue execution in C. 435 */ 436 call _C_LABEL(main) 437 438 ret 439END(start) 440 441/* -------------------------------------------------------------------------- */ 442 443ENTRY(lidt) 444 lidt (%rdi) 445 ret 446END(lidt) 447 448ENTRY(vmmcall) 449 vmmcall 450END(vmmcall) 451 452ENTRY(outsb) 453 movl %edx,%ecx 454 movl %edi,%edx 455 rep 456 outsb 457 ret 458END(outsb) 459 460ENTRY(clts) 461 clts 462 ret 463END(clts) 464 465ENTRY(sti) 466 sti 467 ret 468END(sti) 469 470ENTRY(lcr8) 471 movq %rdi, %cr8 472 ret 473END(lcr8) 474 475ENTRY(rdmsr) 476 movq %rdi, %rcx 477 xorq %rax, %rax 478 rdmsr 479 shlq $32, %rdx 480 orq %rdx, %rax 481 ret 482END(rdmsr) 483 484ENTRY(cpuid) 485 movq %rbx,%r8 486 movq %rdi,%rax 487 movq %rsi,%rcx 488 movq %rdx,%rsi 489 cpuid 490 movl %eax,0(%rsi) 491 movl %ebx,4(%rsi) 492 movl %ecx,8(%rsi) 493 movl %edx,12(%rsi) 494 movq %r8,%rbx 495 ret 496END(cpuid) 497