1/* 2 * librm: a library for interfacing to real-mode code 3 * 4 * Michael Brown <mbrown@fensystems.co.uk> 5 * 6 */ 7 8FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ) 9 10/* Drag in general configuration */ 11#include <config/general.h> 12 13/* Drag in local definitions */ 14#include "librm.h" 15 16/* CR0: protection enabled */ 17#define CR0_PE ( 1 << 0 ) 18 19/* CR0: paging */ 20#define CR0_PG ( 1 << 31 ) 21 22/* CR4: physical address extensions */ 23#define CR4_PAE ( 1 << 5 ) 24 25/* Extended feature enable MSR (EFER) */ 26#define MSR_EFER 0xc0000080 27 28/* EFER: long mode enable */ 29#define EFER_LME ( 1 << 8 ) 30 31/* Page: present */ 32#define PG_P 0x01 33 34/* Page: read/write */ 35#define PG_RW 0x02 36 37/* Page: user/supervisor */ 38#define PG_US 0x04 39 40/* Page: page size */ 41#define PG_PS 0x80 42 43/* Size of various paging-related data structures */ 44#define SIZEOF_PTE_LOG2 3 45#define SIZEOF_PTE ( 1 << SIZEOF_PTE_LOG2 ) 46#define SIZEOF_PT_LOG2 12 47#define SIZEOF_PT ( 1 << SIZEOF_PT_LOG2 ) 48#define SIZEOF_4KB_PAGE_LOG2 12 49#define SIZEOF_4KB_PAGE ( 1 << SIZEOF_4KB_PAGE_LOG2 ) 50#define SIZEOF_2MB_PAGE_LOG2 21 51#define SIZEOF_2MB_PAGE ( 1 << SIZEOF_2MB_PAGE_LOG2 ) 52#define SIZEOF_LOW_4GB_LOG2 32 53#define SIZEOF_LOW_4GB ( 1 << SIZEOF_LOW_4GB_LOG2 ) 54 55/* Size of various C data structures */ 56#define SIZEOF_I386_SEG_REGS 12 57#define SIZEOF_I386_REGS 32 58#define SIZEOF_REAL_MODE_REGS ( SIZEOF_I386_SEG_REGS + SIZEOF_I386_REGS ) 59#define SIZEOF_I386_FLAGS 4 60#define SIZEOF_I386_ALL_REGS ( SIZEOF_REAL_MODE_REGS + SIZEOF_I386_FLAGS ) 61#define SIZEOF_X86_64_REGS 128 62 63/* Size of an address */ 64#ifdef __x86_64__ 65#define SIZEOF_ADDR 8 66#else 67#define SIZEOF_ADDR 4 68#endif 69 70/* Default code size */ 71#ifdef __x86_64__ 72#define CODE_DEFAULT code64 73#else 74#define CODE_DEFAULT code32 75#endif 76 77/* Selectively assemble code for 32-bit/64-bit builds */ 78#ifdef __x86_64__ 79#define if32 if 0 80#define if64 if 1 81#else 82#define if32 if 1 83#define if64 if 0 84#endif 85 86/**************************************************************************** 87 * Global descriptor table 88 * 89 * Call init_librm to set up the GDT before attempting to use any 90 * protected-mode code. 91 * 92 * NOTE: This must be located before prot_to_real, otherwise gas 93 * throws a "can't handle non absolute segment in `ljmp'" error due to 94 * not knowing the value of REAL_CS when the ljmp is encountered. 95 * 96 * Note also that putting ".word gdt_end - gdt - 1" directly into 97 * gdt_limit, rather than going via gdt_length, will also produce the 98 * "non absolute segment" error. This is most probably a bug in gas. 99 **************************************************************************** 100 */ 101 .section ".data16.gdt", "aw", @progbits 102 .align 16 103gdt: 104gdtr: /* The first GDT entry is unused, the GDTR can fit here. */ 105gdt_limit: .word gdt_length - 1 106gdt_base: .long 0 107 .word 0 /* padding */ 108 109 .org gdt + VIRTUAL_CS, 0 110virtual_cs: /* 32 bit protected mode code segment, virtual addresses */ 111 .word 0xffff, 0 112 .byte 0, 0x9f, 0xcf, 0 113 114 .org gdt + VIRTUAL_DS, 0 115virtual_ds: /* 32 bit protected mode data segment, virtual addresses */ 116 .word 0xffff, 0 117 .byte 0, 0x93, 0xcf, 0 118 119 .org gdt + PHYSICAL_CS, 0 120physical_cs: /* 32 bit protected mode code segment, physical addresses */ 121 .word 0xffff, 0 122 .byte 0, 0x9f, 0xcf, 0 123 124 .org gdt + PHYSICAL_DS, 0 125physical_ds: /* 32 bit protected mode data segment, physical addresses */ 126 .word 0xffff, 0 127 .byte 0, 0x93, 0xcf, 0 128 129 .org gdt + REAL_CS, 0 130real_cs: /* 16 bit real mode code segment */ 131 .word 0xffff, 0 132 .byte 0, 0x9b, 0x00, 0 133 134 .org gdt + REAL_DS, 0 135real_ds: /* 16 bit real mode data segment */ 136 .word 0xffff, 0 137 .byte 0, 0x93, 0x00, 0 138 139 .org gdt + P2R_DS, 0 140p2r_ds: /* 16 bit real mode data segment for prot_to_real transition */ 141 .word 0xffff, ( P2R_DS << 4 ) 142 .byte 0, 0x93, 0x00, 0 143 144 .org gdt + LONG_CS, 0 145long_cs: /* 64 bit long mode code segment */ 146 .word 0, 0 147 .byte 0, 0x9a, 0x20, 0 148 149gdt_end: 150 .equ gdt_length, gdt_end - gdt 151 152/**************************************************************************** 153 * Stored real-mode and protected-mode stack pointers 154 * 155 * The real-mode stack pointer is stored here whenever real_to_prot 156 * is called and restored whenever prot_to_real is called. The 157 * converse happens for the protected-mode stack pointer. 158 * 159 * Despite initial appearances this scheme is, in fact re-entrant, 160 * because program flow dictates that we always return via the point 161 * we left by. For example: 162 * PXE API call entry 163 * 1 real => prot 164 * ... 165 * Print a text string 166 * ... 167 * 2 prot => real 168 * INT 10 169 * 3 real => prot 170 * ... 171 * ... 172 * 4 prot => real 173 * PXE API call exit 174 * 175 * At point 1, the RM mode stack value, say RPXE, is stored in 176 * rm_ss,sp. We want this value to still be present in rm_ss,sp when 177 * we reach point 4. 178 * 179 * At point 2, the RM stack value is restored from RPXE. At point 3, 180 * the RM stack value is again stored in rm_ss,sp. This *does* 181 * overwrite the RPXE that we have stored there, but it's the same 182 * value, since the code between points 2 and 3 has managed to return 183 * to us. 184 **************************************************************************** 185 */ 186 .section ".bss.rm_ss_sp", "aw", @nobits 187 .globl rm_sp 188rm_sp: .word 0 189 .globl rm_ss 190rm_ss: .word 0 191 192 .section ".data.pm_esp", "aw", @progbits 193pm_esp: .long VIRTUAL(_estack) 194 195/**************************************************************************** 196 * Temporary static data buffer 197 * 198 * This is used to reduce the amount of real-mode stack space consumed 199 * during mode transitions, since we are sometimes called with very 200 * little real-mode stack space available. 201 **************************************************************************** 202 */ 203 /* Temporary static buffer usage by virt_call */ 204 .struct 0 205VC_TMP_GDT: .space 6 206VC_TMP_IDT: .space 6 207VC_TMP_PAD: .space 4 /* for alignment */ 208.if64 209VC_TMP_CR3: .space 4 210VC_TMP_CR4: .space 4 211VC_TMP_EMER: .space 8 212.endif 213#ifdef TIVOLI_VMM_WORKAROUND 214VC_TMP_FXSAVE: .space 512 215#endif 216VC_TMP_END: 217 .previous 218 219 /* Temporary static buffer usage by real_call */ 220 .struct 0 221RC_TMP_FUNCTION: .space 4 222RC_TMP_END: 223 .previous 224 225 /* Shared temporary static buffer */ 226 .section ".bss16.rm_tmpbuf", "aw", @nobits 227 .align 16 228rm_tmpbuf: 229 .space VC_TMP_END 230 .size rm_tmpbuf, . - rm_tmpbuf 231 232/**************************************************************************** 233 * Virtual address offsets 234 * 235 * These are used by the protected-mode code to map between virtual 236 * and physical addresses, and to access variables in the .text16 or 237 * .data16 segments. 238 **************************************************************************** 239 */ 240 .struct 0 241VA_VIRT_OFFSET: .space SIZEOF_ADDR 242VA_TEXT16: .space SIZEOF_ADDR 243VA_DATA16: .space SIZEOF_ADDR 244VA_SIZE: 245 .previous 246 247 /* Internal copies, used only by librm itself */ 248 .section ".bss16.rm_virt_addrs", "aw", @nobits 249rm_virt_addrs: .space VA_SIZE 250 .equ rm_virt_offset, ( rm_virt_addrs + VA_VIRT_OFFSET ) 251 .equ rm_text16, ( rm_virt_addrs + VA_TEXT16 ) 252 .equ rm_data16, ( rm_virt_addrs + VA_DATA16 ) 253 254 /* Externally visible variables, used by C code */ 255 .section ".bss.virt_addrs", "aw", @nobits 256virt_addrs: .space VA_SIZE 257 .globl virt_offset 258 .equ virt_offset, ( virt_addrs + VA_VIRT_OFFSET ) 259 .globl text16 260 .equ text16, ( virt_addrs + VA_TEXT16 ) 261 .globl data16 262 .equ data16, ( virt_addrs + VA_DATA16 ) 263 264/**************************************************************************** 265 * init_librm (real-mode far call, 16-bit real-mode far return address) 266 * 267 * Initialise the GDT ready for transitions to protected mode. 268 * 269 * Parameters: 270 * %cs : .text16 segment 271 * %ds : .data16 segment 272 * %edi : Physical base of protected-mode code 273 **************************************************************************** 274 */ 275 .section ".text16.init_librm", "ax", @progbits 276 .code16 277 .globl init_librm 278init_librm: 279 /* Preserve registers */ 280 pushl %eax 281 pushl %ebx 282 pushl %edi 283 284 /* Store rm_virt_offset and set up virtual_cs and virtual_ds segments */ 285 subl $VIRTUAL(_textdata), %edi 286 movl %edi, rm_virt_offset 287.if64 ; setae (rm_virt_offset+4) ; .endif 288 movl %edi, %eax 289 movw $virtual_cs, %bx 290 call set_seg_base 291 movw $virtual_ds, %bx 292 call set_seg_base 293 294 /* Store rm_cs and rm_text16, set up real_cs segment */ 295 xorl %eax, %eax 296 movw %cs, %ax 297 movw %ax, %cs:rm_cs 298 shll $4, %eax 299 movw $real_cs, %bx 300 call set_seg_base 301.if32 ; subl %edi, %eax ; .endif 302 movl %eax, rm_text16 303 304 /* Store rm_ds and rm_data16, set up real_ds segment and GDT base */ 305 xorl %eax, %eax 306 movw %ds, %ax 307 movw %ax, %cs:rm_ds 308 shll $4, %eax 309 movw $real_ds, %bx 310 call set_seg_base 311 movl %eax, gdt_base 312 addl $gdt, gdt_base 313.if32 ; subl %edi, %eax ; .endif 314 movl %eax, rm_data16 315 316 /* Configure virt_call for protected mode, if applicable */ 317.if64 ; movl $VIRTUAL(vc_pmode), %cs:vc_jmp_offset ; .endif 318 319 /* Switch to protected mode */ 320 virtcall init_librm_pmode 321 .section ".text.init_librm", "ax", @progbits 322 .code32 323init_librm_pmode: 324 325 /* Store virt_offset, text16, and data16 */ 326 pushw %ds 327 movw $REAL_DS, %ax 328 movw %ax, %ds 329 movl $rm_virt_addrs, %esi 330 movl $VIRTUAL(virt_addrs), %edi 331 movl $( VA_SIZE / 4 ), %ecx 332 rep movsl 333 popw %ds 334 335.if64 ; /* Initialise long mode, if applicable */ 336 movl VIRTUAL(virt_offset), %edi 337 leal VIRTUAL(p2l_ljmp_target)(%edi), %eax 338 movl %eax, VIRTUAL(p2l_ljmp_offset) 339 call init_pages 340.endif 341 /* Return to real mode */ 342 ret 343 .section ".text16.init_librm", "ax", @progbits 344 .code16 345init_librm_rmode: 346 347 /* Configure virt_call for long mode, if applicable */ 348.if64 ; movl $VIRTUAL(vc_lmode), %cs:vc_jmp_offset ; .endif 349 350 /* Initialise IDT */ 351 virtcall init_idt 352 353 /* Restore registers */ 354 popl %edi 355 popl %ebx 356 popl %eax 357 lret 358 359 .section ".text16.set_seg_base", "ax", @progbits 360 .code16 361set_seg_base: 3621: movw %ax, 2(%bx) 363 rorl $16, %eax 364 movb %al, 4(%bx) 365 movb %ah, 7(%bx) 366 roll $16, %eax 367 ret 368 369/**************************************************************************** 370 * real_to_prot (real-mode near call, 32-bit virtual return address) 371 * 372 * Switch from 16-bit real-mode to 32-bit protected mode with virtual 373 * addresses. The real-mode %ss:sp is stored in rm_ss and rm_sp, and 374 * the protected-mode %esp is restored from the saved pm_esp. 375 * Interrupts are disabled. All other registers may be destroyed. 376 * 377 * The return address for this function should be a 32-bit virtual 378 * address. 379 * 380 * Parameters: 381 * %ecx : number of bytes to move from RM stack to PM stack 382 * %edx : number of bytes to copy from RM temporary buffer to PM stack 383 * 384 **************************************************************************** 385 */ 386 .section ".text16.real_to_prot", "ax", @progbits 387 .code16 388real_to_prot: 389 /* Enable A20 line */ 390 call enable_a20 391 /* A failure at this point is fatal, and there's nothing we 392 * can do about it other than lock the machine to make the 393 * problem immediately visible. 394 */ 3951: jc 1b 396 397 /* Make sure we have our data segment available */ 398 movw %cs:rm_ds, %ds 399 400 /* Add protected-mode return address to length of data to be copied */ 401 addw $4, %cx /* %ecx must be less than 64kB anyway */ 402 403 /* Real-mode %ss:%sp => %ebp and virtual address => %esi */ 404 xorl %eax, %eax 405 movw %ss, %ax 406 shll $4, %eax 407 movzwl %sp, %ebp 408 addr32 leal (%eax,%ebp), %esi 409 subl rm_virt_offset, %esi 410 shll $12, %eax 411 orl %eax, %ebp 412 413 /* Real-mode data segment virtual address => %ebx */ 414 movl rm_data16, %ebx 415.if64 ; subl rm_virt_offset, %ebx ; .endif 416 417 /* Load protected-mode global descriptor table */ 418 data32 lgdt gdtr 419 420 /* Zero segment registers. This wastes around 12 cycles on 421 * real hardware, but saves a substantial number of emulated 422 * instructions under KVM. 423 */ 424 xorw %ax, %ax 425 movw %ax, %ds 426 movw %ax, %es 427 movw %ax, %fs 428 movw %ax, %gs 429 movw %ax, %ss 430 431 /* Switch to protected mode (with paging disabled if applicable) */ 432 cli 433 movl %cr0, %eax 434.if64 ; andl $~CR0_PG, %eax ; .endif 435 orb $CR0_PE, %al 436 movl %eax, %cr0 437 data32 ljmp $VIRTUAL_CS, $VIRTUAL(r2p_pmode) 438 .section ".text.real_to_prot", "ax", @progbits 439 .code32 440r2p_pmode: 441 /* Set up protected-mode data segments and stack pointer */ 442 movw $VIRTUAL_DS, %ax 443 movw %ax, %ds 444 movw %ax, %es 445 movw %ax, %fs 446 movw %ax, %gs 447 movw %ax, %ss 448 movl VIRTUAL(pm_esp), %esp 449 450 /* Load protected-mode interrupt descriptor table */ 451 lidt VIRTUAL(idtr32) 452 453 /* Record real-mode %ss:sp (after removal of data) */ 454 addl %ecx, %ebp 455 movl %ebp, VIRTUAL(rm_sp) 456 457 /* Move data from RM stack to PM stack */ 458 subl %edx, %esp 459 subl %ecx, %esp 460 movl %esp, %edi 461 rep movsb 462 463 /* Copy data from RM temporary buffer to PM stack */ 464 leal rm_tmpbuf(%ebx), %esi 465 movl %edx, %ecx 466 rep movsb 467 468 /* Return to virtual address */ 469 ret 470 471/**************************************************************************** 472 * prot_to_real (protected-mode near call, 32-bit real-mode return address) 473 * 474 * Switch from 32-bit protected mode with virtual addresses to 16-bit 475 * real mode. The protected-mode %esp is stored in pm_esp and the 476 * real-mode %ss:sp is restored from the saved rm_ss and rm_sp. The 477 * high word of the real-mode %esp is set to zero. All real-mode data 478 * segment registers are loaded from the saved rm_ds. Interrupts are 479 * *not* enabled, since we want to be able to use prot_to_real in an 480 * ISR. All other registers may be destroyed. 481 * 482 * The return address for this function should be a 32-bit (sic) 483 * real-mode offset within .code16. 484 * 485 * Parameters: 486 * %ecx : number of bytes to move from PM stack to RM stack 487 * %edx : number of bytes to move from PM stack to RM temporary buffer 488 * %esi : real-mode global and interrupt descriptor table registers 489 * 490 **************************************************************************** 491 */ 492 .section ".text.prot_to_real", "ax", @progbits 493 .code32 494prot_to_real: 495 /* Copy real-mode global descriptor table register to RM code segment */ 496 movl VIRTUAL(text16), %edi 497.if64 ; subl VIRTUAL(virt_offset), %edi ; .endif 498 leal rm_gdtr(%edi), %edi 499 movsw 500 movsl 501 502 /* Load real-mode interrupt descriptor table register */ 503 lidt (%esi) 504 505 /* Add return address to data to be moved to RM stack */ 506 addl $4, %ecx 507 508 /* Real-mode %ss:sp => %ebp and virtual address => %edi */ 509 movl VIRTUAL(rm_sp), %ebp 510 subl %ecx, %ebp 511 movzwl VIRTUAL(rm_ss), %eax 512 shll $4, %eax 513 movzwl %bp, %edi 514 addl %eax, %edi 515 subl VIRTUAL(virt_offset), %edi 516 517 /* Move data from PM stack to RM stack */ 518 movl %esp, %esi 519 rep movsb 520 521 /* Move data from PM stack to RM temporary buffer */ 522 movl VIRTUAL(data16), %edi 523.if64 ; subl VIRTUAL(virt_offset), %edi ; .endif 524 addl $rm_tmpbuf, %edi 525 movl %edx, %ecx 526 rep movsb 527 528 /* Record protected-mode %esp (after removal of data) */ 529 movl %esi, VIRTUAL(pm_esp) 530 531 /* Load real-mode segment limits */ 532 movw $P2R_DS, %ax 533 movw %ax, %ds 534 movw %ax, %es 535 movw %ax, %fs 536 movw %ax, %gs 537 movw %ax, %ss 538 ljmp $REAL_CS, $p2r_rmode 539 .section ".text16.prot_to_real", "ax", @progbits 540 .code16 541p2r_rmode: 542 /* Load real-mode GDT */ 543 data32 lgdt %cs:rm_gdtr 544 /* Switch to real mode */ 545 movl %cr0, %eax 546 andb $0!CR0_PE, %al 547 movl %eax, %cr0 548p2r_ljmp_rm_cs: 549 ljmp $0, $1f 5501: 551 /* Set up real-mode data segments and stack pointer */ 552 movw %cs:rm_ds, %ax 553 movw %ax, %ds 554 movw %ax, %es 555 movw %ax, %fs 556 movw %ax, %gs 557 movl %ebp, %eax 558 shrl $16, %eax 559 movw %ax, %ss 560 movzwl %bp, %esp 561 562 /* Return to real-mode address */ 563 data32 ret 564 565 566 /* Real-mode code and data segments. Assigned by the call to 567 * init_librm. rm_cs doubles as the segment part of the jump 568 * instruction used by prot_to_real. Both are located in 569 * .text16 rather than .data16: rm_cs since it forms part of 570 * the jump instruction within the code segment, and rm_ds 571 * since real-mode code needs to be able to locate the data 572 * segment with no other reference available. 573 */ 574 .globl rm_cs 575 .equ rm_cs, ( p2r_ljmp_rm_cs + 3 ) 576 577 .section ".text16.data.rm_ds", "aw", @progbits 578 .globl rm_ds 579rm_ds: .word 0 580 581 /* Real-mode global and interrupt descriptor table registers */ 582 .section ".text16.data.rm_gdtr", "aw", @progbits 583rm_gdtr: 584 .word 0 /* Limit */ 585 .long 0 /* Base */ 586 587/**************************************************************************** 588 * phys_to_prot (protected-mode near call, 32-bit physical return address) 589 * 590 * Switch from 32-bit protected mode with physical addresses to 32-bit 591 * protected mode with virtual addresses. %esp is adjusted to a 592 * virtual address. All other registers are preserved. 593 * 594 * The return address for this function should be a 32-bit physical 595 * (sic) address. 596 * 597 **************************************************************************** 598 */ 599 .section ".text.phys_to_prot", "ax", @progbits 600 .code32 601 .globl phys_to_prot 602phys_to_prot: 603 /* Preserve registers */ 604 pushl %eax 605 pushl %ebp 606 607 /* Switch to virtual code segment */ 608 cli 609 ljmp $VIRTUAL_CS, $VIRTUAL(1f) 6101: 611 /* Switch to virtual data segment and adjust %esp */ 612 movw $VIRTUAL_DS, %ax 613 movw %ax, %ds 614 movw %ax, %es 615 movw %ax, %fs 616 movw %ax, %gs 617 movw %ax, %ss 618 movl VIRTUAL(virt_offset), %ebp 619 subl %ebp, %esp 620 621 /* Adjust return address to a virtual address */ 622 subl %ebp, 8(%esp) 623 624 /* Restore registers and return */ 625 popl %ebp 626 popl %eax 627 ret 628 629.if32 /* Expose as _phys_to_virt for use by COMBOOT, if applicable */ 630 .globl _phys_to_virt 631 .equ _phys_to_virt, phys_to_prot 632.endif 633 634/**************************************************************************** 635 * prot_to_phys (protected-mode near call, 32-bit virtual return address) 636 * 637 * Switch from 32-bit protected mode with virtual addresses to 32-bit 638 * protected mode with physical addresses. %esp is adjusted to a 639 * physical address. All other registers are preserved. 640 * 641 * The return address for this function should be a 32-bit virtual 642 * (sic) address. 643 * 644 **************************************************************************** 645 */ 646 .section ".text.prot_to_phys", "ax", @progbits 647 .code32 648prot_to_phys: 649 /* Preserve registers */ 650 pushl %eax 651 pushl %ebp 652 653 /* Adjust return address to a physical address */ 654 movl VIRTUAL(virt_offset), %ebp 655 addl %ebp, 8(%esp) 656 657 /* Switch to physical code segment */ 658 cli 659 pushl $PHYSICAL_CS 660 leal VIRTUAL(1f)(%ebp), %eax 661 pushl %eax 662 lret 6631: 664 /* Switch to physical data segment and adjust %esp */ 665 movw $PHYSICAL_DS, %ax 666 movw %ax, %ds 667 movw %ax, %es 668 movw %ax, %fs 669 movw %ax, %gs 670 movw %ax, %ss 671 addl %ebp, %esp 672 673 /* Restore registers and return */ 674 popl %ebp 675 popl %eax 676 ret 677 678.if32 /* Expose as _virt_to_phys for use by COMBOOT, if applicable */ 679 .globl _virt_to_phys 680 .equ _virt_to_phys, prot_to_phys 681.endif 682 683/**************************************************************************** 684 * intr_to_prot (protected-mode near call, 32-bit virtual return address) 685 * 686 * Switch from 32-bit protected mode with a virtual code segment and 687 * either a physical or virtual stack segment to 32-bit protected mode 688 * with normal virtual addresses. %esp is adjusted if necessary to a 689 * virtual address. All other registers are preserved. 690 * 691 * The return address for this function should be a 32-bit virtual 692 * address. 693 * 694 **************************************************************************** 695 */ 696 .section ".text.intr_to_prot", "ax", @progbits 697 .code32 698 .globl intr_to_prot 699intr_to_prot: 700 /* Preserve registers */ 701 pushl %eax 702 703 /* Check whether stack segment is physical or virtual */ 704 movw %ss, %ax 705 cmpw $VIRTUAL_DS, %ax 706 movw $VIRTUAL_DS, %ax 707 708 /* Reload data segment registers */ 709 movw %ax, %ds 710 movw %ax, %es 711 movw %ax, %fs 712 movw %ax, %gs 713 714 /* Reload stack segment and adjust %esp if necessary */ 715 je 1f 716 movw %ax, %ss 717 subl VIRTUAL(virt_offset), %esp 7181: 719 /* Restore registers and return */ 720 popl %eax 721 ret 722 723 /* Expose as _intr_to_virt for use by GDB */ 724 .globl _intr_to_virt 725 .equ _intr_to_virt, intr_to_prot 726 727/**************************************************************************** 728 * prot_to_long (protected-mode near call, 32-bit virtual return address) 729 * 730 * Switch from 32-bit protected mode with virtual addresses to 64-bit 731 * long mode. The protected-mode %esp is adjusted to a physical 732 * address. All other registers are preserved. 733 * 734 * The return address for this function should be a 32-bit (sic) 735 * virtual address. 736 * 737 **************************************************************************** 738 */ 739 .if64 740 741 .section ".text.prot_to_long", "ax", @progbits 742 .code32 743prot_to_long: 744 /* Preserve registers */ 745 pushl %eax 746 pushl %ecx 747 pushl %edx 748 749 /* Set up PML4 */ 750 movl VIRTUAL(pml4), %eax 751 movl %eax, %cr3 752 753 /* Enable PAE */ 754 movl %cr4, %eax 755 orb $CR4_PAE, %al 756 movl %eax, %cr4 757 758 /* Enable long mode */ 759 movl $MSR_EFER, %ecx 760 rdmsr 761 orw $EFER_LME, %ax 762 wrmsr 763 764 /* Enable paging */ 765 movl %cr0, %eax 766 orl $CR0_PG, %eax 767 movl %eax, %cr0 768 769 /* Restore registers */ 770 popl %edx 771 popl %ecx 772 popl %eax 773 774 /* Construct 64-bit return address */ 775 pushl (%esp) 776 movl $0xffffffff, 4(%esp) 777p2l_ljmp: 778 /* Switch to long mode (using a physical %rip) */ 779 ljmp $LONG_CS, $0 780 .code64 781p2l_lmode: 782 /* Adjust and zero-extend %esp to a physical address */ 783 addl virt_offset, %esp 784 785 /* Use long-mode IDT */ 786 lidt idtr64 787 788 /* Return to virtual address */ 789 ret 790 791 /* Long mode jump offset and target. Required since an ljmp 792 * in protected mode will zero-extend the offset, and so 793 * cannot reach an address within the negative 2GB as used by 794 * -mcmodel=kernel. Assigned by the call to init_librm. 795 */ 796 .equ p2l_ljmp_offset, ( p2l_ljmp + 1 ) 797 .equ p2l_ljmp_target, p2l_lmode 798 799 .endif 800 801/**************************************************************************** 802 * long_to_prot (long-mode near call, 64-bit virtual return address) 803 * 804 * Switch from 64-bit long mode to 32-bit protected mode with virtual 805 * addresses. The long-mode %rsp is adjusted to a virtual address. 806 * All other registers are preserved. 807 * 808 * The return address for this function should be a 64-bit (sic) 809 * virtual address. 810 * 811 **************************************************************************** 812 */ 813 .if64 814 815 .section ".text.long_to_prot", "ax", @progbits 816 .code64 817long_to_prot: 818 /* Switch to protected mode */ 819 ljmp *l2p_vector 820 .code32 821l2p_pmode: 822 /* Adjust %esp to a virtual address */ 823 subl VIRTUAL(virt_offset), %esp 824 825 /* Preserve registers */ 826 pushl %eax 827 pushl %ecx 828 pushl %edx 829 830 /* Disable paging */ 831 movl %cr0, %eax 832 andl $~CR0_PG, %eax 833 movl %eax, %cr0 834 835 /* Disable PAE (in case external non-PAE-aware code enables paging) */ 836 movl %cr4, %eax 837 andb $~CR4_PAE, %al 838 movl %eax, %cr4 839 840 /* Disable long mode */ 841 movl $MSR_EFER, %ecx 842 rdmsr 843 andw $~EFER_LME, %ax 844 wrmsr 845 846 /* Restore registers */ 847 popl %edx 848 popl %ecx 849 popl %eax 850 851 /* Use protected-mode IDT */ 852 lidt VIRTUAL(idtr32) 853 854 /* Return */ 855 ret $4 856 857 /* Long mode jump vector. Required since there is no "ljmp 858 * immediate" instruction in long mode. 859 */ 860 .section ".data.l2p_vector", "aw", @progbits 861l2p_vector: 862 .long VIRTUAL(l2p_pmode), VIRTUAL_CS 863 864 .endif 865 866/**************************************************************************** 867 * long_save_regs (long-mode near call, 64-bit virtual return address) 868 * 869 * Preserve registers that are accessible only in long mode. This 870 * includes %r8-%r15 and the upper halves of %rax, %rbx, %rcx, %rdx, 871 * %rsi, %rdi, and %rbp. 872 * 873 **************************************************************************** 874 */ 875 .if64 876 877 .section ".text.long_preserve_regs", "ax", @progbits 878 .code64 879long_preserve_regs: 880 /* Preserve registers */ 881 pushq %rax 882 pushq %rcx 883 pushq %rdx 884 pushq %rbx 885 pushq %rsp 886 pushq %rbp 887 pushq %rsi 888 pushq %rdi 889 pushq %r8 890 pushq %r9 891 pushq %r10 892 pushq %r11 893 pushq %r12 894 pushq %r13 895 pushq %r14 896 pushq %r15 897 898 /* Return */ 899 jmp *SIZEOF_X86_64_REGS(%rsp) 900 901 .endif 902 903/**************************************************************************** 904 * long_restore_regs (long-mode near call, 64-bit virtual return address) 905 * 906 * Restore registers that are accessible only in long mode. This 907 * includes %r8-%r15 and the upper halves of %rax, %rbx, %rcx, %rdx, 908 * %rsi, %rdi, and %rbp. 909 * 910 **************************************************************************** 911 */ 912 .if64 913 914 .section ".text.long_restore_regs", "ax", @progbits 915 .code64 916long_restore_regs: 917 /* Move return address above register dump */ 918 popq SIZEOF_X86_64_REGS(%rsp) 919 920 /* Restore registers */ 921 popq %r15 922 popq %r14 923 popq %r13 924 popq %r12 925 popq %r11 926 popq %r10 927 popq %r9 928 popq %r8 929 movl %edi, (%rsp) 930 popq %rdi 931 movl %esi, (%rsp) 932 popq %rsi 933 movl %ebp, (%rsp) 934 popq %rbp 935 leaq 8(%rsp), %rsp /* discard */ 936 movl %ebx, (%rsp) 937 popq %rbx 938 movl %edx, (%rsp) 939 popq %rdx 940 movl %ecx, (%rsp) 941 popq %rcx 942 movl %eax, (%rsp) 943 popq %rax 944 945 /* Return */ 946 ret 947 948 .endif 949 950/**************************************************************************** 951 * virt_call (real-mode near call, 16-bit real-mode near return address) 952 * 953 * Call a specific C function in 32-bit protected mode or 64-bit long 954 * mode (as applicable). The prototype of the C function must be 955 * void function ( struct i386_all_regs *ix86 ); 956 * ix86 will point to a struct containing the real-mode registers 957 * at entry to virt_call(). 958 * 959 * All registers will be preserved across virt_call(), unless the C 960 * function explicitly overwrites values in ix86. Interrupt status 961 * and GDT will also be preserved. Gate A20 will be enabled. 962 * 963 * Note that virt_call() does not rely on the real-mode stack 964 * remaining intact in order to return, since everything relevant is 965 * copied to the protected-mode stack for the duration of the call. 966 * In particular, this means that a real-mode prefix can make a call 967 * to main() which will return correctly even if the prefix's stack 968 * gets vapourised during the Etherboot run. (The prefix cannot rely 969 * on anything else on the stack being preserved, so should move any 970 * critical data to registers before calling main()). 971 * 972 * Parameters: 973 * function : 32-bit virtual address of function to call 974 * 975 * Example usage: 976 * pushl $pxe_api_call 977 * call virt_call 978 * to call in to the C function 979 * void pxe_api_call ( struct i386_all_regs *ix86 ); 980 **************************************************************************** 981 */ 982 .struct 0 983VC_OFFSET_IX86: .space SIZEOF_I386_ALL_REGS 984VC_OFFSET_PADDING: .space 2 /* for alignment */ 985VC_OFFSET_RETADDR: .space 2 986VC_OFFSET_PARAMS: 987VC_OFFSET_FUNCTION: .space 4 988VC_OFFSET_END: 989 .previous 990 991 .section ".text16.virt_call", "ax", @progbits 992 .code16 993 .globl virt_call 994virt_call: 995 /* Preserve registers and flags on external RM stack */ 996 pushw %ss /* padding */ 997 pushfl 998 pushal 999 pushw %gs 1000 pushw %fs 1001 pushw %es 1002 pushw %ds 1003 pushw %ss 1004 pushw %cs 1005 1006 /* Claim ownership of temporary static buffer */ 1007 cli 1008 movw %cs:rm_ds, %ds 1009 1010#ifdef TIVOLI_VMM_WORKAROUND 1011 /* Preserve FPU, MMX and SSE state in temporary static buffer */ 1012 fxsave ( rm_tmpbuf + VC_TMP_FXSAVE ) 1013#endif 1014 /* Preserve GDT and IDT in temporary static buffer */ 1015 sidt ( rm_tmpbuf + VC_TMP_IDT ) 1016 sgdt ( rm_tmpbuf + VC_TMP_GDT ) 1017 1018.if64 ; /* Preserve control registers, if applicable */ 1019 movl $MSR_EFER, %ecx 1020 rdmsr 1021 movl %eax, ( rm_tmpbuf + VC_TMP_EMER + 0 ) 1022 movl %edx, ( rm_tmpbuf + VC_TMP_EMER + 4 ) 1023 movl %cr4, %eax 1024 movl %eax, ( rm_tmpbuf + VC_TMP_CR4 ) 1025 movl %cr3, %eax 1026 movl %eax, ( rm_tmpbuf + VC_TMP_CR3 ) 1027.endif 1028 /* For sanity's sake, clear the direction flag as soon as possible */ 1029 cld 1030 1031 /* Switch to protected mode and move register dump to PM stack */ 1032 movl $VC_OFFSET_END, %ecx 1033 movl $VC_TMP_END, %edx 1034 pushl $VIRTUAL(vc_pmode) 1035vc_jmp: jmp real_to_prot 1036 .section ".text.virt_call", "ax", @progbits 1037 .code32 1038vc_pmode: 1039 /* Call function (in protected mode) */ 1040 pushl %esp 1041 call *(VC_OFFSET_FUNCTION+4)(%esp) 1042 popl %eax /* discard */ 1043 1044.if64 ; /* Switch to long mode */ 1045 jmp 1f 1046vc_lmode: 1047 call prot_to_long 1048 .code64 1049 1050 /* Call function (in long mode) */ 1051 movq %rsp, %rdi 1052 movslq VC_OFFSET_FUNCTION(%rsp), %rax 1053 callq *%rax 1054 1055 /* Switch to protected mode */ 1056 call long_to_prot 10571: .code32 1058.endif 1059 /* Switch to real mode and move register dump back to RM stack */ 1060 movl $VC_OFFSET_END, %ecx 1061 movl $VC_TMP_END, %edx 1062 leal VC_TMP_GDT(%esp, %ecx), %esi 1063 pushl $vc_rmode 1064 jmp prot_to_real 1065 .section ".text16.virt_call", "ax", @progbits 1066 .code16 1067vc_rmode: 1068.if64 ; /* Restore control registers, if applicable */ 1069 movw %sp, %bp 1070 movl ( rm_tmpbuf + VC_TMP_CR3 ), %eax 1071 movl %eax, %cr3 1072 movl ( rm_tmpbuf + VC_TMP_CR4 ), %eax 1073 movl %eax, %cr4 1074 movl ( rm_tmpbuf + VC_TMP_EMER + 0 ), %eax 1075 movl ( rm_tmpbuf + VC_TMP_EMER + 4 ), %edx 1076 movl $MSR_EFER, %ecx 1077 wrmsr 1078.endif 1079 1080#ifdef TIVOLI_VMM_WORKAROUND 1081 /* Restore FPU, MMX and SSE state from temporary static buffer */ 1082 fxrstor ( rm_tmpbuf + VC_TMP_FXSAVE ) 1083#endif 1084 /* Restore registers and flags and return */ 1085 popl %eax /* skip %cs and %ss */ 1086 popw %ds 1087 popw %es 1088 popw %fs 1089 popw %gs 1090 popal 1091 /* popal skips %esp. We therefore want to do "movl -20(%sp), 1092 * %esp", but -20(%sp) is not a valid 80386 expression. 1093 * Fortunately, prot_to_real() zeroes the high word of %esp, so 1094 * we can just use -20(%esp) instead. 1095 */ 1096 addr32 movl -20(%esp), %esp 1097 popfl 1098 popw %ss /* padding */ 1099 1100 /* Return and discard function parameters */ 1101 ret $( VC_OFFSET_END - VC_OFFSET_PARAMS ) 1102 1103 1104 /* Protected-mode jump target */ 1105 .equ vc_jmp_offset, ( vc_jmp - 4 ) 1106 1107/**************************************************************************** 1108 * real_call (protected-mode near call, 32-bit virtual return address) 1109 * real_call (long-mode near call, 64-bit virtual return address) 1110 * 1111 * Call a real-mode function from protected-mode or long-mode code. 1112 * 1113 * The non-segment register values will be passed directly to the 1114 * real-mode code. The segment registers will be set as per 1115 * prot_to_real. The non-segment register values set by the real-mode 1116 * function will be passed back to the protected-mode or long-mode 1117 * caller. A result of this is that this routine cannot be called 1118 * directly from C code, since it clobbers registers that the C ABI 1119 * expects the callee to preserve. 1120 * 1121 * librm.h defines a convenient macro REAL_CODE() for using real_call. 1122 * See librm.h and realmode.h for details and examples. 1123 * 1124 * Parameters: 1125 * function : offset within .text16 of real-mode function to call 1126 * 1127 * Returns: none 1128 **************************************************************************** 1129 */ 1130 .struct 0 1131RC_OFFSET_REGS: .space SIZEOF_I386_REGS 1132RC_OFFSET_REGS_END: 1133RC_OFFSET_FUNCTION_COPY:.space 4 1134.if64 1135RC_OFFSET_LREGS: .space SIZEOF_X86_64_REGS 1136RC_OFFSET_LREG_RETADDR: .space SIZEOF_ADDR 1137.endif 1138RC_OFFSET_RETADDR: .space SIZEOF_ADDR 1139RC_OFFSET_PARAMS: 1140RC_OFFSET_FUNCTION: .space SIZEOF_ADDR 1141RC_OFFSET_END: 1142 .previous 1143 1144 .section ".text.real_call", "ax", @progbits 1145 .CODE_DEFAULT 1146 .globl real_call 1147real_call: 1148.if64 ; /* Preserve registers and switch to protected mode, if applicable */ 1149 call long_preserve_regs 1150 call long_to_prot 1151 .code32 1152.endif 1153 /* Create register dump and function pointer copy on PM stack */ 1154 pushl ( RC_OFFSET_FUNCTION - RC_OFFSET_FUNCTION_COPY - 4 )(%esp) 1155 pushal 1156 1157 /* Switch to real mode and move register dump to RM stack */ 1158 movl $RC_OFFSET_REGS_END, %ecx 1159 movl $RC_TMP_END, %edx 1160 pushl $rc_rmode 1161 movl $VIRTUAL(rm_default_gdtr_idtr), %esi 1162 jmp prot_to_real 1163 .section ".text16.real_call", "ax", @progbits 1164 .code16 1165rc_rmode: 1166 /* Call real-mode function */ 1167 popal 1168 call *( rm_tmpbuf + RC_TMP_FUNCTION ) 1169 pushal 1170 1171 /* For sanity's sake, clear the direction flag as soon as possible */ 1172 cld 1173 1174 /* Switch to protected mode and move register dump back to PM stack */ 1175 movl $RC_OFFSET_REGS_END, %ecx 1176 xorl %edx, %edx 1177 pushl $VIRTUAL(rc_pmode) 1178 jmp real_to_prot 1179 .section ".text.real_call", "ax", @progbits 1180 .code32 1181rc_pmode: 1182 /* Restore registers */ 1183 popal 1184 1185.if64 ; /* Switch to long mode and restore registers, if applicable */ 1186 call prot_to_long 1187 .code64 1188 call long_restore_regs 1189.endif 1190 /* Return and discard function parameters */ 1191 ret $( RC_OFFSET_END - RC_OFFSET_PARAMS ) 1192 1193 1194 /* Default real-mode global and interrupt descriptor table registers */ 1195 .section ".data.rm_default_gdtr_idtr", "aw", @progbits 1196rm_default_gdtr_idtr: 1197 .word 0 /* Global descriptor table limit */ 1198 .long 0 /* Global descriptor table base */ 1199 .word 0x03ff /* Interrupt descriptor table limit */ 1200 .long 0 /* Interrupt descriptor table base */ 1201 1202/**************************************************************************** 1203 * phys_call (protected-mode near call, 32-bit virtual return address) 1204 * phys_call (long-mode near call, 64-bit virtual return address) 1205 * 1206 * Call a function with flat 32-bit physical addressing 1207 * 1208 * The non-segment register values will be passed directly to the 1209 * function. The segment registers will be set for flat 32-bit 1210 * physical addressing. The non-segment register values set by the 1211 * function will be passed back to the caller. 1212 * 1213 * librm.h defines a convenient macro PHYS_CODE() for using phys_call. 1214 * 1215 * Parameters: 1216 * function : virtual (sic) address of function to call 1217 * 1218 **************************************************************************** 1219 */ 1220 .struct 0 1221.if64 1222PHC_OFFSET_LREGS: .space SIZEOF_X86_64_REGS 1223PHC_OFFSET_LREG_RETADDR:.space SIZEOF_ADDR 1224.endif 1225PHC_OFFSET_RETADDR: .space SIZEOF_ADDR 1226PHC_OFFSET_PARAMS: 1227PHC_OFFSET_FUNCTION: .space SIZEOF_ADDR 1228PHC_OFFSET_END: 1229 .previous 1230 1231 .section ".text.phys_call", "ax", @progbits 1232 .CODE_DEFAULT 1233 .globl phys_call 1234phys_call: 1235.if64 ; /* Preserve registers and switch to protected mode, if applicable */ 1236 call long_preserve_regs 1237 call long_to_prot 1238 .code32 1239.endif 1240 /* Adjust function pointer to a physical address */ 1241 pushl %ebp 1242 movl VIRTUAL(virt_offset), %ebp 1243 addl %ebp, ( PHC_OFFSET_FUNCTION + 4 /* saved %ebp */ )(%esp) 1244 popl %ebp 1245 1246 /* Switch to physical addresses */ 1247 call prot_to_phys 1248 1249 /* Call function */ 1250 call *PHC_OFFSET_FUNCTION(%esp) 1251 1252 /* For sanity's sake, clear the direction flag as soon as possible */ 1253 cld 1254 1255 /* Switch to virtual addresses */ 1256 call phys_to_prot 1257 1258.if64 ; /* Switch to long mode and restore registers, if applicable */ 1259 call prot_to_long 1260 .code64 1261 call long_restore_regs 1262.endif 1263 /* Return and discard function parameters */ 1264 ret $( PHC_OFFSET_END - PHC_OFFSET_PARAMS ) 1265 1266/**************************************************************************** 1267 * phys_to_long (protected-mode near call, 32-bit physical return address) 1268 * 1269 * Used by COMBOOT. 1270 * 1271 **************************************************************************** 1272 */ 1273 .if64 1274 1275 .section ".text.phys_to_long", "ax", @progbits 1276 .code32 1277phys_to_long: 1278 1279 /* Switch to virtual addresses */ 1280 call phys_to_prot 1281 1282 /* Convert to 32-bit virtual return address */ 1283 pushl %eax 1284 movl VIRTUAL(virt_offset), %eax 1285 subl %eax, 4(%esp) 1286 popl %eax 1287 1288 /* Switch to long mode and return */ 1289 jmp prot_to_long 1290 1291 /* Expose as _phys_to_virt for use by COMBOOT */ 1292 .globl _phys_to_virt 1293 .equ _phys_to_virt, phys_to_long 1294 1295 .endif 1296 1297/**************************************************************************** 1298 * long_to_phys (long-mode near call, 64-bit virtual return address) 1299 * 1300 * Used by COMBOOT. 1301 * 1302 **************************************************************************** 1303 */ 1304 .if64 1305 1306 .section ".text.long_to_phys", "ax", @progbits 1307 .code64 1308long_to_phys: 1309 1310 /* Switch to protected mode */ 1311 call long_to_prot 1312 .code32 1313 1314 /* Convert to 32-bit virtual return address */ 1315 popl (%esp) 1316 1317 /* Switch to physical addresses and return */ 1318 jmp prot_to_phys 1319 1320 /* Expose as _virt_to_phys for use by COMBOOT */ 1321 .globl _virt_to_phys 1322 .equ _virt_to_phys, long_to_phys 1323 1324 .endif 1325 1326/**************************************************************************** 1327 * flatten_real_mode (real-mode near call) 1328 * 1329 * Switch to flat real mode 1330 * 1331 **************************************************************************** 1332 */ 1333 .section ".text16.flatten_real_mode", "ax", @progbits 1334 .code16 1335 .globl flatten_real_mode 1336flatten_real_mode: 1337 /* Modify GDT to use flat real mode */ 1338 movb $0x8f, real_cs + 6 1339 movb $0x8f, real_ds + 6 1340 /* Call dummy protected-mode function */ 1341 virtcall flatten_dummy 1342 /* Restore GDT */ 1343 movb $0x00, real_cs + 6 1344 movb $0x00, real_ds + 6 1345 /* Return */ 1346 ret 1347 1348 .section ".text.flatten_dummy", "ax", @progbits 1349 .CODE_DEFAULT 1350flatten_dummy: 1351 ret 1352 1353/**************************************************************************** 1354 * Interrupt wrapper 1355 * 1356 * Used by the protected-mode and long-mode interrupt vectors to call 1357 * the interrupt() function. 1358 * 1359 * May be entered with either physical or virtual stack segment. 1360 **************************************************************************** 1361 */ 1362 .section ".text.interrupt_wrapper", "ax", @progbits 1363 .code32 1364 .globl interrupt_wrapper 1365interrupt_wrapper: 1366 /* Preserve registers (excluding already-saved %eax) */ 1367 pushl %ebx 1368 pushl %ecx 1369 pushl %edx 1370 pushl %esi 1371 pushl %edi 1372 pushl %ebp 1373 1374 /* Expand IRQ number to whole %eax register */ 1375 movzbl %al, %eax 1376 1377.if64 ; /* Skip transition to long mode, if applicable */ 1378 xorl %edx, %edx 1379 movw %cs, %bx 1380 cmpw $LONG_CS, %bx 1381 je 1f 1382.endif 1383 /* Preserve segment registers and original %esp */ 1384 pushl %ds 1385 pushl %es 1386 pushl %fs 1387 pushl %gs 1388 pushl %ss 1389 pushl %esp 1390 1391 /* Switch to virtual addressing */ 1392 call intr_to_prot 1393 1394 /* Pass 32-bit interrupt frame pointer in %edx */ 1395 movl %esp, %edx 1396 xorl %ecx, %ecx 1397.if64 1398 /* Switch to long mode */ 1399 call prot_to_long 1400 .code64 1401 14021: /* Preserve long-mode registers */ 1403 pushq %r8 1404 pushq %r9 1405 pushq %r10 1406 pushq %r11 1407 pushq %r12 1408 pushq %r13 1409 pushq %r14 1410 pushq %r15 1411 1412 /* Expand IRQ number to whole %rdi register */ 1413 movl %eax, %edi 1414 1415 /* Pass 32-bit interrupt frame pointer (if applicable) in %rsi */ 1416 testl %edx, %edx 1417 je 1f 1418 movl %edx, %esi 1419 addl virt_offset, %esi 14201: 1421 /* Pass 64-bit interrupt frame pointer in %rdx */ 1422 movq %rsp, %rdx 1423.endif 1424 /* Call interrupt handler */ 1425 call interrupt 1426.if64 1427 /* Restore long-mode registers */ 1428 popq %r15 1429 popq %r14 1430 popq %r13 1431 popq %r12 1432 popq %r11 1433 popq %r10 1434 popq %r9 1435 popq %r8 1436 1437 /* Skip transition back to protected mode, if applicable */ 1438 cmpw $LONG_CS, %bx 1439 je 1f 1440 1441 /* Switch to protected mode */ 1442 call long_to_prot 1443 .code32 1444 cmpw $LONG_CS, %bx 1445.endif 1446 /* Restore segment registers and original %esp */ 1447 lss (%esp), %esp 1448 popl %ss 1449 popl %gs 1450 popl %fs 1451 popl %es 1452 popl %ds 1453 14541: /* Restore registers */ 1455 popl %ebp 1456 popl %edi 1457 popl %esi 1458 popl %edx 1459 popl %ecx 1460 popl %ebx 1461 popl %eax 1462 1463 /* Return from interrupt (with REX prefix if required) */ 1464.if64 ; jne 1f ; .byte 0x48 ; .endif 14651: iret 1466 1467/**************************************************************************** 1468 * Page tables 1469 * 1470 **************************************************************************** 1471 */ 1472 .section ".pages", "aw", @nobits 1473 .align SIZEOF_PT 1474 1475 /* Page map level 4 entries (PML4Es) 1476 * 1477 * This comprises 1478 * 1479 * - PML4E[0x000] covering [0x0000000000000000-0x0000007fffffffff] 1480 * - PML4E[0x1ff] covering [0xffffff8000000000-0xffffffffffffffff] 1481 * 1482 * These point to the PDPT. This creates some aliased 1483 * addresses within unused portions of the 64-bit address 1484 * space, but allows us to use just a single PDPT. 1485 * 1486 * - PDE[...] covering arbitrary 2MB portions of I/O space 1487 * 1488 * These are 2MB pages created by ioremap() to cover I/O 1489 * device addresses. 1490 */ 1491pml4e: 1492 .space SIZEOF_PT 1493 .size pml4e, . - pml4e 1494 1495 .globl io_pages 1496 .equ io_pages, pml4e 1497 1498 /* Page directory pointer table entries (PDPTEs) 1499 * 1500 * This comprises: 1501 * 1502 * - PDPTE[0x000] covering [0x0000000000000000-0x000000003fffffff] 1503 * - PDPTE[0x001] covering [0x0000000040000000-0x000000007fffffff] 1504 * - PDPTE[0x002] covering [0x0000000080000000-0x00000000bfffffff] 1505 * - PDPTE[0x003] covering [0x00000000c0000000-0x00000000ffffffff] 1506 * 1507 * These point to the appropriate page directories (in pde_low) 1508 * used to identity-map the whole of the 32-bit address space. 1509 * 1510 * - PDPTE[0x004] covering [0x0000000100000000-0x000000013fffffff] 1511 * 1512 * This points back to the PML4, allowing the PML4 to be 1513 * (ab)used to hold 2MB pages used for I/O device addresses. 1514 * 1515 * - PDPTE[0x1ff] covering [0xffffffffc0000000-0xffffffffffffffff] 1516 * 1517 * This points back to the PDPT itself, allowing the PDPT to be 1518 * (ab)used to hold PDEs covering .textdata. 1519 * 1520 * - PDE[N-M] covering [_textdata,_end) 1521 * 1522 * These are used to point to the page tables (in pte_textdata) 1523 * used to map our .textdata section. Note that each PDE 1524 * covers 2MB, so we are likely to use only a single PDE in 1525 * practice. 1526 */ 1527pdpte: 1528 .space SIZEOF_PT 1529 .size pdpte, . - pdpte 1530 .equ pde_textdata, pdpte /* (ab)use */ 1531 1532 /* Page directory entries (PDEs) for the low 4GB 1533 * 1534 * This comprises 2048 2MB pages to identity-map the whole of 1535 * the 32-bit address space. 1536 */ 1537pde_low: 1538 .equ PDE_LOW_PTES, ( SIZEOF_LOW_4GB / SIZEOF_2MB_PAGE ) 1539 .equ PDE_LOW_PTS, ( ( PDE_LOW_PTES * SIZEOF_PTE ) / SIZEOF_PT ) 1540 .space ( PDE_LOW_PTS * SIZEOF_PT ) 1541 .size pde_low, . - pde_low 1542 1543 /* Page table entries (PTEs) for .textdata 1544 * 1545 * This comprises enough 4kB pages to map the whole of 1546 * .textdata. The required number of PTEs is calculated by 1547 * the linker script. 1548 * 1549 * Note that these mappings do not cover the PTEs themselves. 1550 * This does not matter, since code running with paging 1551 * enabled never needs to access these PTEs. 1552 */ 1553pte_textdata: 1554 /* Allocated by linker script; must be at the end of .textdata */ 1555 1556 .section ".bss.pml4", "aw", @nobits 1557pml4: .long 0 1558 1559/**************************************************************************** 1560 * init_pages (protected-mode near call) 1561 * 1562 * Initialise the page tables ready for long mode. 1563 * 1564 * Parameters: 1565 * %edi : virt_offset 1566 **************************************************************************** 1567 */ 1568 .section ".text.init_pages", "ax", @progbits 1569 .code32 1570init_pages: 1571 /* Initialise PML4Es for low 4GB and negative 2GB */ 1572 leal ( VIRTUAL(pdpte) + ( PG_P | PG_RW | PG_US ) )(%edi), %eax 1573 movl %eax, VIRTUAL(pml4e) 1574 movl %eax, ( VIRTUAL(pml4e) + SIZEOF_PT - SIZEOF_PTE ) 1575 1576 /* Initialise PDPTE for negative 1GB */ 1577 movl %eax, ( VIRTUAL(pdpte) + SIZEOF_PT - SIZEOF_PTE ) 1578 1579 /* Initialise PDPTE for I/O space */ 1580 leal ( VIRTUAL(pml4e) + ( PG_P | PG_RW | PG_US ) )(%edi), %eax 1581 movl %eax, ( VIRTUAL(pdpte) + ( PDE_LOW_PTS * SIZEOF_PTE ) ) 1582 1583 /* Initialise PDPTEs for low 4GB */ 1584 movl $PDE_LOW_PTS, %ecx 1585 leal ( VIRTUAL(pde_low) + ( PDE_LOW_PTS * SIZEOF_PT ) + \ 1586 ( PG_P | PG_RW | PG_US ) )(%edi), %eax 15871: subl $SIZEOF_PT, %eax 1588 movl %eax, ( VIRTUAL(pdpte) - SIZEOF_PTE )(,%ecx,SIZEOF_PTE) 1589 loop 1b 1590 1591 /* Initialise PDEs for low 4GB */ 1592 movl $PDE_LOW_PTES, %ecx 1593 leal ( 0 + ( PG_P | PG_RW | PG_US | PG_PS ) ), %eax 15941: subl $SIZEOF_2MB_PAGE, %eax 1595 movl %eax, ( VIRTUAL(pde_low) - SIZEOF_PTE )(,%ecx,SIZEOF_PTE) 1596 loop 1b 1597 1598 /* Initialise PDEs for .textdata */ 1599 movl $_textdata_pdes, %ecx 1600 leal ( VIRTUAL(_etextdata) + ( PG_P | PG_RW | PG_US ) )(%edi), %eax 1601 movl $VIRTUAL(_textdata), %ebx 1602 shrl $( SIZEOF_2MB_PAGE_LOG2 - SIZEOF_PTE_LOG2 ), %ebx 1603 andl $( SIZEOF_PT - 1 ), %ebx 16041: subl $SIZEOF_PT, %eax 1605 movl %eax, (VIRTUAL(pde_textdata) - SIZEOF_PTE)(%ebx,%ecx,SIZEOF_PTE) 1606 loop 1b 1607 1608 /* Initialise PTEs for .textdata */ 1609 movl $_textdata_ptes, %ecx 1610 leal ( VIRTUAL(_textdata) + ( PG_P | PG_RW | PG_US ) )(%edi), %eax 1611 addl $_textdata_paged_len, %eax 16121: subl $SIZEOF_4KB_PAGE, %eax 1613 movl %eax, ( VIRTUAL(pte_textdata) - SIZEOF_PTE )(,%ecx,SIZEOF_PTE) 1614 loop 1b 1615 1616 /* Record PML4 physical address */ 1617 leal VIRTUAL(pml4e)(%edi), %eax 1618 movl %eax, VIRTUAL(pml4) 1619 1620 /* Return */ 1621 ret 1622