1 #pragma once 2 3 #ifndef __ASM__ 4 5 #include "intrin_i.h" 6 7 #ifdef __cplusplus 8 extern "C" 9 { 10 #endif 11 12 // 13 // Thread Dispatcher Header DebugActive Mask 14 // 15 #define DR_MASK(x) (1 << (x)) 16 #define DR_REG_MASK 0x4F 17 18 // 19 // INT3 is 1 byte long 20 // 21 #define KD_BREAKPOINT_TYPE UCHAR 22 #define KD_BREAKPOINT_SIZE sizeof(UCHAR) 23 #define KD_BREAKPOINT_VALUE 0xCC 24 25 /* CPUID 1 - EDX flags */ 26 #define X86_FEATURE_FPU 0x00000001 /* x87 FPU is present */ 27 #define X86_FEATURE_VME 0x00000002 /* Virtual 8086 Extensions are present */ 28 #define X86_FEATURE_DBG 0x00000004 /* Debugging extensions are present */ 29 #define X86_FEATURE_PSE 0x00000008 /* Page Size Extension is present */ 30 #define X86_FEATURE_TSC 0x00000010 /* Time Stamp Counters are present */ 31 #define X86_FEATURE_PAE 0x00000040 /* Physical Address Extension is present */ 32 #define X86_FEATURE_CX8 0x00000100 /* CMPXCHG8B instruction present */ 33 #define X86_FEATURE_APIC 0x00000200 /* APIC is present */ 34 #define X86_FEATURE_SYSCALL 0x00000800 /* SYSCALL/SYSRET support present */ 35 #define X86_FEATURE_MTTR 0x00001000 /* Memory type range registers are present */ 36 #define X86_FEATURE_PGE 0x00002000 /* Page Global Enable */ 37 #define X86_FEATURE_CMOV 0x00008000 /* "Conditional move" instruction supported */ 38 #define X86_FEATURE_PAT 0x00010000 /* Page Attribute Table is supported */ 39 #define X86_FEATURE_DS 0x00200000 /* Debug Store is present */ 40 #define X86_FEATURE_MMX 0x00800000 /* MMX extension present */ 41 #define X86_FEATURE_FXSR 0x01000000 /* FXSAVE/FXRSTOR instructions present */ 42 #define X86_FEATURE_SSE 0x02000000 /* SSE extension present */ 43 #define X86_FEATURE_SSE2 0x04000000 /* SSE2 extension present */ 44 #define X86_FEATURE_HT 0x10000000 /* Hyper-Threading present */ 45 46 /* CPUID 0x80000001 - EDX extended flags */ 47 #define X86_FEATURE_NX 0x00100000 /* NX support present */ 48 49 // 50 // One-liners for getting and setting special purpose registers in portable code 51 // 52 FORCEINLINE 53 ULONG_PTR 54 KeGetContextPc(PCONTEXT Context) 55 { 56 return Context->Eip; 57 } 58 59 FORCEINLINE 60 VOID 61 KeSetContextPc(PCONTEXT Context, ULONG_PTR ProgramCounter) 62 { 63 Context->Eip = ProgramCounter; 64 } 65 66 FORCEINLINE 67 ULONG_PTR 68 KeGetContextReturnRegister(PCONTEXT Context) 69 { 70 return Context->Eax; 71 } 72 73 FORCEINLINE 74 VOID 75 KeSetContextReturnRegister(PCONTEXT Context, ULONG_PTR ReturnValue) 76 { 77 Context->Eax = ReturnValue; 78 } 79 80 FORCEINLINE 81 ULONG_PTR 82 KeGetContextFrameRegister(PCONTEXT Context) 83 { 84 return Context->Ebp; 85 } 86 87 FORCEINLINE 88 VOID 89 KeSetContextFrameRegister(PCONTEXT Context, ULONG_PTR Frame) 90 { 91 Context->Ebp = Frame; 92 } 93 94 FORCEINLINE 95 ULONG_PTR 96 KeGetTrapFramePc(PKTRAP_FRAME TrapFrame) 97 { 98 return TrapFrame->Eip; 99 } 100 101 FORCEINLINE 102 PKTRAP_FRAME 103 KiGetLinkedTrapFrame(PKTRAP_FRAME TrapFrame) 104 { 105 return (PKTRAP_FRAME)TrapFrame->Edx; 106 } 107 108 109 FORCEINLINE 110 ULONG_PTR 111 KeGetTrapFrameStackRegister(PKTRAP_FRAME TrapFrame) 112 { 113 if (TrapFrame->PreviousPreviousMode == KernelMode) 114 return TrapFrame->TempEsp; 115 return TrapFrame->HardwareEsp; 116 } 117 118 FORCEINLINE 119 ULONG_PTR 120 KeGetTrapFrameFrameRegister(PKTRAP_FRAME TrapFrame) 121 { 122 return TrapFrame->Ebp; 123 } 124 125 // 126 // Macro to get trap and exception frame from a thread stack 127 // 128 #define KeGetTrapFrame(Thread) \ 129 (PKTRAP_FRAME)((ULONG_PTR)((Thread)->InitialStack) - \ 130 sizeof(KTRAP_FRAME) - \ 131 sizeof(FX_SAVE_AREA)) 132 133 #define KeGetExceptionFrame(Thread) \ 134 NULL 135 136 // 137 // Macro to get context switches from the PRCB 138 // All architectures but x86 have it in the PRCB's KeContextSwitches 139 // 140 #define KeGetContextSwitches(Prcb) \ 141 CONTAINING_RECORD(Prcb, KIPCR, PrcbData)->ContextSwitches 142 143 // 144 // Macro to get the second level cache size field name which differs between 145 // CISC and RISC architectures, as the former has unified I/D cache 146 // 147 #define KiGetSecondLevelDCacheSize() ((PKIPCR)KeGetPcr())->SecondLevelCacheSize 148 149 // 150 // Returns the Interrupt State from a Trap Frame. 151 // ON = TRUE, OFF = FALSE 152 // 153 #define KeGetTrapFrameInterruptState(TrapFrame) \ 154 BooleanFlagOn((TrapFrame)->EFlags, EFLAGS_INTERRUPT_MASK) 155 156 // 157 // Flags for exiting a trap 158 // 159 #define KTE_SKIP_PM_BIT (((KTRAP_EXIT_SKIP_BITS) { { .SkipPreviousMode = TRUE } }).Bits) 160 #define KTE_SKIP_SEG_BIT (((KTRAP_EXIT_SKIP_BITS) { { .SkipSegments = TRUE } }).Bits) 161 #define KTE_SKIP_VOL_BIT (((KTRAP_EXIT_SKIP_BITS) { { .SkipVolatiles = TRUE } }).Bits) 162 163 typedef union _KTRAP_EXIT_SKIP_BITS 164 { 165 struct 166 { 167 UCHAR SkipPreviousMode:1; 168 UCHAR SkipSegments:1; 169 UCHAR SkipVolatiles:1; 170 UCHAR Reserved:5; 171 }; 172 UCHAR Bits; 173 } KTRAP_EXIT_SKIP_BITS, *PKTRAP_EXIT_SKIP_BITS; 174 175 176 // 177 // Flags used by the VDM/V8086 emulation engine for determining instruction prefixes 178 // 179 #define PFX_FLAG_ES 0x00000100 180 #define PFX_FLAG_CS 0x00000200 181 #define PFX_FLAG_SS 0x00000400 182 #define PFX_FLAG_DS 0x00000800 183 #define PFX_FLAG_FS 0x00001000 184 #define PFX_FLAG_GS 0x00002000 185 #define PFX_FLAG_OPER32 0x00004000 186 #define PFX_FLAG_ADDR32 0x00008000 187 #define PFX_FLAG_LOCK 0x00010000 188 #define PFX_FLAG_REPNE 0x00020000 189 #define PFX_FLAG_REP 0x00040000 190 191 // 192 // VDM Helper Macros 193 // 194 // All VDM/V8086 opcode emulators have the same FASTCALL function definition. 195 // We need to keep 2 parameters while the original ASM implementation uses 4: 196 // TrapFrame, PrefixFlags, Eip, InstructionSize; 197 // 198 // We pass the trap frame, and prefix flags, in our two parameters. 199 // 200 // We then realize that since the smallest prefix flag is 0x100, this gives us 201 // a count of up to 0xFF. So we OR in the instruction size with the prefix flags 202 // 203 // We further realize that we always have access to EIP from the trap frame, and 204 // that if we want the *current instruction* EIP, we simply have to add the 205 // instruction size *MINUS ONE*, and that gives us the EIP we should be looking 206 // at now, so we don't need to use the stack to push this parameter. 207 // 208 // We actually only care about the *current instruction* EIP in one location, 209 // so although it may be slightly more expensive to re-calculate the EIP one 210 // more time, this way we don't redefine ALL opcode handlers to have 3 parameters, 211 // which would be forcing stack usage in all other scenarios. 212 // 213 #define KiVdmSetVdmEFlags(x) InterlockedOr((PLONG)KiNtVdmState, (x)); 214 #define KiVdmClearVdmEFlags(x) InterlockedAnd((PLONG)KiNtVdmState, ~(x)) 215 #define KiCallVdmHandler(x) KiVdmOpcode##x(TrapFrame, Flags) 216 #define KiCallVdmPrefixHandler(x) KiVdmOpcodePrefix(TrapFrame, Flags | x) 217 #define KiVdmUnhandledOpcode(x) \ 218 BOOLEAN \ 219 FASTCALL \ 220 KiVdmOpcode##x(IN PKTRAP_FRAME TrapFrame, \ 221 IN ULONG Flags) \ 222 { \ 223 /* Not yet handled */ \ 224 UNIMPLEMENTED_DBGBREAK(); \ 225 return FALSE; \ 226 } 227 228 C_ASSERT(NPX_FRAME_LENGTH == sizeof(FX_SAVE_AREA)); 229 230 // 231 // Local parameters 232 // 233 typedef struct _KV86_FRAME 234 { 235 PVOID ThreadStack; 236 PVOID ThreadTeb; 237 PVOID PcrTeb; 238 } KV86_FRAME, *PKV86_FRAME; 239 240 // 241 // Virtual Stack Frame 242 // 243 typedef struct _KV8086_STACK_FRAME 244 { 245 KTRAP_FRAME TrapFrame; 246 FX_SAVE_AREA NpxArea; 247 KV86_FRAME V86Frame; 248 } KV8086_STACK_FRAME, *PKV8086_STACK_FRAME; 249 250 // 251 // Large Pages Support 252 // 253 typedef struct _LARGE_IDENTITY_MAP 254 { 255 PHARDWARE_PTE TopLevelDirectory; 256 ULONG Cr3; 257 ULONG_PTR StartAddress; 258 ULONG PagesCount; 259 PVOID PagesList[30]; 260 } LARGE_IDENTITY_MAP, *PLARGE_IDENTITY_MAP; 261 262 // 263 // Floating Point Internal Context Structure 264 // 265 typedef struct _FLOATING_SAVE_CONTEXT 266 { 267 PKTHREAD CurrentThread; 268 KIRQL OldNpxIrql; 269 PFX_SAVE_AREA Buffer; 270 PFX_SAVE_AREA PfxSaveArea; 271 } FLOATING_SAVE_CONTEXT, *PFLOATING_SAVE_CONTEXT; 272 273 /* Diable interrupts and return whether they were enabled before */ 274 FORCEINLINE 275 BOOLEAN 276 KeDisableInterrupts(VOID) 277 { 278 ULONG Flags; 279 BOOLEAN Return; 280 281 /* Get EFLAGS and check if the interrupt bit is set */ 282 Flags = __readeflags(); 283 Return = (Flags & EFLAGS_INTERRUPT_MASK) ? TRUE: FALSE; 284 285 /* Disable interrupts */ 286 _disable(); 287 return Return; 288 } 289 290 /* Restore previous interrupt state */ 291 FORCEINLINE 292 VOID 293 KeRestoreInterrupts(BOOLEAN WereEnabled) 294 { 295 if (WereEnabled) _enable(); 296 } 297 298 // 299 // Registers an interrupt handler with an IDT vector 300 // 301 FORCEINLINE 302 VOID 303 KeRegisterInterruptHandler(IN ULONG Vector, 304 IN PVOID Handler) 305 { 306 UCHAR Entry; 307 ULONG_PTR Address; 308 PKIPCR Pcr = (PKIPCR)KeGetPcr(); 309 310 // 311 // Get the entry from the HAL 312 // 313 Entry = HalVectorToIDTEntry(Vector); 314 Address = PtrToUlong(Handler); 315 316 // 317 // Now set the data 318 // 319 Pcr->IDT[Entry].ExtendedOffset = (USHORT)(Address >> 16); 320 Pcr->IDT[Entry].Offset = (USHORT)Address; 321 } 322 323 // 324 // Returns the registered interrupt handler for a given IDT vector 325 // 326 FORCEINLINE 327 PVOID 328 KeQueryInterruptHandler(IN ULONG Vector) 329 { 330 PKIPCR Pcr = (PKIPCR)KeGetPcr(); 331 UCHAR Entry; 332 333 // 334 // Get the entry from the HAL 335 // 336 Entry = HalVectorToIDTEntry(Vector); 337 338 // 339 // Read the entry from the IDT 340 // 341 return (PVOID)(((Pcr->IDT[Entry].ExtendedOffset << 16) & 0xFFFF0000) | 342 (Pcr->IDT[Entry].Offset & 0xFFFF)); 343 } 344 345 // 346 // Invalidates the TLB entry for a specified address 347 // 348 FORCEINLINE 349 VOID 350 KeInvalidateTlbEntry(IN PVOID Address) 351 { 352 /* Invalidate the TLB entry for this address */ 353 __invlpg(Address); 354 } 355 356 FORCEINLINE 357 VOID 358 KeFlushProcessTb(VOID) 359 { 360 /* Flush the TLB by resetting CR3 */ 361 __writecr3(__readcr3()); 362 } 363 364 FORCEINLINE 365 VOID 366 KeSweepICache(IN PVOID BaseAddress, 367 IN SIZE_T FlushSize) 368 { 369 // 370 // Always sweep the whole cache 371 // 372 UNREFERENCED_PARAMETER(BaseAddress); 373 UNREFERENCED_PARAMETER(FlushSize); 374 __wbinvd(); 375 } 376 377 FORCEINLINE 378 PRKTHREAD 379 KeGetCurrentThread(VOID) 380 { 381 /* Return the current thread */ 382 return ((PKIPCR)KeGetPcr())->PrcbData.CurrentThread; 383 } 384 385 FORCEINLINE 386 VOID 387 KiRundownThread(IN PKTHREAD Thread) 388 { 389 #ifndef CONFIG_SMP 390 /* Check if this is the NPX Thread */ 391 if (KeGetCurrentPrcb()->NpxThread == Thread) 392 { 393 /* Clear it */ 394 KeGetCurrentPrcb()->NpxThread = NULL; 395 Ke386FnInit(); 396 } 397 #else 398 /* Nothing to do */ 399 #endif 400 } 401 402 CODE_SEG("INIT") 403 VOID 404 NTAPI 405 KiInitializePcr(IN ULONG ProcessorNumber, 406 IN PKIPCR Pcr, 407 IN PKIDTENTRY Idt, 408 IN PKGDTENTRY Gdt, 409 IN PKTSS Tss, 410 IN PKTHREAD IdleThread, 411 IN PVOID DpcStack); 412 413 FORCEINLINE 414 VOID 415 Ke386SetGdtEntryBase(PKGDTENTRY GdtEntry, PVOID BaseAddress) 416 { 417 GdtEntry->BaseLow = (USHORT)((ULONG_PTR)BaseAddress & 0xFFFF); 418 GdtEntry->HighWord.Bytes.BaseMid = (UCHAR)((ULONG_PTR)BaseAddress >> 16); 419 GdtEntry->HighWord.Bytes.BaseHi = (UCHAR)((ULONG_PTR)BaseAddress >> 24); 420 } 421 422 FORCEINLINE 423 VOID 424 KiSetTebBase(PKPCR Pcr, PNT_TIB TebAddress) 425 { 426 Pcr->NtTib.Self = TebAddress; 427 Ke386SetGdtEntryBase(&Pcr->GDT[KGDT_R3_TEB / sizeof(KGDTENTRY)], TebAddress); 428 } 429 430 CODE_SEG("INIT") 431 VOID 432 FASTCALL 433 Ki386InitializeTss( 434 IN PKTSS Tss, 435 IN PKIDTENTRY Idt, 436 IN PKGDTENTRY Gdt 437 ); 438 439 CODE_SEG("INIT") 440 VOID 441 NTAPI 442 KiSetCR0Bits(VOID); 443 444 CODE_SEG("INIT") 445 VOID 446 NTAPI 447 KiGetCacheInformation(VOID); 448 449 CODE_SEG("INIT") 450 BOOLEAN 451 NTAPI 452 KiIsNpxPresent( 453 VOID 454 ); 455 456 CODE_SEG("INIT") 457 BOOLEAN 458 NTAPI 459 KiIsNpxErrataPresent( 460 VOID 461 ); 462 463 CODE_SEG("INIT") 464 VOID 465 NTAPI 466 KiSetProcessorType(VOID); 467 468 CODE_SEG("INIT") 469 ULONG 470 NTAPI 471 KiGetFeatureBits(VOID); 472 473 #if DBG 474 CODE_SEG("INIT") 475 VOID 476 KiReportCpuFeatures(VOID); 477 #endif 478 479 VOID 480 NTAPI 481 KiThreadStartup(VOID); 482 483 NTSTATUS 484 NTAPI 485 Ke386GetGdtEntryThread( 486 IN PKTHREAD Thread, 487 IN ULONG Offset, 488 IN PKGDTENTRY Descriptor 489 ); 490 491 VOID 492 NTAPI 493 KiFlushNPXState( 494 IN FLOATING_SAVE_AREA *SaveArea 495 ); 496 497 VOID 498 NTAPI 499 Ki386AdjustEsp0( 500 IN PKTRAP_FRAME TrapFrame 501 ); 502 503 VOID 504 NTAPI 505 Ki386SetupAndExitToV86Mode( 506 OUT PTEB VdmTeb 507 ); 508 509 CODE_SEG("INIT") 510 VOID 511 NTAPI 512 KeI386VdmInitialize( 513 VOID 514 ); 515 516 CODE_SEG("INIT") 517 ULONG_PTR 518 NTAPI 519 Ki386EnableGlobalPage( 520 IN ULONG_PTR Context 521 ); 522 523 CODE_SEG("INIT") 524 ULONG_PTR 525 NTAPI 526 Ki386EnableTargetLargePage( 527 IN ULONG_PTR Context 528 ); 529 530 BOOLEAN 531 NTAPI 532 Ki386CreateIdentityMap( 533 IN PLARGE_IDENTITY_MAP IdentityMap, 534 IN PVOID StartPtr, 535 IN ULONG Length 536 ); 537 538 VOID 539 NTAPI 540 Ki386FreeIdentityMap( 541 IN PLARGE_IDENTITY_MAP IdentityMap 542 ); 543 544 VOID 545 NTAPI 546 Ki386EnableCurrentLargePage( 547 IN ULONG_PTR StartAddress, 548 IN ULONG Cr3 549 ); 550 551 CODE_SEG("INIT") 552 VOID 553 NTAPI 554 KiI386PentiumLockErrataFixup( 555 VOID 556 ); 557 558 CODE_SEG("INIT") 559 VOID 560 NTAPI 561 KiInitializePAT( 562 VOID 563 ); 564 565 CODE_SEG("INIT") 566 VOID 567 NTAPI 568 KiInitializeMTRR( 569 IN BOOLEAN FinalCpu 570 ); 571 572 CODE_SEG("INIT") 573 VOID 574 NTAPI 575 KiAmdK6InitializeMTRR( 576 VOID 577 ); 578 579 CODE_SEG("INIT") 580 VOID 581 NTAPI 582 KiRestoreFastSyscallReturnState( 583 VOID 584 ); 585 586 CODE_SEG("INIT") 587 ULONG_PTR 588 NTAPI 589 Ki386EnableDE( 590 IN ULONG_PTR Context 591 ); 592 593 CODE_SEG("INIT") 594 ULONG_PTR 595 NTAPI 596 Ki386EnableFxsr( 597 IN ULONG_PTR Context 598 ); 599 600 CODE_SEG("INIT") 601 ULONG_PTR 602 NTAPI 603 Ki386EnableXMMIExceptions( 604 IN ULONG_PTR Context 605 ); 606 607 BOOLEAN 608 NTAPI 609 VdmDispatchBop( 610 IN PKTRAP_FRAME TrapFrame 611 ); 612 613 BOOLEAN 614 NTAPI 615 VdmDispatchPageFault( 616 _In_ PKTRAP_FRAME TrapFrame 617 ); 618 619 BOOLEAN 620 FASTCALL 621 KiVdmOpcodePrefix( 622 IN PKTRAP_FRAME TrapFrame, 623 IN ULONG Flags 624 ); 625 626 BOOLEAN 627 FASTCALL 628 Ki386HandleOpcodeV86( 629 IN PKTRAP_FRAME TrapFrame 630 ); 631 632 DECLSPEC_NORETURN 633 VOID 634 FASTCALL 635 KiEoiHelper( 636 IN PKTRAP_FRAME TrapFrame 637 ); 638 639 VOID 640 FASTCALL 641 Ki386BiosCallReturnAddress( 642 IN PKTRAP_FRAME TrapFrame 643 ); 644 645 ULONG_PTR 646 FASTCALL 647 KiExitV86Mode( 648 IN PKTRAP_FRAME TrapFrame 649 ); 650 651 DECLSPEC_NORETURN 652 VOID 653 NTAPI 654 KiDispatchExceptionFromTrapFrame( 655 IN NTSTATUS Code, 656 IN ULONG Flags, 657 IN ULONG_PTR Address, 658 IN ULONG ParameterCount, 659 IN ULONG_PTR Parameter1, 660 IN ULONG_PTR Parameter2, 661 IN ULONG_PTR Parameter3, 662 IN PKTRAP_FRAME TrapFrame 663 ); 664 665 NTSTATUS 666 NTAPI 667 KiConvertToGuiThread( 668 VOID 669 ); 670 671 DECLSPEC_NORETURN 672 VOID 673 FASTCALL 674 KiServiceExit( 675 IN PKTRAP_FRAME TrapFrame, 676 IN NTSTATUS Status 677 ); 678 679 DECLSPEC_NORETURN 680 VOID 681 FASTCALL 682 KiServiceExit2( 683 IN PKTRAP_FRAME TrapFrame 684 ); 685 686 FORCEINLINE 687 DECLSPEC_NORETURN 688 VOID 689 KiExceptionExit( 690 _In_ PKTRAP_FRAME TrapFrame, 691 _In_ PKEXCEPTION_FRAME ExceptionFrame) 692 { 693 KiServiceExit2(TrapFrame); 694 } 695 696 // 697 // Global x86 only Kernel data 698 // 699 extern PVOID Ki386IopmSaveArea; 700 extern ULONG KeI386EFlagsAndMaskV86; 701 extern ULONG KeI386EFlagsOrMaskV86; 702 extern BOOLEAN KeI386VirtualIntExtensions; 703 extern KIDTENTRY KiIdt[MAXIMUM_IDTVECTOR+1]; 704 extern KDESCRIPTOR KiIdtDescriptor; 705 extern BOOLEAN KiI386PentiumLockErrataPresent; 706 extern ULONG KeI386NpxPresent; 707 extern ULONG KeI386XMMIPresent; 708 extern ULONG KeI386FxsrPresent; 709 extern ULONG KiMXCsrMask; 710 extern ULONG KeI386CpuType; 711 extern ULONG KeI386CpuStep; 712 extern ULONG KiFastSystemCallDisable; 713 extern UCHAR KiDebugRegisterTrapOffsets[9]; 714 extern UCHAR KiDebugRegisterContextOffsets[9]; 715 extern VOID __cdecl KiTrap02(VOID); 716 extern VOID __cdecl KiTrap08(VOID); 717 extern VOID __cdecl KiTrap13(VOID); 718 extern VOID __cdecl KiFastCallEntry(VOID); 719 extern VOID NTAPI ExpInterlockedPopEntrySListFault(VOID); 720 extern VOID NTAPI ExpInterlockedPopEntrySListResume(VOID); 721 extern VOID __cdecl CopyParams(VOID); 722 extern VOID __cdecl ReadBatch(VOID); 723 extern CHAR KiSystemCallExitBranch[]; 724 extern CHAR KiSystemCallExit[]; 725 extern CHAR KiSystemCallExit2[]; 726 727 // 728 // Trap Macros 729 // 730 #include "trap_x.h" 731 732 // 733 // Returns a thread's FPU save area 734 // 735 FORCEINLINE 736 PFX_SAVE_AREA 737 KiGetThreadNpxArea(IN PKTHREAD Thread) 738 { 739 ASSERT((ULONG_PTR)Thread->InitialStack % 16 == 0); 740 return (PFX_SAVE_AREA)((ULONG_PTR)Thread->InitialStack - sizeof(FX_SAVE_AREA)); 741 } 742 743 // 744 // Sanitizes a selector 745 // 746 FORCEINLINE 747 ULONG 748 Ke386SanitizeSeg(IN ULONG Cs, 749 IN KPROCESSOR_MODE Mode) 750 { 751 // 752 // Check if we're in kernel-mode, and force CPL 0 if so. 753 // Otherwise, force CPL 3. 754 // 755 return ((Mode == KernelMode) ? 756 (Cs & (0xFFFF & ~RPL_MASK)) : 757 (RPL_MASK | (Cs & 0xFFFF))); 758 } 759 760 // 761 // Sanitizes EFLAGS 762 // 763 FORCEINLINE 764 ULONG 765 Ke386SanitizeFlags(IN ULONG Eflags, 766 IN KPROCESSOR_MODE Mode) 767 { 768 // 769 // Check if we're in kernel-mode, and sanitize EFLAGS if so. 770 // Otherwise, also force interrupt mask on. 771 // 772 return ((Mode == KernelMode) ? 773 (Eflags & (EFLAGS_USER_SANITIZE | EFLAGS_INTERRUPT_MASK)) : 774 (EFLAGS_INTERRUPT_MASK | (Eflags & EFLAGS_USER_SANITIZE))); 775 } 776 777 // 778 // Sanitizes a Debug Register 779 // 780 FORCEINLINE 781 PVOID 782 Ke386SanitizeDr(IN PVOID DrAddress, 783 IN KPROCESSOR_MODE Mode) 784 { 785 // 786 // Check if we're in kernel-mode, and return the address directly if so. 787 // Otherwise, make sure it's not inside the kernel-mode address space. 788 // If it is, then clear the address. 789 // 790 return ((Mode == KernelMode) ? DrAddress : 791 (DrAddress <= MM_HIGHEST_USER_ADDRESS) ? DrAddress : 0); 792 } 793 794 // 795 // Exception with no arguments 796 // 797 FORCEINLINE 798 DECLSPEC_NORETURN 799 VOID 800 KiDispatchException0Args(IN NTSTATUS Code, 801 IN ULONG_PTR Address, 802 IN PKTRAP_FRAME TrapFrame) 803 { 804 /* Helper for exceptions with no arguments */ 805 KiDispatchExceptionFromTrapFrame(Code, 0, Address, 0, 0, 0, 0, TrapFrame); 806 } 807 808 // 809 // Exception with one argument 810 // 811 FORCEINLINE 812 DECLSPEC_NORETURN 813 VOID 814 KiDispatchException1Args(IN NTSTATUS Code, 815 IN ULONG_PTR Address, 816 IN ULONG P1, 817 IN PKTRAP_FRAME TrapFrame) 818 { 819 /* Helper for exceptions with no arguments */ 820 KiDispatchExceptionFromTrapFrame(Code, 0, Address, 1, P1, 0, 0, TrapFrame); 821 } 822 823 // 824 // Exception with two arguments 825 // 826 FORCEINLINE 827 DECLSPEC_NORETURN 828 VOID 829 KiDispatchException2Args(IN NTSTATUS Code, 830 IN ULONG_PTR Address, 831 IN ULONG P1, 832 IN ULONG P2, 833 IN PKTRAP_FRAME TrapFrame) 834 { 835 /* Helper for exceptions with no arguments */ 836 KiDispatchExceptionFromTrapFrame(Code, 0, Address, 2, P1, P2, 0, TrapFrame); 837 } 838 839 // 840 // Performs a system call 841 // 842 NTSTATUS 843 NTAPI 844 KiSystemCallTrampoline(_In_ PVOID Handler, 845 _In_ PVOID Arguments, 846 _In_ ULONG StackBytes); 847 848 849 // 850 // Checks for pending APCs 851 // 852 FORCEINLINE 853 VOID 854 KiCheckForApcDelivery(IN PKTRAP_FRAME TrapFrame) 855 { 856 PKTHREAD Thread; 857 KIRQL OldIrql; 858 859 /* Check for V8086 or user-mode trap */ 860 if ((TrapFrame->EFlags & EFLAGS_V86_MASK) || (KiUserTrap(TrapFrame))) 861 { 862 /* Get the thread */ 863 Thread = KeGetCurrentThread(); 864 while (TRUE) 865 { 866 /* Turn off the alerted state for kernel mode */ 867 Thread->Alerted[KernelMode] = FALSE; 868 869 /* Are there pending user APCs? */ 870 if (!Thread->ApcState.UserApcPending) break; 871 872 /* Raise to APC level and enable interrupts */ 873 OldIrql = KfRaiseIrql(APC_LEVEL); 874 _enable(); 875 876 /* Deliver APCs */ 877 KiDeliverApc(UserMode, NULL, TrapFrame); 878 879 /* Restore IRQL and disable interrupts once again */ 880 KfLowerIrql(OldIrql); 881 _disable(); 882 } 883 } 884 } 885 886 // 887 // Switches from boot loader to initial kernel stack 888 // 889 CODE_SEG("INIT") 890 FORCEINLINE 891 DECLSPEC_NORETURN 892 VOID 893 KiSwitchToBootStack(IN ULONG_PTR InitialStack) 894 { 895 CODE_SEG("INIT") DECLSPEC_NORETURN VOID NTAPI KiSystemStartupBootStack(VOID); 896 897 /* We have to switch to a new stack before continuing kernel initialization */ 898 #ifdef __GNUC__ 899 __asm__ 900 ( 901 "movl %0, %%esp\n\t" 902 "subl %1, %%esp\n\t" 903 "pushl %2\n\t" 904 "jmp _KiSystemStartupBootStack@0" 905 : 906 : "c"(InitialStack), 907 "i"(NPX_FRAME_LENGTH + KTRAP_FRAME_ALIGN + KTRAP_FRAME_LENGTH), 908 "i"(CR0_EM | CR0_TS | CR0_MP), 909 "p"(KiSystemStartupBootStack) 910 : "%esp" 911 ); 912 #elif defined(_MSC_VER) 913 __asm 914 { 915 mov esp, InitialStack 916 sub esp, (NPX_FRAME_LENGTH + KTRAP_FRAME_ALIGN + KTRAP_FRAME_LENGTH) 917 push (CR0_EM | CR0_TS | CR0_MP) 918 jmp KiSystemStartupBootStack 919 } 920 #else 921 #error Unknown Compiler 922 #endif 923 924 UNREACHABLE; 925 } 926 927 // 928 // Emits the iret instruction for C code 929 // 930 FORCEINLINE 931 DECLSPEC_NORETURN 932 VOID 933 KiIret(VOID) 934 { 935 #if defined(__GNUC__) 936 __asm__ __volatile__ 937 ( 938 "iret" 939 ); 940 #elif defined(_MSC_VER) 941 __asm 942 { 943 iretd 944 } 945 #else 946 #error Unsupported compiler 947 #endif 948 UNREACHABLE; 949 } 950 951 // 952 // Normally this is done by the HAL, but on x86 as an optimization, the kernel 953 // initiates the end by calling back into the HAL and exiting the trap here. 954 // 955 FORCEINLINE 956 VOID 957 KiEndInterrupt(IN KIRQL Irql, 958 IN PKTRAP_FRAME TrapFrame) 959 { 960 /* Disable interrupts and end the interrupt */ 961 _disable(); 962 HalEndSystemInterrupt(Irql, TrapFrame); 963 964 /* Exit the interrupt */ 965 KiEoiHelper(TrapFrame); 966 } 967 968 // 969 // PERF Code 970 // 971 FORCEINLINE 972 VOID 973 Ki386PerfEnd(VOID) 974 { 975 extern ULONGLONG BootCyclesEnd, BootCycles; 976 BootCyclesEnd = __rdtsc(); 977 DbgPrint("Boot took %I64u cycles!\n", BootCyclesEnd - BootCycles); 978 DbgPrint("Interrupts: %u System Calls: %u Context Switches: %u\n", 979 KeGetCurrentPrcb()->InterruptCount, 980 KeGetCurrentPrcb()->KeSystemCalls, 981 KeGetContextSwitches(KeGetCurrentPrcb())); 982 } 983 984 FORCEINLINE 985 PULONG 986 KiGetUserModeStackAddress(void) 987 { 988 return &(KeGetCurrentThread()->TrapFrame->HardwareEsp); 989 } 990 991 #ifdef __cplusplus 992 } // extern "C" 993 #endif 994 995 #endif 996