1 #pragma once 2 3 #ifndef __ASM__ 4 5 #include "intrin_i.h" 6 7 #ifdef __cplusplus 8 extern "C" 9 { 10 #endif 11 12 // 13 // Thread Dispatcher Header DebugActive Mask 14 // 15 #define DR_MASK(x) (1 << (x)) 16 #define DR_REG_MASK 0x4F 17 18 // 19 // INT3 is 1 byte long 20 // 21 #define KD_BREAKPOINT_TYPE UCHAR 22 #define KD_BREAKPOINT_SIZE sizeof(UCHAR) 23 #define KD_BREAKPOINT_VALUE 0xCC 24 25 /* CPUID 1 - EDX flags */ 26 #define X86_FEATURE_FPU 0x00000001 /* x87 FPU is present */ 27 #define X86_FEATURE_VME 0x00000002 /* Virtual 8086 Extensions are present */ 28 #define X86_FEATURE_DBG 0x00000004 /* Debugging extensions are present */ 29 #define X86_FEATURE_PSE 0x00000008 /* Page Size Extension is present */ 30 #define X86_FEATURE_TSC 0x00000010 /* Time Stamp Counters are present */ 31 #define X86_FEATURE_PAE 0x00000040 /* Physical Address Extension is present */ 32 #define X86_FEATURE_CX8 0x00000100 /* CMPXCHG8B instruction present */ 33 #define X86_FEATURE_APIC 0x00000200 /* APIC is present */ 34 #define X86_FEATURE_SYSCALL 0x00000800 /* SYSCALL/SYSRET support present */ 35 #define X86_FEATURE_MTTR 0x00001000 /* Memory type range registers are present */ 36 #define X86_FEATURE_PGE 0x00002000 /* Page Global Enable */ 37 #define X86_FEATURE_CMOV 0x00008000 /* "Conditional move" instruction supported */ 38 #define X86_FEATURE_PAT 0x00010000 /* Page Attribute Table is supported */ 39 #define X86_FEATURE_DS 0x00200000 /* Debug Store is present */ 40 #define X86_FEATURE_MMX 0x00800000 /* MMX extension present */ 41 #define X86_FEATURE_FXSR 0x01000000 /* FXSAVE/FXRSTOR instructions present */ 42 #define X86_FEATURE_SSE 0x02000000 /* SSE extension present */ 43 #define X86_FEATURE_SSE2 0x04000000 /* SSE2 extension present */ 44 #define X86_FEATURE_HT 0x10000000 /* Hyper-Threading present */ 45 46 /* CPUID 0x80000001 - EDX extended flags */ 47 #define X86_FEATURE_NX 0x00100000 /* NX support present */ 48 49 // 50 // One-liners for getting and setting special purpose registers in portable code 51 // 52 FORCEINLINE 53 ULONG_PTR 54 KeGetContextPc(PCONTEXT Context) 55 { 56 return Context->Eip; 57 } 58 59 FORCEINLINE 60 VOID 61 KeSetContextPc(PCONTEXT Context, ULONG_PTR ProgramCounter) 62 { 63 Context->Eip = ProgramCounter; 64 } 65 66 FORCEINLINE 67 ULONG_PTR 68 KeGetContextReturnRegister(PCONTEXT Context) 69 { 70 return Context->Eax; 71 } 72 73 FORCEINLINE 74 VOID 75 KeSetContextReturnRegister(PCONTEXT Context, ULONG_PTR ReturnValue) 76 { 77 Context->Eax = ReturnValue; 78 } 79 80 FORCEINLINE 81 ULONG_PTR 82 KeGetContextFrameRegister(PCONTEXT Context) 83 { 84 return Context->Ebp; 85 } 86 87 FORCEINLINE 88 VOID 89 KeSetContextFrameRegister(PCONTEXT Context, ULONG_PTR Frame) 90 { 91 Context->Ebp = Frame; 92 } 93 94 FORCEINLINE 95 ULONG_PTR 96 KeGetTrapFramePc(PKTRAP_FRAME TrapFrame) 97 { 98 return TrapFrame->Eip; 99 } 100 101 FORCEINLINE 102 PKTRAP_FRAME 103 KiGetLinkedTrapFrame(PKTRAP_FRAME TrapFrame) 104 { 105 return (PKTRAP_FRAME)TrapFrame->Edx; 106 } 107 108 109 FORCEINLINE 110 ULONG_PTR 111 KeGetTrapFrameStackRegister(PKTRAP_FRAME TrapFrame) 112 { 113 if (TrapFrame->PreviousPreviousMode == KernelMode) 114 return TrapFrame->TempEsp; 115 return TrapFrame->HardwareEsp; 116 } 117 118 FORCEINLINE 119 ULONG_PTR 120 KeGetTrapFrameFrameRegister(PKTRAP_FRAME TrapFrame) 121 { 122 return TrapFrame->Ebp; 123 } 124 125 // 126 // Macro to get trap and exception frame from a thread stack 127 // 128 #define KeGetTrapFrame(Thread) \ 129 (PKTRAP_FRAME)((ULONG_PTR)((Thread)->InitialStack) - \ 130 sizeof(KTRAP_FRAME) - \ 131 sizeof(FX_SAVE_AREA)) 132 133 #define KeGetExceptionFrame(Thread) \ 134 NULL 135 136 // 137 // Macro to get context switches from the PRCB 138 // All architectures but x86 have it in the PRCB's KeContextSwitches 139 // 140 #define KeGetContextSwitches(Prcb) \ 141 CONTAINING_RECORD(Prcb, KIPCR, PrcbData)->ContextSwitches 142 143 // 144 // Macro to get the second level cache size field name which differs between 145 // CISC and RISC architectures, as the former has unified I/D cache 146 // 147 #define KiGetSecondLevelDCacheSize() ((PKIPCR)KeGetPcr())->SecondLevelCacheSize 148 149 // 150 // Returns the Interrupt State from a Trap Frame. 151 // ON = TRUE, OFF = FALSE 152 // 153 #define KeGetTrapFrameInterruptState(TrapFrame) \ 154 BooleanFlagOn((TrapFrame)->EFlags, EFLAGS_INTERRUPT_MASK) 155 156 // 157 // Flags for exiting a trap 158 // 159 #define KTE_SKIP_PM_BIT (((KTRAP_EXIT_SKIP_BITS) { { .SkipPreviousMode = TRUE } }).Bits) 160 #define KTE_SKIP_SEG_BIT (((KTRAP_EXIT_SKIP_BITS) { { .SkipSegments = TRUE } }).Bits) 161 #define KTE_SKIP_VOL_BIT (((KTRAP_EXIT_SKIP_BITS) { { .SkipVolatiles = TRUE } }).Bits) 162 163 typedef union _KTRAP_EXIT_SKIP_BITS 164 { 165 struct 166 { 167 UCHAR SkipPreviousMode:1; 168 UCHAR SkipSegments:1; 169 UCHAR SkipVolatiles:1; 170 UCHAR Reserved:5; 171 }; 172 UCHAR Bits; 173 } KTRAP_EXIT_SKIP_BITS, *PKTRAP_EXIT_SKIP_BITS; 174 175 176 // 177 // Flags used by the VDM/V8086 emulation engine for determining instruction prefixes 178 // 179 #define PFX_FLAG_ES 0x00000100 180 #define PFX_FLAG_CS 0x00000200 181 #define PFX_FLAG_SS 0x00000400 182 #define PFX_FLAG_DS 0x00000800 183 #define PFX_FLAG_FS 0x00001000 184 #define PFX_FLAG_GS 0x00002000 185 #define PFX_FLAG_OPER32 0x00004000 186 #define PFX_FLAG_ADDR32 0x00008000 187 #define PFX_FLAG_LOCK 0x00010000 188 #define PFX_FLAG_REPNE 0x00020000 189 #define PFX_FLAG_REP 0x00040000 190 191 // 192 // VDM Helper Macros 193 // 194 // All VDM/V8086 opcode emulators have the same FASTCALL function definition. 195 // We need to keep 2 parameters while the original ASM implementation uses 4: 196 // TrapFrame, PrefixFlags, Eip, InstructionSize; 197 // 198 // We pass the trap frame, and prefix flags, in our two parameters. 199 // 200 // We then realize that since the smallest prefix flag is 0x100, this gives us 201 // a count of up to 0xFF. So we OR in the instruction size with the prefix flags 202 // 203 // We further realize that we always have access to EIP from the trap frame, and 204 // that if we want the *current instruction* EIP, we simply have to add the 205 // instruction size *MINUS ONE*, and that gives us the EIP we should be looking 206 // at now, so we don't need to use the stack to push this parameter. 207 // 208 // We actually only care about the *current instruction* EIP in one location, 209 // so although it may be slightly more expensive to re-calculate the EIP one 210 // more time, this way we don't redefine ALL opcode handlers to have 3 parameters, 211 // which would be forcing stack usage in all other scenarios. 212 // 213 #define KiVdmSetVdmEFlags(x) InterlockedOr((PLONG)KiNtVdmState, (x)); 214 #define KiVdmClearVdmEFlags(x) InterlockedAnd((PLONG)KiNtVdmState, ~(x)) 215 #define KiCallVdmHandler(x) KiVdmOpcode##x(TrapFrame, Flags) 216 #define KiCallVdmPrefixHandler(x) KiVdmOpcodePrefix(TrapFrame, Flags | x) 217 #define KiVdmUnhandledOpcode(x) \ 218 BOOLEAN \ 219 FASTCALL \ 220 KiVdmOpcode##x(IN PKTRAP_FRAME TrapFrame, \ 221 IN ULONG Flags) \ 222 { \ 223 /* Not yet handled */ \ 224 UNIMPLEMENTED_DBGBREAK(); \ 225 return FALSE; \ 226 } 227 228 C_ASSERT(NPX_FRAME_LENGTH == sizeof(FX_SAVE_AREA)); 229 230 // 231 // Local parameters 232 // 233 typedef struct _KV86_FRAME 234 { 235 PVOID ThreadStack; 236 PVOID ThreadTeb; 237 PVOID PcrTeb; 238 } KV86_FRAME, *PKV86_FRAME; 239 240 // 241 // Virtual Stack Frame 242 // 243 typedef struct _KV8086_STACK_FRAME 244 { 245 KTRAP_FRAME TrapFrame; 246 FX_SAVE_AREA NpxArea; 247 KV86_FRAME V86Frame; 248 } KV8086_STACK_FRAME, *PKV8086_STACK_FRAME; 249 250 // 251 // Large Pages Support 252 // 253 typedef struct _LARGE_IDENTITY_MAP 254 { 255 PHARDWARE_PTE TopLevelDirectory; 256 ULONG Cr3; 257 ULONG_PTR StartAddress; 258 ULONG PagesCount; 259 PVOID PagesList[30]; 260 } LARGE_IDENTITY_MAP, *PLARGE_IDENTITY_MAP; 261 262 // 263 // Floating Point Internal Context Structure 264 // 265 typedef struct _FLOATING_SAVE_CONTEXT 266 { 267 PKTHREAD CurrentThread; 268 KIRQL OldNpxIrql; 269 PFX_SAVE_AREA Buffer; 270 PFX_SAVE_AREA PfxSaveArea; 271 } FLOATING_SAVE_CONTEXT, *PFLOATING_SAVE_CONTEXT; 272 273 /* Diable interrupts and return whether they were enabled before */ 274 FORCEINLINE 275 BOOLEAN 276 KeDisableInterrupts(VOID) 277 { 278 ULONG Flags; 279 BOOLEAN Return; 280 281 /* Get EFLAGS and check if the interrupt bit is set */ 282 Flags = __readeflags(); 283 Return = (Flags & EFLAGS_INTERRUPT_MASK) ? TRUE: FALSE; 284 285 /* Disable interrupts */ 286 _disable(); 287 return Return; 288 } 289 290 /* Restore previous interrupt state */ 291 FORCEINLINE 292 VOID 293 KeRestoreInterrupts(BOOLEAN WereEnabled) 294 { 295 if (WereEnabled) _enable(); 296 } 297 298 // 299 // Registers an interrupt handler with an IDT vector 300 // 301 FORCEINLINE 302 VOID 303 KeRegisterInterruptHandler(IN ULONG Vector, 304 IN PVOID Handler) 305 { 306 UCHAR Entry; 307 ULONG_PTR Address; 308 PKIPCR Pcr = (PKIPCR)KeGetPcr(); 309 310 // 311 // Get the entry from the HAL 312 // 313 Entry = HalVectorToIDTEntry(Vector); 314 Address = PtrToUlong(Handler); 315 316 // 317 // Now set the data 318 // 319 Pcr->IDT[Entry].ExtendedOffset = (USHORT)(Address >> 16); 320 Pcr->IDT[Entry].Offset = (USHORT)Address; 321 } 322 323 // 324 // Returns the registered interrupt handler for a given IDT vector 325 // 326 FORCEINLINE 327 PVOID 328 KeQueryInterruptHandler(IN ULONG Vector) 329 { 330 PKIPCR Pcr = (PKIPCR)KeGetPcr(); 331 UCHAR Entry; 332 333 // 334 // Get the entry from the HAL 335 // 336 Entry = HalVectorToIDTEntry(Vector); 337 338 // 339 // Read the entry from the IDT 340 // 341 return (PVOID)(((Pcr->IDT[Entry].ExtendedOffset << 16) & 0xFFFF0000) | 342 (Pcr->IDT[Entry].Offset & 0xFFFF)); 343 } 344 345 // 346 // Invalidates the TLB entry for a specified address 347 // 348 FORCEINLINE 349 VOID 350 KeInvalidateTlbEntry(IN PVOID Address) 351 { 352 /* Invalidate the TLB entry for this address */ 353 __invlpg(Address); 354 } 355 356 FORCEINLINE 357 VOID 358 KeFlushProcessTb(VOID) 359 { 360 /* Flush the TLB by resetting CR3 */ 361 __writecr3(__readcr3()); 362 } 363 364 FORCEINLINE 365 VOID 366 KeSweepICache(IN PVOID BaseAddress, 367 IN SIZE_T FlushSize) 368 { 369 // 370 // Always sweep the whole cache 371 // 372 UNREFERENCED_PARAMETER(BaseAddress); 373 UNREFERENCED_PARAMETER(FlushSize); 374 __wbinvd(); 375 } 376 377 FORCEINLINE 378 PRKTHREAD 379 KeGetCurrentThread(VOID) 380 { 381 /* Return the current thread */ 382 return ((PKIPCR)KeGetPcr())->PrcbData.CurrentThread; 383 } 384 385 FORCEINLINE 386 VOID 387 KiRundownThread(IN PKTHREAD Thread) 388 { 389 #ifndef CONFIG_SMP 390 /* Check if this is the NPX Thread */ 391 if (KeGetCurrentPrcb()->NpxThread == Thread) 392 { 393 /* Clear it */ 394 KeGetCurrentPrcb()->NpxThread = NULL; 395 Ke386FnInit(); 396 } 397 #else 398 /* Nothing to do */ 399 #endif 400 } 401 402 FORCEINLINE 403 VOID 404 Ke386SetGdtEntryBase(PKGDTENTRY GdtEntry, PVOID BaseAddress) 405 { 406 GdtEntry->BaseLow = (USHORT)((ULONG_PTR)BaseAddress & 0xFFFF); 407 GdtEntry->HighWord.Bytes.BaseMid = (UCHAR)((ULONG_PTR)BaseAddress >> 16); 408 GdtEntry->HighWord.Bytes.BaseHi = (UCHAR)((ULONG_PTR)BaseAddress >> 24); 409 } 410 411 FORCEINLINE 412 VOID 413 KiSetTebBase(PKPCR Pcr, PNT_TIB TebAddress) 414 { 415 Pcr->NtTib.Self = TebAddress; 416 Ke386SetGdtEntryBase(&Pcr->GDT[KGDT_R3_TEB / sizeof(KGDTENTRY)], TebAddress); 417 } 418 419 CODE_SEG("INIT") 420 VOID 421 FASTCALL 422 Ki386InitializeTss( 423 IN PKTSS Tss, 424 IN PKIDTENTRY Idt, 425 IN PKGDTENTRY Gdt 426 ); 427 428 CODE_SEG("INIT") 429 VOID 430 NTAPI 431 KiSetCR0Bits(VOID); 432 433 CODE_SEG("INIT") 434 VOID 435 NTAPI 436 KiGetCacheInformation(VOID); 437 438 CODE_SEG("INIT") 439 BOOLEAN 440 NTAPI 441 KiIsNpxPresent( 442 VOID 443 ); 444 445 CODE_SEG("INIT") 446 BOOLEAN 447 NTAPI 448 KiIsNpxErrataPresent( 449 VOID 450 ); 451 452 CODE_SEG("INIT") 453 VOID 454 NTAPI 455 KiSetProcessorType(VOID); 456 457 CODE_SEG("INIT") 458 ULONG 459 NTAPI 460 KiGetFeatureBits(VOID); 461 462 #if DBG 463 CODE_SEG("INIT") 464 VOID 465 KiReportCpuFeatures(VOID); 466 #endif 467 468 VOID 469 NTAPI 470 KiThreadStartup(VOID); 471 472 NTSTATUS 473 NTAPI 474 Ke386GetGdtEntryThread( 475 IN PKTHREAD Thread, 476 IN ULONG Offset, 477 IN PKGDTENTRY Descriptor 478 ); 479 480 VOID 481 NTAPI 482 KiFlushNPXState( 483 IN FLOATING_SAVE_AREA *SaveArea 484 ); 485 486 VOID 487 NTAPI 488 Ki386AdjustEsp0( 489 IN PKTRAP_FRAME TrapFrame 490 ); 491 492 VOID 493 NTAPI 494 Ki386SetupAndExitToV86Mode( 495 OUT PTEB VdmTeb 496 ); 497 498 CODE_SEG("INIT") 499 VOID 500 NTAPI 501 KeI386VdmInitialize( 502 VOID 503 ); 504 505 CODE_SEG("INIT") 506 ULONG_PTR 507 NTAPI 508 Ki386EnableGlobalPage( 509 IN ULONG_PTR Context 510 ); 511 512 CODE_SEG("INIT") 513 ULONG_PTR 514 NTAPI 515 Ki386EnableTargetLargePage( 516 IN ULONG_PTR Context 517 ); 518 519 BOOLEAN 520 NTAPI 521 Ki386CreateIdentityMap( 522 IN PLARGE_IDENTITY_MAP IdentityMap, 523 IN PVOID StartPtr, 524 IN ULONG Length 525 ); 526 527 VOID 528 NTAPI 529 Ki386FreeIdentityMap( 530 IN PLARGE_IDENTITY_MAP IdentityMap 531 ); 532 533 VOID 534 NTAPI 535 Ki386EnableCurrentLargePage( 536 IN ULONG_PTR StartAddress, 537 IN ULONG Cr3 538 ); 539 540 CODE_SEG("INIT") 541 VOID 542 NTAPI 543 KiI386PentiumLockErrataFixup( 544 VOID 545 ); 546 547 CODE_SEG("INIT") 548 VOID 549 NTAPI 550 KiInitializePAT( 551 VOID 552 ); 553 554 CODE_SEG("INIT") 555 VOID 556 NTAPI 557 KiInitializeMTRR( 558 IN BOOLEAN FinalCpu 559 ); 560 561 CODE_SEG("INIT") 562 VOID 563 NTAPI 564 KiAmdK6InitializeMTRR( 565 VOID 566 ); 567 568 CODE_SEG("INIT") 569 VOID 570 NTAPI 571 KiRestoreFastSyscallReturnState( 572 VOID 573 ); 574 575 CODE_SEG("INIT") 576 ULONG_PTR 577 NTAPI 578 Ki386EnableDE( 579 IN ULONG_PTR Context 580 ); 581 582 CODE_SEG("INIT") 583 ULONG_PTR 584 NTAPI 585 Ki386EnableFxsr( 586 IN ULONG_PTR Context 587 ); 588 589 CODE_SEG("INIT") 590 ULONG_PTR 591 NTAPI 592 Ki386EnableXMMIExceptions( 593 IN ULONG_PTR Context 594 ); 595 596 BOOLEAN 597 NTAPI 598 VdmDispatchBop( 599 IN PKTRAP_FRAME TrapFrame 600 ); 601 602 BOOLEAN 603 NTAPI 604 VdmDispatchPageFault( 605 _In_ PKTRAP_FRAME TrapFrame 606 ); 607 608 BOOLEAN 609 FASTCALL 610 KiVdmOpcodePrefix( 611 IN PKTRAP_FRAME TrapFrame, 612 IN ULONG Flags 613 ); 614 615 BOOLEAN 616 FASTCALL 617 Ki386HandleOpcodeV86( 618 IN PKTRAP_FRAME TrapFrame 619 ); 620 621 DECLSPEC_NORETURN 622 VOID 623 FASTCALL 624 KiEoiHelper( 625 IN PKTRAP_FRAME TrapFrame 626 ); 627 628 VOID 629 FASTCALL 630 Ki386BiosCallReturnAddress( 631 IN PKTRAP_FRAME TrapFrame 632 ); 633 634 ULONG_PTR 635 FASTCALL 636 KiExitV86Mode( 637 IN PKTRAP_FRAME TrapFrame 638 ); 639 640 DECLSPEC_NORETURN 641 VOID 642 NTAPI 643 KiDispatchExceptionFromTrapFrame( 644 IN NTSTATUS Code, 645 IN ULONG Flags, 646 IN ULONG_PTR Address, 647 IN ULONG ParameterCount, 648 IN ULONG_PTR Parameter1, 649 IN ULONG_PTR Parameter2, 650 IN ULONG_PTR Parameter3, 651 IN PKTRAP_FRAME TrapFrame 652 ); 653 654 NTSTATUS 655 NTAPI 656 KiConvertToGuiThread( 657 VOID 658 ); 659 660 // 661 // Global x86 only Kernel data 662 // 663 extern PVOID Ki386IopmSaveArea; 664 extern ULONG KeI386EFlagsAndMaskV86; 665 extern ULONG KeI386EFlagsOrMaskV86; 666 extern BOOLEAN KeI386VirtualIntExtensions; 667 extern KIDTENTRY KiIdt[MAXIMUM_IDTVECTOR+1]; 668 extern KDESCRIPTOR KiIdtDescriptor; 669 extern BOOLEAN KiI386PentiumLockErrataPresent; 670 extern ULONG KeI386NpxPresent; 671 extern ULONG KeI386XMMIPresent; 672 extern ULONG KeI386FxsrPresent; 673 extern ULONG KiMXCsrMask; 674 extern ULONG KeI386CpuType; 675 extern ULONG KeI386CpuStep; 676 extern ULONG KiFastSystemCallDisable; 677 extern UCHAR KiDebugRegisterTrapOffsets[9]; 678 extern UCHAR KiDebugRegisterContextOffsets[9]; 679 extern VOID __cdecl KiTrap02(VOID); 680 extern VOID __cdecl KiTrap08(VOID); 681 extern VOID __cdecl KiTrap13(VOID); 682 extern VOID __cdecl KiFastCallEntry(VOID); 683 extern VOID NTAPI ExpInterlockedPopEntrySListFault(VOID); 684 extern VOID NTAPI ExpInterlockedPopEntrySListResume(VOID); 685 extern VOID __cdecl CopyParams(VOID); 686 extern VOID __cdecl ReadBatch(VOID); 687 extern CHAR KiSystemCallExitBranch[]; 688 extern CHAR KiSystemCallExit[]; 689 extern CHAR KiSystemCallExit2[]; 690 691 // 692 // Trap Macros 693 // 694 #include "trap_x.h" 695 696 // 697 // Returns a thread's FPU save area 698 // 699 FORCEINLINE 700 PFX_SAVE_AREA 701 KiGetThreadNpxArea(IN PKTHREAD Thread) 702 { 703 ASSERT((ULONG_PTR)Thread->InitialStack % 16 == 0); 704 return (PFX_SAVE_AREA)((ULONG_PTR)Thread->InitialStack - sizeof(FX_SAVE_AREA)); 705 } 706 707 // 708 // Sanitizes a selector 709 // 710 FORCEINLINE 711 ULONG 712 Ke386SanitizeSeg(IN ULONG Cs, 713 IN KPROCESSOR_MODE Mode) 714 { 715 // 716 // Check if we're in kernel-mode, and force CPL 0 if so. 717 // Otherwise, force CPL 3. 718 // 719 return ((Mode == KernelMode) ? 720 (Cs & (0xFFFF & ~RPL_MASK)) : 721 (RPL_MASK | (Cs & 0xFFFF))); 722 } 723 724 // 725 // Sanitizes EFLAGS 726 // 727 FORCEINLINE 728 ULONG 729 Ke386SanitizeFlags(IN ULONG Eflags, 730 IN KPROCESSOR_MODE Mode) 731 { 732 // 733 // Check if we're in kernel-mode, and sanitize EFLAGS if so. 734 // Otherwise, also force interrupt mask on. 735 // 736 return ((Mode == KernelMode) ? 737 (Eflags & (EFLAGS_USER_SANITIZE | EFLAGS_INTERRUPT_MASK)) : 738 (EFLAGS_INTERRUPT_MASK | (Eflags & EFLAGS_USER_SANITIZE))); 739 } 740 741 // 742 // Sanitizes a Debug Register 743 // 744 FORCEINLINE 745 PVOID 746 Ke386SanitizeDr(IN PVOID DrAddress, 747 IN KPROCESSOR_MODE Mode) 748 { 749 // 750 // Check if we're in kernel-mode, and return the address directly if so. 751 // Otherwise, make sure it's not inside the kernel-mode address space. 752 // If it is, then clear the address. 753 // 754 return ((Mode == KernelMode) ? DrAddress : 755 (DrAddress <= MM_HIGHEST_USER_ADDRESS) ? DrAddress : 0); 756 } 757 758 // 759 // Exception with no arguments 760 // 761 FORCEINLINE 762 DECLSPEC_NORETURN 763 VOID 764 KiDispatchException0Args(IN NTSTATUS Code, 765 IN ULONG_PTR Address, 766 IN PKTRAP_FRAME TrapFrame) 767 { 768 /* Helper for exceptions with no arguments */ 769 KiDispatchExceptionFromTrapFrame(Code, 0, Address, 0, 0, 0, 0, TrapFrame); 770 } 771 772 // 773 // Exception with one argument 774 // 775 FORCEINLINE 776 DECLSPEC_NORETURN 777 VOID 778 KiDispatchException1Args(IN NTSTATUS Code, 779 IN ULONG_PTR Address, 780 IN ULONG P1, 781 IN PKTRAP_FRAME TrapFrame) 782 { 783 /* Helper for exceptions with no arguments */ 784 KiDispatchExceptionFromTrapFrame(Code, 0, Address, 1, P1, 0, 0, TrapFrame); 785 } 786 787 // 788 // Exception with two arguments 789 // 790 FORCEINLINE 791 DECLSPEC_NORETURN 792 VOID 793 KiDispatchException2Args(IN NTSTATUS Code, 794 IN ULONG_PTR Address, 795 IN ULONG P1, 796 IN ULONG P2, 797 IN PKTRAP_FRAME TrapFrame) 798 { 799 /* Helper for exceptions with no arguments */ 800 KiDispatchExceptionFromTrapFrame(Code, 0, Address, 2, P1, P2, 0, TrapFrame); 801 } 802 803 // 804 // Performs a system call 805 // 806 NTSTATUS 807 NTAPI 808 KiSystemCallTrampoline(_In_ PVOID Handler, 809 _In_ PVOID Arguments, 810 _In_ ULONG StackBytes); 811 812 813 // 814 // Checks for pending APCs 815 // 816 FORCEINLINE 817 VOID 818 KiCheckForApcDelivery(IN PKTRAP_FRAME TrapFrame) 819 { 820 PKTHREAD Thread; 821 KIRQL OldIrql; 822 823 /* Check for V8086 or user-mode trap */ 824 if ((TrapFrame->EFlags & EFLAGS_V86_MASK) || (KiUserTrap(TrapFrame))) 825 { 826 /* Get the thread */ 827 Thread = KeGetCurrentThread(); 828 while (TRUE) 829 { 830 /* Turn off the alerted state for kernel mode */ 831 Thread->Alerted[KernelMode] = FALSE; 832 833 /* Are there pending user APCs? */ 834 if (!Thread->ApcState.UserApcPending) break; 835 836 /* Raise to APC level and enable interrupts */ 837 OldIrql = KfRaiseIrql(APC_LEVEL); 838 _enable(); 839 840 /* Deliver APCs */ 841 KiDeliverApc(UserMode, NULL, TrapFrame); 842 843 /* Restore IRQL and disable interrupts once again */ 844 KfLowerIrql(OldIrql); 845 _disable(); 846 } 847 } 848 } 849 850 // 851 // Switches from boot loader to initial kernel stack 852 // 853 CODE_SEG("INIT") 854 FORCEINLINE 855 DECLSPEC_NORETURN 856 VOID 857 KiSwitchToBootStack(IN ULONG_PTR InitialStack) 858 { 859 CODE_SEG("INIT") DECLSPEC_NORETURN VOID NTAPI KiSystemStartupBootStack(VOID); 860 861 /* We have to switch to a new stack before continuing kernel initialization */ 862 #ifdef __GNUC__ 863 __asm__ 864 ( 865 "movl %0, %%esp\n\t" 866 "subl %1, %%esp\n\t" 867 "pushl %2\n\t" 868 "jmp _KiSystemStartupBootStack@0" 869 : 870 : "c"(InitialStack), 871 "i"(NPX_FRAME_LENGTH + KTRAP_FRAME_ALIGN + KTRAP_FRAME_LENGTH), 872 "i"(CR0_EM | CR0_TS | CR0_MP), 873 "p"(KiSystemStartupBootStack) 874 : "%esp" 875 ); 876 #elif defined(_MSC_VER) 877 __asm 878 { 879 mov esp, InitialStack 880 sub esp, (NPX_FRAME_LENGTH + KTRAP_FRAME_ALIGN + KTRAP_FRAME_LENGTH) 881 push (CR0_EM | CR0_TS | CR0_MP) 882 jmp KiSystemStartupBootStack 883 } 884 #else 885 #error Unknown Compiler 886 #endif 887 888 UNREACHABLE; 889 } 890 891 // 892 // Emits the iret instruction for C code 893 // 894 FORCEINLINE 895 DECLSPEC_NORETURN 896 VOID 897 KiIret(VOID) 898 { 899 #if defined(__GNUC__) 900 __asm__ __volatile__ 901 ( 902 "iret" 903 ); 904 #elif defined(_MSC_VER) 905 __asm 906 { 907 iretd 908 } 909 #else 910 #error Unsupported compiler 911 #endif 912 UNREACHABLE; 913 } 914 915 // 916 // Normally this is done by the HAL, but on x86 as an optimization, the kernel 917 // initiates the end by calling back into the HAL and exiting the trap here. 918 // 919 FORCEINLINE 920 VOID 921 KiEndInterrupt(IN KIRQL Irql, 922 IN PKTRAP_FRAME TrapFrame) 923 { 924 /* Disable interrupts and end the interrupt */ 925 _disable(); 926 HalEndSystemInterrupt(Irql, TrapFrame); 927 928 /* Exit the interrupt */ 929 KiEoiHelper(TrapFrame); 930 } 931 932 // 933 // PERF Code 934 // 935 FORCEINLINE 936 VOID 937 Ki386PerfEnd(VOID) 938 { 939 extern ULONGLONG BootCyclesEnd, BootCycles; 940 BootCyclesEnd = __rdtsc(); 941 DbgPrint("Boot took %I64u cycles!\n", BootCyclesEnd - BootCycles); 942 DbgPrint("Interrupts: %u System Calls: %u Context Switches: %u\n", 943 KeGetCurrentPrcb()->InterruptCount, 944 KeGetCurrentPrcb()->KeSystemCalls, 945 KeGetContextSwitches(KeGetCurrentPrcb())); 946 } 947 948 FORCEINLINE 949 PULONG 950 KiGetUserModeStackAddress(void) 951 { 952 return &(KeGetCurrentThread()->TrapFrame->HardwareEsp); 953 } 954 955 #ifdef __cplusplus 956 } // extern "C" 957 #endif 958 959 #endif 960