1 #pragma once 2 3 #ifndef __ASM__ 4 5 #include "intrin_i.h" 6 7 #ifdef __cplusplus 8 extern "C" 9 { 10 #endif 11 12 // 13 // Thread Dispatcher Header DebugActive Mask 14 // 15 #define DR_MASK(x) (1 << (x)) 16 #define DR_REG_MASK 0x4F 17 18 // 19 // INT3 is 1 byte long 20 // 21 #define KD_BREAKPOINT_TYPE UCHAR 22 #define KD_BREAKPOINT_SIZE sizeof(UCHAR) 23 #define KD_BREAKPOINT_VALUE 0xCC 24 25 /* CPUID 1 - EDX flags */ 26 #define X86_FEATURE_FPU 0x00000001 /* x87 FPU is present */ 27 #define X86_FEATURE_VME 0x00000002 /* Virtual 8086 Extensions are present */ 28 #define X86_FEATURE_DBG 0x00000004 /* Debugging extensions are present */ 29 #define X86_FEATURE_PSE 0x00000008 /* Page Size Extension is present */ 30 #define X86_FEATURE_TSC 0x00000010 /* Time Stamp Counters are present */ 31 #define X86_FEATURE_PAE 0x00000040 /* Physical Address Extension is present */ 32 #define X86_FEATURE_CX8 0x00000100 /* CMPXCHG8B instruction present */ 33 #define X86_FEATURE_APIC 0x00000200 /* APIC is present */ 34 #define X86_FEATURE_SYSCALL 0x00000800 /* SYSCALL/SYSRET support present */ 35 #define X86_FEATURE_MTTR 0x00001000 /* Memory type range registers are present */ 36 #define X86_FEATURE_PGE 0x00002000 /* Page Global Enable */ 37 #define X86_FEATURE_CMOV 0x00008000 /* "Conditional move" instruction supported */ 38 #define X86_FEATURE_PAT 0x00010000 /* Page Attribute Table is supported */ 39 #define X86_FEATURE_DS 0x00200000 /* Debug Store is present */ 40 #define X86_FEATURE_MMX 0x00800000 /* MMX extension present */ 41 #define X86_FEATURE_FXSR 0x01000000 /* FXSAVE/FXRSTOR instructions present */ 42 #define X86_FEATURE_SSE 0x02000000 /* SSE extension present */ 43 #define X86_FEATURE_SSE2 0x04000000 /* SSE2 extension present */ 44 #define X86_FEATURE_HT 0x10000000 /* Hyper-Threading present */ 45 46 /* CPUID 0x80000001 - EDX extended flags */ 47 #define X86_FEATURE_NX 0x00100000 /* NX support present */ 48 49 // 50 // One-liners for getting and setting special purpose registers in portable code 51 // 52 FORCEINLINE 53 ULONG_PTR 54 KeGetContextPc(PCONTEXT Context) 55 { 56 return Context->Eip; 57 } 58 59 FORCEINLINE 60 VOID 61 KeSetContextPc(PCONTEXT Context, ULONG_PTR ProgramCounter) 62 { 63 Context->Eip = ProgramCounter; 64 } 65 66 FORCEINLINE 67 ULONG_PTR 68 KeGetContextReturnRegister(PCONTEXT Context) 69 { 70 return Context->Eax; 71 } 72 73 FORCEINLINE 74 VOID 75 KeSetContextReturnRegister(PCONTEXT Context, ULONG_PTR ReturnValue) 76 { 77 Context->Eax = ReturnValue; 78 } 79 80 FORCEINLINE 81 ULONG_PTR 82 KeGetContextFrameRegister(PCONTEXT Context) 83 { 84 return Context->Ebp; 85 } 86 87 FORCEINLINE 88 VOID 89 KeSetContextFrameRegister(PCONTEXT Context, ULONG_PTR Frame) 90 { 91 Context->Ebp = Frame; 92 } 93 94 FORCEINLINE 95 ULONG_PTR 96 KeGetTrapFramePc(PKTRAP_FRAME TrapFrame) 97 { 98 return TrapFrame->Eip; 99 } 100 101 FORCEINLINE 102 PKTRAP_FRAME 103 KiGetLinkedTrapFrame(PKTRAP_FRAME TrapFrame) 104 { 105 return (PKTRAP_FRAME)TrapFrame->Edx; 106 } 107 108 109 FORCEINLINE 110 ULONG_PTR 111 KeGetTrapFrameStackRegister(PKTRAP_FRAME TrapFrame) 112 { 113 if (TrapFrame->PreviousPreviousMode == KernelMode) 114 return TrapFrame->TempEsp; 115 return TrapFrame->HardwareEsp; 116 } 117 118 FORCEINLINE 119 ULONG_PTR 120 KeGetTrapFrameFrameRegister(PKTRAP_FRAME TrapFrame) 121 { 122 return TrapFrame->Ebp; 123 } 124 125 // 126 // Macro to get trap and exception frame from a thread stack 127 // 128 #define KeGetTrapFrame(Thread) \ 129 (PKTRAP_FRAME)((ULONG_PTR)((Thread)->InitialStack) - \ 130 sizeof(KTRAP_FRAME) - \ 131 sizeof(FX_SAVE_AREA)) 132 133 #define KeGetExceptionFrame(Thread) \ 134 NULL 135 136 // 137 // Macro to get context switches from the PRCB 138 // All architectures but x86 have it in the PRCB's KeContextSwitches 139 // 140 #define KeGetContextSwitches(Prcb) \ 141 CONTAINING_RECORD(Prcb, KIPCR, PrcbData)->ContextSwitches 142 143 // 144 // Macro to get the second level cache size field name which differs between 145 // CISC and RISC architectures, as the former has unified I/D cache 146 // 147 #define KiGetSecondLevelDCacheSize() ((PKIPCR)KeGetPcr())->SecondLevelCacheSize 148 149 // 150 // Returns the Interrupt State from a Trap Frame. 151 // ON = TRUE, OFF = FALSE 152 // 153 #define KeGetTrapFrameInterruptState(TrapFrame) \ 154 BooleanFlagOn((TrapFrame)->EFlags, EFLAGS_INTERRUPT_MASK) 155 156 // 157 // Flags for exiting a trap 158 // 159 #define KTE_SKIP_PM_BIT (((KTRAP_EXIT_SKIP_BITS) { { .SkipPreviousMode = TRUE } }).Bits) 160 #define KTE_SKIP_SEG_BIT (((KTRAP_EXIT_SKIP_BITS) { { .SkipSegments = TRUE } }).Bits) 161 #define KTE_SKIP_VOL_BIT (((KTRAP_EXIT_SKIP_BITS) { { .SkipVolatiles = TRUE } }).Bits) 162 163 typedef union _KTRAP_EXIT_SKIP_BITS 164 { 165 struct 166 { 167 UCHAR SkipPreviousMode:1; 168 UCHAR SkipSegments:1; 169 UCHAR SkipVolatiles:1; 170 UCHAR Reserved:5; 171 }; 172 UCHAR Bits; 173 } KTRAP_EXIT_SKIP_BITS, *PKTRAP_EXIT_SKIP_BITS; 174 175 176 // 177 // Flags used by the VDM/V8086 emulation engine for determining instruction prefixes 178 // 179 #define PFX_FLAG_ES 0x00000100 180 #define PFX_FLAG_CS 0x00000200 181 #define PFX_FLAG_SS 0x00000400 182 #define PFX_FLAG_DS 0x00000800 183 #define PFX_FLAG_FS 0x00001000 184 #define PFX_FLAG_GS 0x00002000 185 #define PFX_FLAG_OPER32 0x00004000 186 #define PFX_FLAG_ADDR32 0x00008000 187 #define PFX_FLAG_LOCK 0x00010000 188 #define PFX_FLAG_REPNE 0x00020000 189 #define PFX_FLAG_REP 0x00040000 190 191 // 192 // VDM Helper Macros 193 // 194 // All VDM/V8086 opcode emulators have the same FASTCALL function definition. 195 // We need to keep 2 parameters while the original ASM implementation uses 4: 196 // TrapFrame, PrefixFlags, Eip, InstructionSize; 197 // 198 // We pass the trap frame, and prefix flags, in our two parameters. 199 // 200 // We then realize that since the smallest prefix flag is 0x100, this gives us 201 // a count of up to 0xFF. So we OR in the instruction size with the prefix flags 202 // 203 // We further realize that we always have access to EIP from the trap frame, and 204 // that if we want the *current instruction* EIP, we simply have to add the 205 // instruction size *MINUS ONE*, and that gives us the EIP we should be looking 206 // at now, so we don't need to use the stack to push this parameter. 207 // 208 // We actually only care about the *current instruction* EIP in one location, 209 // so although it may be slightly more expensive to re-calculate the EIP one 210 // more time, this way we don't redefine ALL opcode handlers to have 3 parameters, 211 // which would be forcing stack usage in all other scenarios. 212 // 213 #define KiVdmSetVdmEFlags(x) InterlockedOr((PLONG)KiNtVdmState, (x)); 214 #define KiVdmClearVdmEFlags(x) InterlockedAnd((PLONG)KiNtVdmState, ~(x)) 215 #define KiCallVdmHandler(x) KiVdmOpcode##x(TrapFrame, Flags) 216 #define KiCallVdmPrefixHandler(x) KiVdmOpcodePrefix(TrapFrame, Flags | x) 217 #define KiVdmUnhandledOpcode(x) \ 218 BOOLEAN \ 219 FASTCALL \ 220 KiVdmOpcode##x(IN PKTRAP_FRAME TrapFrame, \ 221 IN ULONG Flags) \ 222 { \ 223 /* Not yet handled */ \ 224 UNIMPLEMENTED_DBGBREAK(); \ 225 return FALSE; \ 226 } 227 228 C_ASSERT(NPX_FRAME_LENGTH == sizeof(FX_SAVE_AREA)); 229 230 // 231 // Local parameters 232 // 233 typedef struct _KV86_FRAME 234 { 235 PVOID ThreadStack; 236 PVOID ThreadTeb; 237 PVOID PcrTeb; 238 } KV86_FRAME, *PKV86_FRAME; 239 240 // 241 // Virtual Stack Frame 242 // 243 typedef struct _KV8086_STACK_FRAME 244 { 245 KTRAP_FRAME TrapFrame; 246 FX_SAVE_AREA NpxArea; 247 KV86_FRAME V86Frame; 248 } KV8086_STACK_FRAME, *PKV8086_STACK_FRAME; 249 250 // 251 // Large Pages Support 252 // 253 typedef struct _LARGE_IDENTITY_MAP 254 { 255 PHARDWARE_PTE TopLevelDirectory; 256 ULONG Cr3; 257 ULONG_PTR StartAddress; 258 ULONG PagesCount; 259 PVOID PagesList[30]; 260 } LARGE_IDENTITY_MAP, *PLARGE_IDENTITY_MAP; 261 262 // 263 // Floating Point Internal Context Structure 264 // 265 typedef struct _FLOATING_SAVE_CONTEXT 266 { 267 PKTHREAD CurrentThread; 268 KIRQL OldNpxIrql; 269 PFX_SAVE_AREA Buffer; 270 PFX_SAVE_AREA PfxSaveArea; 271 } FLOATING_SAVE_CONTEXT, *PFLOATING_SAVE_CONTEXT; 272 273 /* Diable interrupts and return whether they were enabled before */ 274 FORCEINLINE 275 BOOLEAN 276 KeDisableInterrupts(VOID) 277 { 278 ULONG Flags; 279 BOOLEAN Return; 280 281 /* Get EFLAGS and check if the interrupt bit is set */ 282 Flags = __readeflags(); 283 Return = (Flags & EFLAGS_INTERRUPT_MASK) ? TRUE: FALSE; 284 285 /* Disable interrupts */ 286 _disable(); 287 return Return; 288 } 289 290 /* Restore previous interrupt state */ 291 FORCEINLINE 292 VOID 293 KeRestoreInterrupts(BOOLEAN WereEnabled) 294 { 295 if (WereEnabled) _enable(); 296 } 297 298 // 299 // Registers an interrupt handler with an IDT vector 300 // 301 FORCEINLINE 302 VOID 303 KeRegisterInterruptHandler(IN ULONG Vector, 304 IN PVOID Handler) 305 { 306 UCHAR Entry; 307 ULONG_PTR Address; 308 PKIPCR Pcr = (PKIPCR)KeGetPcr(); 309 310 // 311 // Get the entry from the HAL 312 // 313 Entry = HalVectorToIDTEntry(Vector); 314 Address = PtrToUlong(Handler); 315 316 // 317 // Now set the data 318 // 319 Pcr->IDT[Entry].ExtendedOffset = (USHORT)(Address >> 16); 320 Pcr->IDT[Entry].Offset = (USHORT)Address; 321 } 322 323 // 324 // Returns the registered interrupt handler for a given IDT vector 325 // 326 FORCEINLINE 327 PVOID 328 KeQueryInterruptHandler(IN ULONG Vector) 329 { 330 PKIPCR Pcr = (PKIPCR)KeGetPcr(); 331 UCHAR Entry; 332 333 // 334 // Get the entry from the HAL 335 // 336 Entry = HalVectorToIDTEntry(Vector); 337 338 // 339 // Read the entry from the IDT 340 // 341 return (PVOID)(((Pcr->IDT[Entry].ExtendedOffset << 16) & 0xFFFF0000) | 342 (Pcr->IDT[Entry].Offset & 0xFFFF)); 343 } 344 345 // 346 // Invalidates the TLB entry for a specified address 347 // 348 FORCEINLINE 349 VOID 350 KeInvalidateTlbEntry(IN PVOID Address) 351 { 352 /* Invalidate the TLB entry for this address */ 353 __invlpg(Address); 354 } 355 356 FORCEINLINE 357 VOID 358 KeFlushProcessTb(VOID) 359 { 360 /* Flush the TLB by resetting CR3 */ 361 __writecr3(__readcr3()); 362 } 363 364 FORCEINLINE 365 VOID 366 KeSweepICache(IN PVOID BaseAddress, 367 IN SIZE_T FlushSize) 368 { 369 // 370 // Always sweep the whole cache 371 // 372 UNREFERENCED_PARAMETER(BaseAddress); 373 UNREFERENCED_PARAMETER(FlushSize); 374 __wbinvd(); 375 } 376 377 FORCEINLINE 378 PRKTHREAD 379 KeGetCurrentThread(VOID) 380 { 381 /* Return the current thread */ 382 return ((PKIPCR)KeGetPcr())->PrcbData.CurrentThread; 383 } 384 385 FORCEINLINE 386 VOID 387 KiRundownThread(IN PKTHREAD Thread) 388 { 389 #ifndef CONFIG_SMP 390 /* Check if this is the NPX Thread */ 391 if (KeGetCurrentPrcb()->NpxThread == Thread) 392 { 393 /* Clear it */ 394 KeGetCurrentPrcb()->NpxThread = NULL; 395 Ke386FnInit(); 396 } 397 #else 398 /* Nothing to do */ 399 #endif 400 } 401 402 CODE_SEG("INIT") 403 VOID 404 NTAPI 405 KiInitializePcr(IN ULONG ProcessorNumber, 406 IN PKIPCR Pcr, 407 IN PKIDTENTRY Idt, 408 IN PKGDTENTRY Gdt, 409 IN PKTSS Tss, 410 IN PKTHREAD IdleThread, 411 IN PVOID DpcStack); 412 413 FORCEINLINE 414 VOID 415 Ke386SetGdtEntryBase(PKGDTENTRY GdtEntry, PVOID BaseAddress) 416 { 417 GdtEntry->BaseLow = (USHORT)((ULONG_PTR)BaseAddress & 0xFFFF); 418 GdtEntry->HighWord.Bytes.BaseMid = (UCHAR)((ULONG_PTR)BaseAddress >> 16); 419 GdtEntry->HighWord.Bytes.BaseHi = (UCHAR)((ULONG_PTR)BaseAddress >> 24); 420 } 421 422 FORCEINLINE 423 VOID 424 KiSetTebBase(PKPCR Pcr, PNT_TIB TebAddress) 425 { 426 Pcr->NtTib.Self = TebAddress; 427 Ke386SetGdtEntryBase(&Pcr->GDT[KGDT_R3_TEB / sizeof(KGDTENTRY)], TebAddress); 428 } 429 430 CODE_SEG("INIT") 431 VOID 432 FASTCALL 433 Ki386InitializeTss( 434 IN PKTSS Tss, 435 IN PKIDTENTRY Idt, 436 IN PKGDTENTRY Gdt 437 ); 438 439 CODE_SEG("INIT") 440 VOID 441 NTAPI 442 KiSetCR0Bits(VOID); 443 444 CODE_SEG("INIT") 445 VOID 446 NTAPI 447 KiGetCacheInformation(VOID); 448 449 CODE_SEG("INIT") 450 BOOLEAN 451 NTAPI 452 KiIsNpxPresent( 453 VOID 454 ); 455 456 CODE_SEG("INIT") 457 BOOLEAN 458 NTAPI 459 KiIsNpxErrataPresent( 460 VOID 461 ); 462 463 CODE_SEG("INIT") 464 VOID 465 NTAPI 466 KiSetProcessorType(VOID); 467 468 CODE_SEG("INIT") 469 ULONG 470 NTAPI 471 KiGetFeatureBits(VOID); 472 473 #if DBG 474 CODE_SEG("INIT") 475 VOID 476 KiReportCpuFeatures(VOID); 477 #endif 478 479 VOID 480 NTAPI 481 KiThreadStartup(VOID); 482 483 NTSTATUS 484 NTAPI 485 Ke386GetGdtEntryThread( 486 IN PKTHREAD Thread, 487 IN ULONG Offset, 488 IN PKGDTENTRY Descriptor 489 ); 490 491 VOID 492 NTAPI 493 KiFlushNPXState( 494 IN FLOATING_SAVE_AREA *SaveArea 495 ); 496 497 VOID 498 NTAPI 499 Ki386AdjustEsp0( 500 IN PKTRAP_FRAME TrapFrame 501 ); 502 503 VOID 504 NTAPI 505 Ki386SetupAndExitToV86Mode( 506 OUT PTEB VdmTeb 507 ); 508 509 CODE_SEG("INIT") 510 VOID 511 NTAPI 512 KeI386VdmInitialize( 513 VOID 514 ); 515 516 CODE_SEG("INIT") 517 ULONG_PTR 518 NTAPI 519 Ki386EnableGlobalPage( 520 IN ULONG_PTR Context 521 ); 522 523 CODE_SEG("INIT") 524 ULONG_PTR 525 NTAPI 526 Ki386EnableTargetLargePage( 527 IN ULONG_PTR Context 528 ); 529 530 BOOLEAN 531 NTAPI 532 Ki386CreateIdentityMap( 533 IN PLARGE_IDENTITY_MAP IdentityMap, 534 IN PVOID StartPtr, 535 IN ULONG Length 536 ); 537 538 VOID 539 NTAPI 540 Ki386FreeIdentityMap( 541 IN PLARGE_IDENTITY_MAP IdentityMap 542 ); 543 544 VOID 545 NTAPI 546 Ki386EnableCurrentLargePage( 547 IN ULONG_PTR StartAddress, 548 IN ULONG Cr3 549 ); 550 551 CODE_SEG("INIT") 552 VOID 553 NTAPI 554 KiI386PentiumLockErrataFixup( 555 VOID 556 ); 557 558 CODE_SEG("INIT") 559 VOID 560 NTAPI 561 KiInitializePAT( 562 VOID 563 ); 564 565 CODE_SEG("INIT") 566 VOID 567 NTAPI 568 KiInitializeMTRR( 569 IN BOOLEAN FinalCpu 570 ); 571 572 CODE_SEG("INIT") 573 VOID 574 NTAPI 575 KiAmdK6InitializeMTRR( 576 VOID 577 ); 578 579 CODE_SEG("INIT") 580 VOID 581 NTAPI 582 KiRestoreFastSyscallReturnState( 583 VOID 584 ); 585 586 CODE_SEG("INIT") 587 ULONG_PTR 588 NTAPI 589 Ki386EnableDE( 590 IN ULONG_PTR Context 591 ); 592 593 CODE_SEG("INIT") 594 ULONG_PTR 595 NTAPI 596 Ki386EnableFxsr( 597 IN ULONG_PTR Context 598 ); 599 600 CODE_SEG("INIT") 601 ULONG_PTR 602 NTAPI 603 Ki386EnableXMMIExceptions( 604 IN ULONG_PTR Context 605 ); 606 607 BOOLEAN 608 NTAPI 609 VdmDispatchBop( 610 IN PKTRAP_FRAME TrapFrame 611 ); 612 613 BOOLEAN 614 NTAPI 615 VdmDispatchPageFault( 616 _In_ PKTRAP_FRAME TrapFrame 617 ); 618 619 BOOLEAN 620 FASTCALL 621 KiVdmOpcodePrefix( 622 IN PKTRAP_FRAME TrapFrame, 623 IN ULONG Flags 624 ); 625 626 BOOLEAN 627 FASTCALL 628 Ki386HandleOpcodeV86( 629 IN PKTRAP_FRAME TrapFrame 630 ); 631 632 DECLSPEC_NORETURN 633 VOID 634 FASTCALL 635 KiEoiHelper( 636 IN PKTRAP_FRAME TrapFrame 637 ); 638 639 VOID 640 FASTCALL 641 Ki386BiosCallReturnAddress( 642 IN PKTRAP_FRAME TrapFrame 643 ); 644 645 ULONG_PTR 646 FASTCALL 647 KiExitV86Mode( 648 IN PKTRAP_FRAME TrapFrame 649 ); 650 651 DECLSPEC_NORETURN 652 VOID 653 NTAPI 654 KiDispatchExceptionFromTrapFrame( 655 IN NTSTATUS Code, 656 IN ULONG Flags, 657 IN ULONG_PTR Address, 658 IN ULONG ParameterCount, 659 IN ULONG_PTR Parameter1, 660 IN ULONG_PTR Parameter2, 661 IN ULONG_PTR Parameter3, 662 IN PKTRAP_FRAME TrapFrame 663 ); 664 665 NTSTATUS 666 NTAPI 667 KiConvertToGuiThread( 668 VOID 669 ); 670 671 // 672 // Global x86 only Kernel data 673 // 674 extern PVOID Ki386IopmSaveArea; 675 extern ULONG KeI386EFlagsAndMaskV86; 676 extern ULONG KeI386EFlagsOrMaskV86; 677 extern BOOLEAN KeI386VirtualIntExtensions; 678 extern KIDTENTRY KiIdt[MAXIMUM_IDTVECTOR+1]; 679 extern KDESCRIPTOR KiIdtDescriptor; 680 extern BOOLEAN KiI386PentiumLockErrataPresent; 681 extern ULONG KeI386NpxPresent; 682 extern ULONG KeI386XMMIPresent; 683 extern ULONG KeI386FxsrPresent; 684 extern ULONG KiMXCsrMask; 685 extern ULONG KeI386CpuType; 686 extern ULONG KeI386CpuStep; 687 extern ULONG KiFastSystemCallDisable; 688 extern UCHAR KiDebugRegisterTrapOffsets[9]; 689 extern UCHAR KiDebugRegisterContextOffsets[9]; 690 extern VOID __cdecl KiTrap02(VOID); 691 extern VOID __cdecl KiTrap08(VOID); 692 extern VOID __cdecl KiTrap13(VOID); 693 extern VOID __cdecl KiFastCallEntry(VOID); 694 extern VOID NTAPI ExpInterlockedPopEntrySListFault(VOID); 695 extern VOID NTAPI ExpInterlockedPopEntrySListResume(VOID); 696 extern VOID __cdecl CopyParams(VOID); 697 extern VOID __cdecl ReadBatch(VOID); 698 extern CHAR KiSystemCallExitBranch[]; 699 extern CHAR KiSystemCallExit[]; 700 extern CHAR KiSystemCallExit2[]; 701 702 // 703 // Trap Macros 704 // 705 #include "trap_x.h" 706 707 // 708 // Returns a thread's FPU save area 709 // 710 FORCEINLINE 711 PFX_SAVE_AREA 712 KiGetThreadNpxArea(IN PKTHREAD Thread) 713 { 714 ASSERT((ULONG_PTR)Thread->InitialStack % 16 == 0); 715 return (PFX_SAVE_AREA)((ULONG_PTR)Thread->InitialStack - sizeof(FX_SAVE_AREA)); 716 } 717 718 // 719 // Sanitizes a selector 720 // 721 FORCEINLINE 722 ULONG 723 Ke386SanitizeSeg(IN ULONG Cs, 724 IN KPROCESSOR_MODE Mode) 725 { 726 // 727 // Check if we're in kernel-mode, and force CPL 0 if so. 728 // Otherwise, force CPL 3. 729 // 730 return ((Mode == KernelMode) ? 731 (Cs & (0xFFFF & ~RPL_MASK)) : 732 (RPL_MASK | (Cs & 0xFFFF))); 733 } 734 735 // 736 // Sanitizes EFLAGS 737 // 738 FORCEINLINE 739 ULONG 740 Ke386SanitizeFlags(IN ULONG Eflags, 741 IN KPROCESSOR_MODE Mode) 742 { 743 // 744 // Check if we're in kernel-mode, and sanitize EFLAGS if so. 745 // Otherwise, also force interrupt mask on. 746 // 747 return ((Mode == KernelMode) ? 748 (Eflags & (EFLAGS_USER_SANITIZE | EFLAGS_INTERRUPT_MASK)) : 749 (EFLAGS_INTERRUPT_MASK | (Eflags & EFLAGS_USER_SANITIZE))); 750 } 751 752 // 753 // Sanitizes a Debug Register 754 // 755 FORCEINLINE 756 PVOID 757 Ke386SanitizeDr(IN PVOID DrAddress, 758 IN KPROCESSOR_MODE Mode) 759 { 760 // 761 // Check if we're in kernel-mode, and return the address directly if so. 762 // Otherwise, make sure it's not inside the kernel-mode address space. 763 // If it is, then clear the address. 764 // 765 return ((Mode == KernelMode) ? DrAddress : 766 (DrAddress <= MM_HIGHEST_USER_ADDRESS) ? DrAddress : 0); 767 } 768 769 // 770 // Exception with no arguments 771 // 772 FORCEINLINE 773 DECLSPEC_NORETURN 774 VOID 775 KiDispatchException0Args(IN NTSTATUS Code, 776 IN ULONG_PTR Address, 777 IN PKTRAP_FRAME TrapFrame) 778 { 779 /* Helper for exceptions with no arguments */ 780 KiDispatchExceptionFromTrapFrame(Code, 0, Address, 0, 0, 0, 0, TrapFrame); 781 } 782 783 // 784 // Exception with one argument 785 // 786 FORCEINLINE 787 DECLSPEC_NORETURN 788 VOID 789 KiDispatchException1Args(IN NTSTATUS Code, 790 IN ULONG_PTR Address, 791 IN ULONG P1, 792 IN PKTRAP_FRAME TrapFrame) 793 { 794 /* Helper for exceptions with no arguments */ 795 KiDispatchExceptionFromTrapFrame(Code, 0, Address, 1, P1, 0, 0, TrapFrame); 796 } 797 798 // 799 // Exception with two arguments 800 // 801 FORCEINLINE 802 DECLSPEC_NORETURN 803 VOID 804 KiDispatchException2Args(IN NTSTATUS Code, 805 IN ULONG_PTR Address, 806 IN ULONG P1, 807 IN ULONG P2, 808 IN PKTRAP_FRAME TrapFrame) 809 { 810 /* Helper for exceptions with no arguments */ 811 KiDispatchExceptionFromTrapFrame(Code, 0, Address, 2, P1, P2, 0, TrapFrame); 812 } 813 814 // 815 // Performs a system call 816 // 817 NTSTATUS 818 NTAPI 819 KiSystemCallTrampoline(_In_ PVOID Handler, 820 _In_ PVOID Arguments, 821 _In_ ULONG StackBytes); 822 823 824 // 825 // Checks for pending APCs 826 // 827 FORCEINLINE 828 VOID 829 KiCheckForApcDelivery(IN PKTRAP_FRAME TrapFrame) 830 { 831 PKTHREAD Thread; 832 KIRQL OldIrql; 833 834 /* Check for V8086 or user-mode trap */ 835 if ((TrapFrame->EFlags & EFLAGS_V86_MASK) || (KiUserTrap(TrapFrame))) 836 { 837 /* Get the thread */ 838 Thread = KeGetCurrentThread(); 839 while (TRUE) 840 { 841 /* Turn off the alerted state for kernel mode */ 842 Thread->Alerted[KernelMode] = FALSE; 843 844 /* Are there pending user APCs? */ 845 if (!Thread->ApcState.UserApcPending) break; 846 847 /* Raise to APC level and enable interrupts */ 848 OldIrql = KfRaiseIrql(APC_LEVEL); 849 _enable(); 850 851 /* Deliver APCs */ 852 KiDeliverApc(UserMode, NULL, TrapFrame); 853 854 /* Restore IRQL and disable interrupts once again */ 855 KfLowerIrql(OldIrql); 856 _disable(); 857 } 858 } 859 } 860 861 // 862 // Switches from boot loader to initial kernel stack 863 // 864 CODE_SEG("INIT") 865 FORCEINLINE 866 DECLSPEC_NORETURN 867 VOID 868 KiSwitchToBootStack(IN ULONG_PTR InitialStack) 869 { 870 CODE_SEG("INIT") DECLSPEC_NORETURN VOID NTAPI KiSystemStartupBootStack(VOID); 871 872 /* We have to switch to a new stack before continuing kernel initialization */ 873 #ifdef __GNUC__ 874 __asm__ 875 ( 876 "movl %0, %%esp\n\t" 877 "subl %1, %%esp\n\t" 878 "pushl %2\n\t" 879 "jmp _KiSystemStartupBootStack@0" 880 : 881 : "c"(InitialStack), 882 "i"(NPX_FRAME_LENGTH + KTRAP_FRAME_ALIGN + KTRAP_FRAME_LENGTH), 883 "i"(CR0_EM | CR0_TS | CR0_MP), 884 "p"(KiSystemStartupBootStack) 885 : "%esp" 886 ); 887 #elif defined(_MSC_VER) 888 __asm 889 { 890 mov esp, InitialStack 891 sub esp, (NPX_FRAME_LENGTH + KTRAP_FRAME_ALIGN + KTRAP_FRAME_LENGTH) 892 push (CR0_EM | CR0_TS | CR0_MP) 893 jmp KiSystemStartupBootStack 894 } 895 #else 896 #error Unknown Compiler 897 #endif 898 899 UNREACHABLE; 900 } 901 902 // 903 // Emits the iret instruction for C code 904 // 905 FORCEINLINE 906 DECLSPEC_NORETURN 907 VOID 908 KiIret(VOID) 909 { 910 #if defined(__GNUC__) 911 __asm__ __volatile__ 912 ( 913 "iret" 914 ); 915 #elif defined(_MSC_VER) 916 __asm 917 { 918 iretd 919 } 920 #else 921 #error Unsupported compiler 922 #endif 923 UNREACHABLE; 924 } 925 926 // 927 // Normally this is done by the HAL, but on x86 as an optimization, the kernel 928 // initiates the end by calling back into the HAL and exiting the trap here. 929 // 930 FORCEINLINE 931 VOID 932 KiEndInterrupt(IN KIRQL Irql, 933 IN PKTRAP_FRAME TrapFrame) 934 { 935 /* Disable interrupts and end the interrupt */ 936 _disable(); 937 HalEndSystemInterrupt(Irql, TrapFrame); 938 939 /* Exit the interrupt */ 940 KiEoiHelper(TrapFrame); 941 } 942 943 // 944 // PERF Code 945 // 946 FORCEINLINE 947 VOID 948 Ki386PerfEnd(VOID) 949 { 950 extern ULONGLONG BootCyclesEnd, BootCycles; 951 BootCyclesEnd = __rdtsc(); 952 DbgPrint("Boot took %I64u cycles!\n", BootCyclesEnd - BootCycles); 953 DbgPrint("Interrupts: %u System Calls: %u Context Switches: %u\n", 954 KeGetCurrentPrcb()->InterruptCount, 955 KeGetCurrentPrcb()->KeSystemCalls, 956 KeGetContextSwitches(KeGetCurrentPrcb())); 957 } 958 959 FORCEINLINE 960 PULONG 961 KiGetUserModeStackAddress(void) 962 { 963 return &(KeGetCurrentThread()->TrapFrame->HardwareEsp); 964 } 965 966 #ifdef __cplusplus 967 } // extern "C" 968 #endif 969 970 #endif 971