1 #pragma once 2 3 #ifndef __ASM__ 4 5 #include "intrin_i.h" 6 7 // 8 // Thread Dispatcher Header DebugActive Mask 9 // 10 #define DR_MASK(x) (1 << (x)) 11 #define DR_REG_MASK 0x4F 12 13 // 14 // INT3 is 1 byte long 15 // 16 #define KD_BREAKPOINT_TYPE UCHAR 17 #define KD_BREAKPOINT_SIZE sizeof(UCHAR) 18 #define KD_BREAKPOINT_VALUE 0xCC 19 20 // 21 // Macros for getting and setting special purpose registers in portable code 22 // 23 #define KeGetContextPc(Context) \ 24 ((Context)->Eip) 25 26 #define KeSetContextPc(Context, ProgramCounter) \ 27 ((Context)->Eip = (ProgramCounter)) 28 29 #define KeGetTrapFramePc(TrapFrame) \ 30 ((TrapFrame)->Eip) 31 32 #define KiGetLinkedTrapFrame(x) \ 33 (PKTRAP_FRAME)((x)->Edx) 34 35 #define KeGetContextReturnRegister(Context) \ 36 ((Context)->Eax) 37 38 #define KeSetContextReturnRegister(Context, ReturnValue) \ 39 ((Context)->Eax = (ReturnValue)) 40 41 // 42 // Macro to get trap and exception frame from a thread stack 43 // 44 #define KeGetTrapFrame(Thread) \ 45 (PKTRAP_FRAME)((ULONG_PTR)((Thread)->InitialStack) - \ 46 sizeof(KTRAP_FRAME) - \ 47 sizeof(FX_SAVE_AREA)) 48 49 #define KeGetExceptionFrame(Thread) \ 50 NULL 51 52 // 53 // Macro to get context switches from the PRCB 54 // All architectures but x86 have it in the PRCB's KeContextSwitches 55 // 56 #define KeGetContextSwitches(Prcb) \ 57 CONTAINING_RECORD(Prcb, KIPCR, PrcbData)->ContextSwitches 58 59 // 60 // Macro to get the second level cache size field name which differs between 61 // CISC and RISC architectures, as the former has unified I/D cache 62 // 63 #define KiGetSecondLevelDCacheSize() ((PKIPCR)KeGetPcr())->SecondLevelCacheSize 64 65 // 66 // Returns the Interrupt State from a Trap Frame. 67 // ON = TRUE, OFF = FALSE 68 // 69 #define KeGetTrapFrameInterruptState(TrapFrame) \ 70 BooleanFlagOn((TrapFrame)->EFlags, EFLAGS_INTERRUPT_MASK) 71 72 // 73 // Flags for exiting a trap 74 // 75 #define KTE_SKIP_PM_BIT (((KTRAP_EXIT_SKIP_BITS) { { .SkipPreviousMode = TRUE } }).Bits) 76 #define KTE_SKIP_SEG_BIT (((KTRAP_EXIT_SKIP_BITS) { { .SkipSegments = TRUE } }).Bits) 77 #define KTE_SKIP_VOL_BIT (((KTRAP_EXIT_SKIP_BITS) { { .SkipVolatiles = TRUE } }).Bits) 78 79 typedef union _KTRAP_EXIT_SKIP_BITS 80 { 81 struct 82 { 83 UCHAR SkipPreviousMode:1; 84 UCHAR SkipSegments:1; 85 UCHAR SkipVolatiles:1; 86 UCHAR Reserved:5; 87 }; 88 UCHAR Bits; 89 } KTRAP_EXIT_SKIP_BITS, *PKTRAP_EXIT_SKIP_BITS; 90 91 92 // 93 // Flags used by the VDM/V8086 emulation engine for determining instruction prefixes 94 // 95 #define PFX_FLAG_ES 0x00000100 96 #define PFX_FLAG_CS 0x00000200 97 #define PFX_FLAG_SS 0x00000400 98 #define PFX_FLAG_DS 0x00000800 99 #define PFX_FLAG_FS 0x00001000 100 #define PFX_FLAG_GS 0x00002000 101 #define PFX_FLAG_OPER32 0x00004000 102 #define PFX_FLAG_ADDR32 0x00008000 103 #define PFX_FLAG_LOCK 0x00010000 104 #define PFX_FLAG_REPNE 0x00020000 105 #define PFX_FLAG_REP 0x00040000 106 107 // 108 // VDM Helper Macros 109 // 110 // All VDM/V8086 opcode emulators have the same FASTCALL function definition. 111 // We need to keep 2 parameters while the original ASM implementation uses 4: 112 // TrapFrame, PrefixFlags, Eip, InstructionSize; 113 // 114 // We pass the trap frame, and prefix flags, in our two parameters. 115 // 116 // We then realize that since the smallest prefix flag is 0x100, this gives us 117 // a count of up to 0xFF. So we OR in the instruction size with the prefix flags 118 // 119 // We further realize that we always have access to EIP from the trap frame, and 120 // that if we want the *current instruction* EIP, we simply have to add the 121 // instruction size *MINUS ONE*, and that gives us the EIP we should be looking 122 // at now, so we don't need to use the stack to push this parameter. 123 // 124 // We actually only care about the *current instruction* EIP in one location, 125 // so although it may be slightly more expensive to re-calculate the EIP one 126 // more time, this way we don't redefine ALL opcode handlers to have 3 parameters, 127 // which would be forcing stack usage in all other scenarios. 128 // 129 #define KiVdmSetVdmEFlags(x) InterlockedOr((PLONG)KiNtVdmState, (x)); 130 #define KiVdmClearVdmEFlags(x) InterlockedAnd((PLONG)KiNtVdmState, ~(x)) 131 #define KiCallVdmHandler(x) KiVdmOpcode##x(TrapFrame, Flags) 132 #define KiCallVdmPrefixHandler(x) KiVdmOpcodePrefix(TrapFrame, Flags | x) 133 #define KiVdmUnhandledOpcode(x) \ 134 BOOLEAN \ 135 FASTCALL \ 136 KiVdmOpcode##x(IN PKTRAP_FRAME TrapFrame, \ 137 IN ULONG Flags) \ 138 { \ 139 /* Not yet handled */ \ 140 UNIMPLEMENTED_DBGBREAK(); \ 141 return FALSE; \ 142 } 143 144 C_ASSERT(NPX_FRAME_LENGTH == sizeof(FX_SAVE_AREA)); 145 146 // 147 // Local parameters 148 // 149 typedef struct _KV86_FRAME 150 { 151 PVOID ThreadStack; 152 PVOID ThreadTeb; 153 PVOID PcrTeb; 154 } KV86_FRAME, *PKV86_FRAME; 155 156 // 157 // Virtual Stack Frame 158 // 159 typedef struct _KV8086_STACK_FRAME 160 { 161 KTRAP_FRAME TrapFrame; 162 FX_SAVE_AREA NpxArea; 163 KV86_FRAME V86Frame; 164 } KV8086_STACK_FRAME, *PKV8086_STACK_FRAME; 165 166 // 167 // Large Pages Support 168 // 169 typedef struct _LARGE_IDENTITY_MAP 170 { 171 PHARDWARE_PTE TopLevelDirectory; 172 ULONG Cr3; 173 ULONG_PTR StartAddress; 174 ULONG PagesCount; 175 PVOID PagesList[30]; 176 } LARGE_IDENTITY_MAP, *PLARGE_IDENTITY_MAP; 177 178 /* Diable interrupts and return whether they were enabled before */ 179 FORCEINLINE 180 BOOLEAN 181 KeDisableInterrupts(VOID) 182 { 183 ULONG Flags; 184 BOOLEAN Return; 185 186 /* Get EFLAGS and check if the interrupt bit is set */ 187 Flags = __readeflags(); 188 Return = (Flags & EFLAGS_INTERRUPT_MASK) ? TRUE: FALSE; 189 190 /* Disable interrupts */ 191 _disable(); 192 return Return; 193 } 194 195 /* Restore previous interrupt state */ 196 FORCEINLINE 197 VOID 198 KeRestoreInterrupts(BOOLEAN WereEnabled) 199 { 200 if (WereEnabled) _enable(); 201 } 202 203 // 204 // Registers an interrupt handler with an IDT vector 205 // 206 FORCEINLINE 207 VOID 208 KeRegisterInterruptHandler(IN ULONG Vector, 209 IN PVOID Handler) 210 { 211 UCHAR Entry; 212 ULONG_PTR Address; 213 PKIPCR Pcr = (PKIPCR)KeGetPcr(); 214 215 // 216 // Get the entry from the HAL 217 // 218 Entry = HalVectorToIDTEntry(Vector); 219 Address = PtrToUlong(Handler); 220 221 // 222 // Now set the data 223 // 224 Pcr->IDT[Entry].ExtendedOffset = (USHORT)(Address >> 16); 225 Pcr->IDT[Entry].Offset = (USHORT)Address; 226 } 227 228 // 229 // Returns the registered interrupt handler for a given IDT vector 230 // 231 FORCEINLINE 232 PVOID 233 KeQueryInterruptHandler(IN ULONG Vector) 234 { 235 PKIPCR Pcr = (PKIPCR)KeGetPcr(); 236 UCHAR Entry; 237 238 // 239 // Get the entry from the HAL 240 // 241 Entry = HalVectorToIDTEntry(Vector); 242 243 // 244 // Read the entry from the IDT 245 // 246 return (PVOID)(((Pcr->IDT[Entry].ExtendedOffset << 16) & 0xFFFF0000) | 247 (Pcr->IDT[Entry].Offset & 0xFFFF)); 248 } 249 250 // 251 // Invalidates the TLB entry for a specified address 252 // 253 FORCEINLINE 254 VOID 255 KeInvalidateTlbEntry(IN PVOID Address) 256 { 257 /* Invalidate the TLB entry for this address */ 258 __invlpg(Address); 259 } 260 261 FORCEINLINE 262 VOID 263 KeFlushProcessTb(VOID) 264 { 265 /* Flush the TLB by resetting CR3 */ 266 __writecr3(__readcr3()); 267 } 268 269 FORCEINLINE 270 VOID 271 KeSweepICache(IN PVOID BaseAddress, 272 IN SIZE_T FlushSize) 273 { 274 // 275 // Always sweep the whole cache 276 // 277 UNREFERENCED_PARAMETER(BaseAddress); 278 UNREFERENCED_PARAMETER(FlushSize); 279 __wbinvd(); 280 } 281 282 FORCEINLINE 283 PRKTHREAD 284 KeGetCurrentThread(VOID) 285 { 286 /* Return the current thread */ 287 return ((PKIPCR)KeGetPcr())->PrcbData.CurrentThread; 288 } 289 290 FORCEINLINE 291 VOID 292 KiRundownThread(IN PKTHREAD Thread) 293 { 294 #ifndef CONFIG_SMP 295 /* Check if this is the NPX Thread */ 296 if (KeGetCurrentPrcb()->NpxThread == Thread) 297 { 298 /* Clear it */ 299 KeGetCurrentPrcb()->NpxThread = NULL; 300 Ke386FnInit(); 301 } 302 #else 303 /* Nothing to do */ 304 #endif 305 } 306 307 FORCEINLINE 308 VOID 309 Ke386SetGdtEntryBase(PKGDTENTRY GdtEntry, PVOID BaseAddress) 310 { 311 GdtEntry->BaseLow = (USHORT)((ULONG_PTR)BaseAddress & 0xFFFF); 312 GdtEntry->HighWord.Bytes.BaseMid = (UCHAR)((ULONG_PTR)BaseAddress >> 16); 313 GdtEntry->HighWord.Bytes.BaseHi = (UCHAR)((ULONG_PTR)BaseAddress >> 24); 314 } 315 316 FORCEINLINE 317 VOID 318 KiSetTebBase(PKPCR Pcr, PVOID TebAddress) 319 { 320 Pcr->NtTib.Self = TebAddress; 321 Ke386SetGdtEntryBase(&Pcr->GDT[KGDT_R3_TEB / sizeof(KGDTENTRY)], TebAddress); 322 } 323 324 INIT_FUNCTION 325 VOID 326 FASTCALL 327 Ki386InitializeTss( 328 IN PKTSS Tss, 329 IN PKIDTENTRY Idt, 330 IN PKGDTENTRY Gdt 331 ); 332 333 INIT_FUNCTION 334 VOID 335 NTAPI 336 KiSetCR0Bits(VOID); 337 338 INIT_FUNCTION 339 VOID 340 NTAPI 341 KiGetCacheInformation(VOID); 342 343 INIT_FUNCTION 344 BOOLEAN 345 NTAPI 346 KiIsNpxPresent( 347 VOID 348 ); 349 350 INIT_FUNCTION 351 BOOLEAN 352 NTAPI 353 KiIsNpxErrataPresent( 354 VOID 355 ); 356 357 INIT_FUNCTION 358 VOID 359 NTAPI 360 KiSetProcessorType(VOID); 361 362 INIT_FUNCTION 363 ULONG 364 NTAPI 365 KiGetFeatureBits(VOID); 366 367 VOID 368 NTAPI 369 KiThreadStartup(VOID); 370 371 NTSTATUS 372 NTAPI 373 Ke386GetGdtEntryThread( 374 IN PKTHREAD Thread, 375 IN ULONG Offset, 376 IN PKGDTENTRY Descriptor 377 ); 378 379 VOID 380 NTAPI 381 KiFlushNPXState( 382 IN FLOATING_SAVE_AREA *SaveArea 383 ); 384 385 VOID 386 NTAPI 387 Ki386AdjustEsp0( 388 IN PKTRAP_FRAME TrapFrame 389 ); 390 391 VOID 392 NTAPI 393 Ki386SetupAndExitToV86Mode( 394 OUT PTEB VdmTeb 395 ); 396 397 INIT_FUNCTION 398 VOID 399 NTAPI 400 KeI386VdmInitialize( 401 VOID 402 ); 403 404 INIT_FUNCTION 405 ULONG_PTR 406 NTAPI 407 Ki386EnableGlobalPage( 408 IN ULONG_PTR Context 409 ); 410 411 INIT_FUNCTION 412 ULONG_PTR 413 NTAPI 414 Ki386EnableTargetLargePage( 415 IN ULONG_PTR Context 416 ); 417 418 BOOLEAN 419 NTAPI 420 Ki386CreateIdentityMap( 421 IN PLARGE_IDENTITY_MAP IdentityMap, 422 IN PVOID StartPtr, 423 IN ULONG Length 424 ); 425 426 VOID 427 NTAPI 428 Ki386FreeIdentityMap( 429 IN PLARGE_IDENTITY_MAP IdentityMap 430 ); 431 432 VOID 433 NTAPI 434 Ki386EnableCurrentLargePage( 435 IN ULONG_PTR StartAddress, 436 IN ULONG Cr3 437 ); 438 439 INIT_FUNCTION 440 VOID 441 NTAPI 442 KiI386PentiumLockErrataFixup( 443 VOID 444 ); 445 446 INIT_FUNCTION 447 VOID 448 NTAPI 449 KiInitializePAT( 450 VOID 451 ); 452 453 INIT_FUNCTION 454 VOID 455 NTAPI 456 KiInitializeMTRR( 457 IN BOOLEAN FinalCpu 458 ); 459 460 INIT_FUNCTION 461 VOID 462 NTAPI 463 KiAmdK6InitializeMTRR( 464 VOID 465 ); 466 467 INIT_FUNCTION 468 VOID 469 NTAPI 470 KiRestoreFastSyscallReturnState( 471 VOID 472 ); 473 474 INIT_FUNCTION 475 ULONG_PTR 476 NTAPI 477 Ki386EnableDE( 478 IN ULONG_PTR Context 479 ); 480 481 INIT_FUNCTION 482 ULONG_PTR 483 NTAPI 484 Ki386EnableFxsr( 485 IN ULONG_PTR Context 486 ); 487 488 INIT_FUNCTION 489 ULONG_PTR 490 NTAPI 491 Ki386EnableXMMIExceptions( 492 IN ULONG_PTR Context 493 ); 494 495 BOOLEAN 496 NTAPI 497 VdmDispatchBop( 498 IN PKTRAP_FRAME TrapFrame 499 ); 500 501 BOOLEAN 502 NTAPI 503 VdmDispatchPageFault( 504 _In_ PKTRAP_FRAME TrapFrame 505 ); 506 507 BOOLEAN 508 FASTCALL 509 KiVdmOpcodePrefix( 510 IN PKTRAP_FRAME TrapFrame, 511 IN ULONG Flags 512 ); 513 514 BOOLEAN 515 FASTCALL 516 Ki386HandleOpcodeV86( 517 IN PKTRAP_FRAME TrapFrame 518 ); 519 520 DECLSPEC_NORETURN 521 VOID 522 FASTCALL 523 KiEoiHelper( 524 IN PKTRAP_FRAME TrapFrame 525 ); 526 527 VOID 528 FASTCALL 529 Ki386BiosCallReturnAddress( 530 IN PKTRAP_FRAME TrapFrame 531 ); 532 533 ULONG_PTR 534 FASTCALL 535 KiExitV86Mode( 536 IN PKTRAP_FRAME TrapFrame 537 ); 538 539 DECLSPEC_NORETURN 540 VOID 541 NTAPI 542 KiDispatchExceptionFromTrapFrame( 543 IN NTSTATUS Code, 544 IN ULONG Flags, 545 IN ULONG_PTR Address, 546 IN ULONG ParameterCount, 547 IN ULONG_PTR Parameter1, 548 IN ULONG_PTR Parameter2, 549 IN ULONG_PTR Parameter3, 550 IN PKTRAP_FRAME TrapFrame 551 ); 552 553 NTSTATUS 554 NTAPI 555 KiConvertToGuiThread( 556 VOID 557 ); 558 559 // 560 // Global x86 only Kernel data 561 // 562 extern PVOID Ki386IopmSaveArea; 563 extern ULONG KeI386EFlagsAndMaskV86; 564 extern ULONG KeI386EFlagsOrMaskV86; 565 extern BOOLEAN KeI386VirtualIntExtensions; 566 extern KIDTENTRY KiIdt[MAXIMUM_IDTVECTOR+1]; 567 extern KDESCRIPTOR KiIdtDescriptor; 568 extern BOOLEAN KiI386PentiumLockErrataPresent; 569 extern ULONG KeI386NpxPresent; 570 extern ULONG KeI386XMMIPresent; 571 extern ULONG KeI386FxsrPresent; 572 extern ULONG KiMXCsrMask; 573 extern ULONG KeI386CpuType; 574 extern ULONG KeI386CpuStep; 575 extern ULONG KiFastSystemCallDisable; 576 extern UCHAR KiDebugRegisterTrapOffsets[9]; 577 extern UCHAR KiDebugRegisterContextOffsets[9]; 578 extern DECLSPEC_NORETURN VOID __cdecl KiTrap02(VOID); 579 extern VOID __cdecl KiTrap08(VOID); 580 extern VOID __cdecl KiTrap13(VOID); 581 extern VOID __cdecl KiFastCallEntry(VOID); 582 extern VOID NTAPI ExpInterlockedPopEntrySListFault(VOID); 583 extern VOID NTAPI ExpInterlockedPopEntrySListResume(VOID); 584 extern VOID __cdecl CopyParams(VOID); 585 extern VOID __cdecl ReadBatch(VOID); 586 extern CHAR KiSystemCallExitBranch[]; 587 extern CHAR KiSystemCallExit[]; 588 extern CHAR KiSystemCallExit2[]; 589 590 // 591 // Trap Macros 592 // 593 #include "trap_x.h" 594 595 // 596 // Returns a thread's FPU save area 597 // 598 FORCEINLINE 599 PFX_SAVE_AREA 600 KiGetThreadNpxArea(IN PKTHREAD Thread) 601 { 602 ASSERT((ULONG_PTR)Thread->InitialStack % 16 == 0); 603 return (PFX_SAVE_AREA)((ULONG_PTR)Thread->InitialStack - sizeof(FX_SAVE_AREA)); 604 } 605 606 // 607 // Sanitizes a selector 608 // 609 FORCEINLINE 610 ULONG 611 Ke386SanitizeSeg(IN ULONG Cs, 612 IN KPROCESSOR_MODE Mode) 613 { 614 // 615 // Check if we're in kernel-mode, and force CPL 0 if so. 616 // Otherwise, force CPL 3. 617 // 618 return ((Mode == KernelMode) ? 619 (Cs & (0xFFFF & ~RPL_MASK)) : 620 (RPL_MASK | (Cs & 0xFFFF))); 621 } 622 623 // 624 // Sanitizes EFLAGS 625 // 626 FORCEINLINE 627 ULONG 628 Ke386SanitizeFlags(IN ULONG Eflags, 629 IN KPROCESSOR_MODE Mode) 630 { 631 // 632 // Check if we're in kernel-mode, and sanitize EFLAGS if so. 633 // Otherwise, also force interrupt mask on. 634 // 635 return ((Mode == KernelMode) ? 636 (Eflags & (EFLAGS_USER_SANITIZE | EFLAGS_INTERRUPT_MASK)) : 637 (EFLAGS_INTERRUPT_MASK | (Eflags & EFLAGS_USER_SANITIZE))); 638 } 639 640 // 641 // Sanitizes a Debug Register 642 // 643 FORCEINLINE 644 PVOID 645 Ke386SanitizeDr(IN PVOID DrAddress, 646 IN KPROCESSOR_MODE Mode) 647 { 648 // 649 // Check if we're in kernel-mode, and return the address directly if so. 650 // Otherwise, make sure it's not inside the kernel-mode address space. 651 // If it is, then clear the address. 652 // 653 return ((Mode == KernelMode) ? DrAddress : 654 (DrAddress <= MM_HIGHEST_USER_ADDRESS) ? DrAddress : 0); 655 } 656 657 // 658 // Exception with no arguments 659 // 660 FORCEINLINE 661 DECLSPEC_NORETURN 662 VOID 663 KiDispatchException0Args(IN NTSTATUS Code, 664 IN ULONG_PTR Address, 665 IN PKTRAP_FRAME TrapFrame) 666 { 667 /* Helper for exceptions with no arguments */ 668 KiDispatchExceptionFromTrapFrame(Code, 0, Address, 0, 0, 0, 0, TrapFrame); 669 } 670 671 // 672 // Exception with one argument 673 // 674 FORCEINLINE 675 DECLSPEC_NORETURN 676 VOID 677 KiDispatchException1Args(IN NTSTATUS Code, 678 IN ULONG_PTR Address, 679 IN ULONG P1, 680 IN PKTRAP_FRAME TrapFrame) 681 { 682 /* Helper for exceptions with no arguments */ 683 KiDispatchExceptionFromTrapFrame(Code, 0, Address, 1, P1, 0, 0, TrapFrame); 684 } 685 686 // 687 // Exception with two arguments 688 // 689 FORCEINLINE 690 DECLSPEC_NORETURN 691 VOID 692 KiDispatchException2Args(IN NTSTATUS Code, 693 IN ULONG_PTR Address, 694 IN ULONG P1, 695 IN ULONG P2, 696 IN PKTRAP_FRAME TrapFrame) 697 { 698 /* Helper for exceptions with no arguments */ 699 KiDispatchExceptionFromTrapFrame(Code, 0, Address, 2, P1, P2, 0, TrapFrame); 700 } 701 702 // 703 // Performs a system call 704 // 705 706 /* 707 * This sequence does a RtlCopyMemory(Stack - StackBytes, Arguments, StackBytes) 708 * and then calls the function associated with the system call. 709 * 710 * It's done in assembly for two reasons: we need to muck with the stack, 711 * and the call itself restores the stack back for us. The only way to do 712 * this in C is to do manual C handlers for every possible number of args on 713 * the stack, and then have the handler issue a call by pointer. This is 714 * wasteful since it'll basically push the values twice and require another 715 * level of call indirection. 716 * 717 * The ARM kernel currently does this, but it should probably be changed 718 * later to function like this as well. 719 * 720 */ 721 #ifdef __GNUC__ 722 FORCEINLINE 723 NTSTATUS 724 KiSystemCallTrampoline(IN PVOID Handler, 725 IN PVOID Arguments, 726 IN ULONG StackBytes) 727 { 728 NTSTATUS Result; 729 730 __asm__ __volatile__ 731 ( 732 "subl %1, %%esp\n\t" 733 "movl %%esp, %%edi\n\t" 734 "movl %2, %%esi\n\t" 735 "shrl $2, %1\n\t" 736 "rep movsd\n\t" 737 "call *%3\n\t" 738 "movl %%eax, %0" 739 : "=r"(Result) 740 : "c"(StackBytes), 741 "d"(Arguments), 742 "r"(Handler) 743 : "%esp", "%esi", "%edi" 744 ); 745 return Result; 746 } 747 #elif defined(_MSC_VER) 748 FORCEINLINE 749 NTSTATUS 750 KiSystemCallTrampoline(IN PVOID Handler, 751 IN PVOID Arguments, 752 IN ULONG StackBytes) 753 { 754 __asm 755 { 756 mov ecx, StackBytes 757 mov esi, Arguments 758 mov eax, Handler 759 sub esp, ecx 760 mov edi, esp 761 shr ecx, 2 762 rep movsd 763 call eax 764 } 765 /* Return with result in EAX */ 766 } 767 #else 768 #error Unknown Compiler 769 #endif 770 771 772 // 773 // Checks for pending APCs 774 // 775 FORCEINLINE 776 VOID 777 KiCheckForApcDelivery(IN PKTRAP_FRAME TrapFrame) 778 { 779 PKTHREAD Thread; 780 KIRQL OldIrql; 781 782 /* Check for V8086 or user-mode trap */ 783 if ((TrapFrame->EFlags & EFLAGS_V86_MASK) || (KiUserTrap(TrapFrame))) 784 { 785 /* Get the thread */ 786 Thread = KeGetCurrentThread(); 787 while (TRUE) 788 { 789 /* Turn off the alerted state for kernel mode */ 790 Thread->Alerted[KernelMode] = FALSE; 791 792 /* Are there pending user APCs? */ 793 if (!Thread->ApcState.UserApcPending) break; 794 795 /* Raise to APC level and enable interrupts */ 796 OldIrql = KfRaiseIrql(APC_LEVEL); 797 _enable(); 798 799 /* Deliver APCs */ 800 KiDeliverApc(UserMode, NULL, TrapFrame); 801 802 /* Restore IRQL and disable interrupts once again */ 803 KfLowerIrql(OldIrql); 804 _disable(); 805 } 806 } 807 } 808 809 // 810 // Switches from boot loader to initial kernel stack 811 // 812 INIT_FUNCTION 813 FORCEINLINE 814 VOID 815 KiSwitchToBootStack(IN ULONG_PTR InitialStack) 816 { 817 INIT_FUNCTION VOID NTAPI KiSystemStartupBootStack(VOID); 818 819 /* We have to switch to a new stack before continuing kernel initialization */ 820 #ifdef __GNUC__ 821 __asm__ 822 ( 823 "movl %0, %%esp\n\t" 824 "subl %1, %%esp\n\t" 825 "pushl %2\n\t" 826 "jmp _KiSystemStartupBootStack@0" 827 : 828 : "c"(InitialStack), 829 "i"(NPX_FRAME_LENGTH + KTRAP_FRAME_ALIGN + KTRAP_FRAME_LENGTH), 830 "i"(CR0_EM | CR0_TS | CR0_MP), 831 "p"(KiSystemStartupBootStack) 832 : "%esp" 833 ); 834 #elif defined(_MSC_VER) 835 __asm 836 { 837 mov esp, InitialStack 838 sub esp, (NPX_FRAME_LENGTH + KTRAP_FRAME_ALIGN + KTRAP_FRAME_LENGTH) 839 push (CR0_EM | CR0_TS | CR0_MP) 840 jmp KiSystemStartupBootStack 841 } 842 #else 843 #error Unknown Compiler 844 #endif 845 } 846 847 // 848 // Emits the iret instruction for C code 849 // 850 FORCEINLINE 851 DECLSPEC_NORETURN 852 VOID 853 KiIret(VOID) 854 { 855 #if defined(__GNUC__) 856 __asm__ __volatile__ 857 ( 858 "iret" 859 ); 860 #elif defined(_MSC_VER) 861 __asm 862 { 863 iretd 864 } 865 #else 866 #error Unsupported compiler 867 #endif 868 UNREACHABLE; 869 } 870 871 // 872 // Normally this is done by the HAL, but on x86 as an optimization, the kernel 873 // initiates the end by calling back into the HAL and exiting the trap here. 874 // 875 FORCEINLINE 876 VOID 877 KiEndInterrupt(IN KIRQL Irql, 878 IN PKTRAP_FRAME TrapFrame) 879 { 880 /* Disable interrupts and end the interrupt */ 881 _disable(); 882 HalEndSystemInterrupt(Irql, TrapFrame); 883 884 /* Exit the interrupt */ 885 KiEoiHelper(TrapFrame); 886 } 887 888 // 889 // PERF Code 890 // 891 FORCEINLINE 892 VOID 893 Ki386PerfEnd(VOID) 894 { 895 extern ULONGLONG BootCyclesEnd, BootCycles; 896 BootCyclesEnd = __rdtsc(); 897 DbgPrint("Boot took %I64u cycles!\n", BootCyclesEnd - BootCycles); 898 DbgPrint("Interrupts: %u System Calls: %u Context Switches: %u\n", 899 KeGetCurrentPrcb()->InterruptCount, 900 KeGetCurrentPrcb()->KeSystemCalls, 901 KeGetContextSwitches(KeGetCurrentPrcb())); 902 } 903 904 FORCEINLINE 905 PULONG 906 KiGetUserModeStackAddress(void) 907 { 908 return &(KeGetCurrentThread()->TrapFrame->HardwareEsp); 909 } 910 911 #endif 912