xref: /reactos/ntoskrnl/include/internal/i386/ke.h (revision bbccad0e) 
1 #pragma once
2 
3 #ifndef __ASM__
4 
5 #include "intrin_i.h"
6 
7 #ifdef __cplusplus
8 extern "C"
9 {
10 #endif
11 
12 //
13 // Thread Dispatcher Header DebugActive Mask
14 //
15 #define DR_MASK(x)                              (1 << (x))
16 #define DR_REG_MASK                             0x4F
17 
18 //
19 // INT3 is 1 byte long
20 //
21 #define KD_BREAKPOINT_TYPE        UCHAR
22 #define KD_BREAKPOINT_SIZE        sizeof(UCHAR)
23 #define KD_BREAKPOINT_VALUE       0xCC
24 
25 //
26 // One-liners for getting and setting special purpose registers in portable code
27 //
28 FORCEINLINE
29 ULONG_PTR
30 KeGetContextPc(PCONTEXT Context)
31 {
32     return Context->Eip;
33 }
34 
35 FORCEINLINE
36 VOID
37 KeSetContextPc(PCONTEXT Context, ULONG_PTR ProgramCounter)
38 {
39     Context->Eip = ProgramCounter;
40 }
41 
42 FORCEINLINE
43 ULONG_PTR
44 KeGetContextReturnRegister(PCONTEXT Context)
45 {
46     return Context->Eax;
47 }
48 
49 FORCEINLINE
50 VOID
51 KeSetContextReturnRegister(PCONTEXT Context, ULONG_PTR ReturnValue)
52 {
53     Context->Eax = ReturnValue;
54 }
55 
56 FORCEINLINE
57 ULONG_PTR
58 KeGetContextFrameRegister(PCONTEXT Context)
59 {
60     return Context->Ebp;
61 }
62 
63 FORCEINLINE
64 VOID
65 KeSetContextFrameRegister(PCONTEXT Context, ULONG_PTR Frame)
66 {
67     Context->Ebp = Frame;
68 }
69 
70 FORCEINLINE
71 ULONG_PTR
72 KeGetTrapFramePc(PKTRAP_FRAME TrapFrame)
73 {
74     return TrapFrame->Eip;
75 }
76 
77 FORCEINLINE
78 PKTRAP_FRAME
79 KiGetLinkedTrapFrame(PKTRAP_FRAME TrapFrame)
80 {
81     return (PKTRAP_FRAME)TrapFrame->Edx;
82 }
83 
84 
85 FORCEINLINE
86 ULONG_PTR
87 KeGetTrapFrameStackRegister(PKTRAP_FRAME TrapFrame)
88 {
89     if (TrapFrame->PreviousPreviousMode == KernelMode)
90         return TrapFrame->TempEsp;
91     return TrapFrame->HardwareEsp;
92 }
93 
94 FORCEINLINE
95 ULONG_PTR
96 KeGetTrapFrameFrameRegister(PKTRAP_FRAME TrapFrame)
97 {
98     return TrapFrame->Ebp;
99 }
100 
101 //
102 // Macro to get trap and exception frame from a thread stack
103 //
104 #define KeGetTrapFrame(Thread) \
105     (PKTRAP_FRAME)((ULONG_PTR)((Thread)->InitialStack) - \
106                    sizeof(KTRAP_FRAME) - \
107                    sizeof(FX_SAVE_AREA))
108 
109 #define KeGetExceptionFrame(Thread) \
110     NULL
111 
112 //
113 // Macro to get context switches from the PRCB
114 // All architectures but x86 have it in the PRCB's KeContextSwitches
115 //
116 #define KeGetContextSwitches(Prcb)  \
117     CONTAINING_RECORD(Prcb, KIPCR, PrcbData)->ContextSwitches
118 
119 //
120 // Macro to get the second level cache size field name which differs between
121 // CISC and RISC architectures, as the former has unified I/D cache
122 //
123 #define KiGetSecondLevelDCacheSize() ((PKIPCR)KeGetPcr())->SecondLevelCacheSize
124 
125 //
126 // Returns the Interrupt State from a Trap Frame.
127 // ON = TRUE, OFF = FALSE
128 //
129 #define KeGetTrapFrameInterruptState(TrapFrame) \
130         BooleanFlagOn((TrapFrame)->EFlags, EFLAGS_INTERRUPT_MASK)
131 
132 //
133 // Flags for exiting a trap
134 //
135 #define KTE_SKIP_PM_BIT  (((KTRAP_EXIT_SKIP_BITS) { { .SkipPreviousMode = TRUE } }).Bits)
136 #define KTE_SKIP_SEG_BIT (((KTRAP_EXIT_SKIP_BITS) { { .SkipSegments  = TRUE } }).Bits)
137 #define KTE_SKIP_VOL_BIT (((KTRAP_EXIT_SKIP_BITS) { { .SkipVolatiles = TRUE } }).Bits)
138 
139 typedef union _KTRAP_EXIT_SKIP_BITS
140 {
141     struct
142     {
143         UCHAR SkipPreviousMode:1;
144         UCHAR SkipSegments:1;
145         UCHAR SkipVolatiles:1;
146         UCHAR Reserved:5;
147     };
148     UCHAR Bits;
149 } KTRAP_EXIT_SKIP_BITS, *PKTRAP_EXIT_SKIP_BITS;
150 
151 
152 //
153 // Flags used by the VDM/V8086 emulation engine for determining instruction prefixes
154 //
155 #define PFX_FLAG_ES                0x00000100
156 #define PFX_FLAG_CS                0x00000200
157 #define PFX_FLAG_SS                0x00000400
158 #define PFX_FLAG_DS                0x00000800
159 #define PFX_FLAG_FS                0x00001000
160 #define PFX_FLAG_GS                0x00002000
161 #define PFX_FLAG_OPER32            0x00004000
162 #define PFX_FLAG_ADDR32            0x00008000
163 #define PFX_FLAG_LOCK              0x00010000
164 #define PFX_FLAG_REPNE             0x00020000
165 #define PFX_FLAG_REP               0x00040000
166 
167 //
168 // VDM Helper Macros
169 //
170 // All VDM/V8086 opcode emulators have the same FASTCALL function definition.
171 // We need to keep 2 parameters while the original ASM implementation uses 4:
172 // TrapFrame, PrefixFlags, Eip, InstructionSize;
173 //
174 // We pass the trap frame, and prefix flags, in our two parameters.
175 //
176 // We then realize that since the smallest prefix flag is 0x100, this gives us
177 // a count of up to 0xFF. So we OR in the instruction size with the prefix flags
178 //
179 // We further realize that we always have access to EIP from the trap frame, and
180 // that if we want the *current instruction* EIP, we simply have to add the
181 // instruction size *MINUS ONE*, and that gives us the EIP we should be looking
182 // at now, so we don't need to use the stack to push this parameter.
183 //
184 // We actually only care about the *current instruction* EIP in one location,
185 // so although it may be slightly more expensive to re-calculate the EIP one
186 // more time, this way we don't redefine ALL opcode handlers to have 3 parameters,
187 // which would be forcing stack usage in all other scenarios.
188 //
189 #define KiVdmSetVdmEFlags(x)        InterlockedOr((PLONG)KiNtVdmState, (x));
190 #define KiVdmClearVdmEFlags(x)      InterlockedAnd((PLONG)KiNtVdmState, ~(x))
191 #define KiCallVdmHandler(x)         KiVdmOpcode##x(TrapFrame, Flags)
192 #define KiCallVdmPrefixHandler(x)   KiVdmOpcodePrefix(TrapFrame, Flags | x)
193 #define KiVdmUnhandledOpcode(x)                     \
194     BOOLEAN                                         \
195     FASTCALL                                        \
196     KiVdmOpcode##x(IN PKTRAP_FRAME TrapFrame,       \
197                    IN ULONG Flags)                  \
198     {                                               \
199         /* Not yet handled */                       \
200         UNIMPLEMENTED_DBGBREAK();                   \
201         return FALSE;                               \
202     }
203 
204 C_ASSERT(NPX_FRAME_LENGTH == sizeof(FX_SAVE_AREA));
205 
206 //
207 // Local parameters
208 //
209 typedef struct _KV86_FRAME
210 {
211     PVOID ThreadStack;
212     PVOID ThreadTeb;
213     PVOID PcrTeb;
214 } KV86_FRAME, *PKV86_FRAME;
215 
216 //
217 // Virtual Stack Frame
218 //
219 typedef struct _KV8086_STACK_FRAME
220 {
221     KTRAP_FRAME TrapFrame;
222     FX_SAVE_AREA NpxArea;
223     KV86_FRAME V86Frame;
224 } KV8086_STACK_FRAME, *PKV8086_STACK_FRAME;
225 
226 //
227 // Large Pages Support
228 //
229 typedef struct _LARGE_IDENTITY_MAP
230 {
231     PHARDWARE_PTE TopLevelDirectory;
232     ULONG Cr3;
233     ULONG_PTR StartAddress;
234     ULONG PagesCount;
235     PVOID PagesList[30];
236 } LARGE_IDENTITY_MAP, *PLARGE_IDENTITY_MAP;
237 
238 //
239 // Floating Point Internal Context Structure
240 //
241 typedef struct _FLOATING_SAVE_CONTEXT
242 {
243     PKTHREAD CurrentThread;
244     KIRQL OldNpxIrql;
245     PFX_SAVE_AREA Buffer;
246     PFX_SAVE_AREA PfxSaveArea;
247 } FLOATING_SAVE_CONTEXT, *PFLOATING_SAVE_CONTEXT;
248 
249 /* Diable interrupts and return whether they were enabled before */
250 FORCEINLINE
251 BOOLEAN
252 KeDisableInterrupts(VOID)
253 {
254     ULONG Flags;
255     BOOLEAN Return;
256 
257     /* Get EFLAGS and check if the interrupt bit is set */
258     Flags = __readeflags();
259     Return = (Flags & EFLAGS_INTERRUPT_MASK) ? TRUE: FALSE;
260 
261     /* Disable interrupts */
262     _disable();
263     return Return;
264 }
265 
266 /* Restore previous interrupt state */
267 FORCEINLINE
268 VOID
269 KeRestoreInterrupts(BOOLEAN WereEnabled)
270 {
271     if (WereEnabled) _enable();
272 }
273 
274 //
275 // Registers an interrupt handler with an IDT vector
276 //
277 FORCEINLINE
278 VOID
279 KeRegisterInterruptHandler(IN ULONG Vector,
280                            IN PVOID Handler)
281 {
282     UCHAR Entry;
283     ULONG_PTR Address;
284     PKIPCR Pcr = (PKIPCR)KeGetPcr();
285 
286     //
287     // Get the entry from the HAL
288     //
289     Entry = HalVectorToIDTEntry(Vector);
290     Address = PtrToUlong(Handler);
291 
292     //
293     // Now set the data
294     //
295     Pcr->IDT[Entry].ExtendedOffset = (USHORT)(Address >> 16);
296     Pcr->IDT[Entry].Offset = (USHORT)Address;
297 }
298 
299 //
300 // Returns the registered interrupt handler for a given IDT vector
301 //
302 FORCEINLINE
303 PVOID
304 KeQueryInterruptHandler(IN ULONG Vector)
305 {
306     PKIPCR Pcr = (PKIPCR)KeGetPcr();
307     UCHAR Entry;
308 
309     //
310     // Get the entry from the HAL
311     //
312     Entry = HalVectorToIDTEntry(Vector);
313 
314     //
315     // Read the entry from the IDT
316     //
317     return (PVOID)(((Pcr->IDT[Entry].ExtendedOffset << 16) & 0xFFFF0000) |
318                     (Pcr->IDT[Entry].Offset & 0xFFFF));
319 }
320 
321 //
322 // Invalidates the TLB entry for a specified address
323 //
324 FORCEINLINE
325 VOID
326 KeInvalidateTlbEntry(IN PVOID Address)
327 {
328     /* Invalidate the TLB entry for this address */
329     __invlpg(Address);
330 }
331 
332 FORCEINLINE
333 VOID
334 KeFlushProcessTb(VOID)
335 {
336     /* Flush the TLB by resetting CR3 */
337     __writecr3(__readcr3());
338 }
339 
340 FORCEINLINE
341 VOID
342 KeSweepICache(IN PVOID BaseAddress,
343               IN SIZE_T FlushSize)
344 {
345     //
346     // Always sweep the whole cache
347     //
348     UNREFERENCED_PARAMETER(BaseAddress);
349     UNREFERENCED_PARAMETER(FlushSize);
350     __wbinvd();
351 }
352 
353 FORCEINLINE
354 PRKTHREAD
355 KeGetCurrentThread(VOID)
356 {
357     /* Return the current thread */
358     return ((PKIPCR)KeGetPcr())->PrcbData.CurrentThread;
359 }
360 
361 FORCEINLINE
362 VOID
363 KiRundownThread(IN PKTHREAD Thread)
364 {
365 #ifndef CONFIG_SMP
366     /* Check if this is the NPX Thread */
367     if (KeGetCurrentPrcb()->NpxThread == Thread)
368     {
369         /* Clear it */
370         KeGetCurrentPrcb()->NpxThread = NULL;
371         Ke386FnInit();
372     }
373 #else
374     /* Nothing to do */
375 #endif
376 }
377 
378 FORCEINLINE
379 VOID
380 Ke386SetGdtEntryBase(PKGDTENTRY GdtEntry, PVOID BaseAddress)
381 {
382     GdtEntry->BaseLow = (USHORT)((ULONG_PTR)BaseAddress & 0xFFFF);
383     GdtEntry->HighWord.Bytes.BaseMid = (UCHAR)((ULONG_PTR)BaseAddress >> 16);
384     GdtEntry->HighWord.Bytes.BaseHi = (UCHAR)((ULONG_PTR)BaseAddress >> 24);
385 }
386 
387 FORCEINLINE
388 VOID
389 KiSetTebBase(PKPCR Pcr, PNT_TIB TebAddress)
390 {
391     Pcr->NtTib.Self = TebAddress;
392     Ke386SetGdtEntryBase(&Pcr->GDT[KGDT_R3_TEB / sizeof(KGDTENTRY)], TebAddress);
393 }
394 
395 CODE_SEG("INIT")
396 VOID
397 FASTCALL
398 Ki386InitializeTss(
399     IN PKTSS Tss,
400     IN PKIDTENTRY Idt,
401     IN PKGDTENTRY Gdt
402 );
403 
404 CODE_SEG("INIT")
405 VOID
406 NTAPI
407 KiSetCR0Bits(VOID);
408 
409 CODE_SEG("INIT")
410 VOID
411 NTAPI
412 KiGetCacheInformation(VOID);
413 
414 CODE_SEG("INIT")
415 BOOLEAN
416 NTAPI
417 KiIsNpxPresent(
418     VOID
419 );
420 
421 CODE_SEG("INIT")
422 BOOLEAN
423 NTAPI
424 KiIsNpxErrataPresent(
425     VOID
426 );
427 
428 CODE_SEG("INIT")
429 VOID
430 NTAPI
431 KiSetProcessorType(VOID);
432 
433 CODE_SEG("INIT")
434 ULONG
435 NTAPI
436 KiGetFeatureBits(VOID);
437 
438 VOID
439 NTAPI
440 KiThreadStartup(VOID);
441 
442 NTSTATUS
443 NTAPI
444 Ke386GetGdtEntryThread(
445     IN PKTHREAD Thread,
446     IN ULONG Offset,
447     IN PKGDTENTRY Descriptor
448 );
449 
450 VOID
451 NTAPI
452 KiFlushNPXState(
453     IN FLOATING_SAVE_AREA *SaveArea
454 );
455 
456 VOID
457 NTAPI
458 Ki386AdjustEsp0(
459     IN PKTRAP_FRAME TrapFrame
460 );
461 
462 VOID
463 NTAPI
464 Ki386SetupAndExitToV86Mode(
465     OUT PTEB VdmTeb
466 );
467 
468 CODE_SEG("INIT")
469 VOID
470 NTAPI
471 KeI386VdmInitialize(
472     VOID
473 );
474 
475 CODE_SEG("INIT")
476 ULONG_PTR
477 NTAPI
478 Ki386EnableGlobalPage(
479     IN ULONG_PTR Context
480 );
481 
482 CODE_SEG("INIT")
483 ULONG_PTR
484 NTAPI
485 Ki386EnableTargetLargePage(
486     IN ULONG_PTR Context
487 );
488 
489 BOOLEAN
490 NTAPI
491 Ki386CreateIdentityMap(
492     IN PLARGE_IDENTITY_MAP IdentityMap,
493     IN PVOID StartPtr,
494     IN ULONG Length
495 );
496 
497 VOID
498 NTAPI
499 Ki386FreeIdentityMap(
500     IN PLARGE_IDENTITY_MAP IdentityMap
501 );
502 
503 VOID
504 NTAPI
505 Ki386EnableCurrentLargePage(
506     IN ULONG_PTR StartAddress,
507     IN ULONG Cr3
508 );
509 
510 CODE_SEG("INIT")
511 VOID
512 NTAPI
513 KiI386PentiumLockErrataFixup(
514     VOID
515 );
516 
517 CODE_SEG("INIT")
518 VOID
519 NTAPI
520 KiInitializePAT(
521     VOID
522 );
523 
524 CODE_SEG("INIT")
525 VOID
526 NTAPI
527 KiInitializeMTRR(
528     IN BOOLEAN FinalCpu
529 );
530 
531 CODE_SEG("INIT")
532 VOID
533 NTAPI
534 KiAmdK6InitializeMTRR(
535     VOID
536 );
537 
538 CODE_SEG("INIT")
539 VOID
540 NTAPI
541 KiRestoreFastSyscallReturnState(
542     VOID
543 );
544 
545 CODE_SEG("INIT")
546 ULONG_PTR
547 NTAPI
548 Ki386EnableDE(
549     IN ULONG_PTR Context
550 );
551 
552 CODE_SEG("INIT")
553 ULONG_PTR
554 NTAPI
555 Ki386EnableFxsr(
556     IN ULONG_PTR Context
557 );
558 
559 CODE_SEG("INIT")
560 ULONG_PTR
561 NTAPI
562 Ki386EnableXMMIExceptions(
563     IN ULONG_PTR Context
564 );
565 
566 BOOLEAN
567 NTAPI
568 VdmDispatchBop(
569     IN PKTRAP_FRAME TrapFrame
570 );
571 
572 BOOLEAN
573 NTAPI
574 VdmDispatchPageFault(
575     _In_ PKTRAP_FRAME TrapFrame
576 );
577 
578 BOOLEAN
579 FASTCALL
580 KiVdmOpcodePrefix(
581     IN PKTRAP_FRAME TrapFrame,
582     IN ULONG Flags
583 );
584 
585 BOOLEAN
586 FASTCALL
587 Ki386HandleOpcodeV86(
588     IN PKTRAP_FRAME TrapFrame
589 );
590 
591 DECLSPEC_NORETURN
592 VOID
593 FASTCALL
594 KiEoiHelper(
595     IN PKTRAP_FRAME TrapFrame
596 );
597 
598 VOID
599 FASTCALL
600 Ki386BiosCallReturnAddress(
601     IN PKTRAP_FRAME TrapFrame
602 );
603 
604 ULONG_PTR
605 FASTCALL
606 KiExitV86Mode(
607     IN PKTRAP_FRAME TrapFrame
608 );
609 
610 DECLSPEC_NORETURN
611 VOID
612 NTAPI
613 KiDispatchExceptionFromTrapFrame(
614     IN NTSTATUS Code,
615     IN ULONG Flags,
616     IN ULONG_PTR Address,
617     IN ULONG ParameterCount,
618     IN ULONG_PTR Parameter1,
619     IN ULONG_PTR Parameter2,
620     IN ULONG_PTR Parameter3,
621     IN PKTRAP_FRAME TrapFrame
622 );
623 
624 NTSTATUS
625 NTAPI
626 KiConvertToGuiThread(
627     VOID
628 );
629 
630 //
631 // Global x86 only Kernel data
632 //
633 extern PVOID Ki386IopmSaveArea;
634 extern ULONG KeI386EFlagsAndMaskV86;
635 extern ULONG KeI386EFlagsOrMaskV86;
636 extern BOOLEAN KeI386VirtualIntExtensions;
637 extern KIDTENTRY KiIdt[MAXIMUM_IDTVECTOR+1];
638 extern KDESCRIPTOR KiIdtDescriptor;
639 extern BOOLEAN KiI386PentiumLockErrataPresent;
640 extern ULONG KeI386NpxPresent;
641 extern ULONG KeI386XMMIPresent;
642 extern ULONG KeI386FxsrPresent;
643 extern ULONG KiMXCsrMask;
644 extern ULONG KeI386CpuType;
645 extern ULONG KeI386CpuStep;
646 extern ULONG KiFastSystemCallDisable;
647 extern UCHAR KiDebugRegisterTrapOffsets[9];
648 extern UCHAR KiDebugRegisterContextOffsets[9];
649 extern VOID __cdecl KiTrap02(VOID);
650 extern VOID __cdecl KiTrap08(VOID);
651 extern VOID __cdecl KiTrap13(VOID);
652 extern VOID __cdecl KiFastCallEntry(VOID);
653 extern VOID NTAPI ExpInterlockedPopEntrySListFault(VOID);
654 extern VOID NTAPI ExpInterlockedPopEntrySListResume(VOID);
655 extern VOID __cdecl CopyParams(VOID);
656 extern VOID __cdecl ReadBatch(VOID);
657 extern CHAR KiSystemCallExitBranch[];
658 extern CHAR KiSystemCallExit[];
659 extern CHAR KiSystemCallExit2[];
660 
661 //
662 // Trap Macros
663 //
664 #include "trap_x.h"
665 
666 //
667 // Returns a thread's FPU save area
668 //
669 FORCEINLINE
670 PFX_SAVE_AREA
671 KiGetThreadNpxArea(IN PKTHREAD Thread)
672 {
673     ASSERT((ULONG_PTR)Thread->InitialStack % 16 == 0);
674     return (PFX_SAVE_AREA)((ULONG_PTR)Thread->InitialStack - sizeof(FX_SAVE_AREA));
675 }
676 
677 //
678 // Sanitizes a selector
679 //
680 FORCEINLINE
681 ULONG
682 Ke386SanitizeSeg(IN ULONG Cs,
683                 IN KPROCESSOR_MODE Mode)
684 {
685     //
686     // Check if we're in kernel-mode, and force CPL 0 if so.
687     // Otherwise, force CPL 3.
688     //
689     return ((Mode == KernelMode) ?
690             (Cs & (0xFFFF & ~RPL_MASK)) :
691             (RPL_MASK | (Cs & 0xFFFF)));
692 }
693 
694 //
695 // Sanitizes EFLAGS
696 //
697 FORCEINLINE
698 ULONG
699 Ke386SanitizeFlags(IN ULONG Eflags,
700                    IN KPROCESSOR_MODE Mode)
701 {
702     //
703     // Check if we're in kernel-mode, and sanitize EFLAGS if so.
704     // Otherwise, also force interrupt mask on.
705     //
706     return ((Mode == KernelMode) ?
707             (Eflags & (EFLAGS_USER_SANITIZE | EFLAGS_INTERRUPT_MASK)) :
708             (EFLAGS_INTERRUPT_MASK | (Eflags & EFLAGS_USER_SANITIZE)));
709 }
710 
711 //
712 // Sanitizes a Debug Register
713 //
714 FORCEINLINE
715 PVOID
716 Ke386SanitizeDr(IN PVOID DrAddress,
717                 IN KPROCESSOR_MODE Mode)
718 {
719     //
720     // Check if we're in kernel-mode, and return the address directly if so.
721     // Otherwise, make sure it's not inside the kernel-mode address space.
722     // If it is, then clear the address.
723     //
724     return ((Mode == KernelMode) ? DrAddress :
725             (DrAddress <= MM_HIGHEST_USER_ADDRESS) ? DrAddress : 0);
726 }
727 
728 //
729 // Exception with no arguments
730 //
731 FORCEINLINE
732 DECLSPEC_NORETURN
733 VOID
734 KiDispatchException0Args(IN NTSTATUS Code,
735                          IN ULONG_PTR Address,
736                          IN PKTRAP_FRAME TrapFrame)
737 {
738     /* Helper for exceptions with no arguments */
739     KiDispatchExceptionFromTrapFrame(Code, 0, Address, 0, 0, 0, 0, TrapFrame);
740 }
741 
742 //
743 // Exception with one argument
744 //
745 FORCEINLINE
746 DECLSPEC_NORETURN
747 VOID
748 KiDispatchException1Args(IN NTSTATUS Code,
749                          IN ULONG_PTR Address,
750                          IN ULONG P1,
751                          IN PKTRAP_FRAME TrapFrame)
752 {
753     /* Helper for exceptions with no arguments */
754     KiDispatchExceptionFromTrapFrame(Code, 0, Address, 1, P1, 0, 0, TrapFrame);
755 }
756 
757 //
758 // Exception with two arguments
759 //
760 FORCEINLINE
761 DECLSPEC_NORETURN
762 VOID
763 KiDispatchException2Args(IN NTSTATUS Code,
764                          IN ULONG_PTR Address,
765                          IN ULONG P1,
766                          IN ULONG P2,
767                          IN PKTRAP_FRAME TrapFrame)
768 {
769     /* Helper for exceptions with no arguments */
770     KiDispatchExceptionFromTrapFrame(Code, 0, Address, 2, P1, P2, 0, TrapFrame);
771 }
772 
773 //
774 // Performs a system call
775 //
776 NTSTATUS
777 NTAPI
778 KiSystemCallTrampoline(_In_ PVOID Handler,
779                        _In_ PVOID Arguments,
780                        _In_ ULONG StackBytes);
781 
782 
783 //
784 // Checks for pending APCs
785 //
786 FORCEINLINE
787 VOID
788 KiCheckForApcDelivery(IN PKTRAP_FRAME TrapFrame)
789 {
790     PKTHREAD Thread;
791     KIRQL OldIrql;
792 
793     /* Check for V8086 or user-mode trap */
794     if ((TrapFrame->EFlags & EFLAGS_V86_MASK) || (KiUserTrap(TrapFrame)))
795     {
796         /* Get the thread */
797         Thread = KeGetCurrentThread();
798         while (TRUE)
799         {
800             /* Turn off the alerted state for kernel mode */
801             Thread->Alerted[KernelMode] = FALSE;
802 
803             /* Are there pending user APCs? */
804             if (!Thread->ApcState.UserApcPending) break;
805 
806             /* Raise to APC level and enable interrupts */
807             OldIrql = KfRaiseIrql(APC_LEVEL);
808             _enable();
809 
810             /* Deliver APCs */
811             KiDeliverApc(UserMode, NULL, TrapFrame);
812 
813             /* Restore IRQL and disable interrupts once again */
814             KfLowerIrql(OldIrql);
815             _disable();
816         }
817     }
818 }
819 
820 //
821 // Switches from boot loader to initial kernel stack
822 //
823 CODE_SEG("INIT")
824 FORCEINLINE
825 DECLSPEC_NORETURN
826 VOID
827 KiSwitchToBootStack(IN ULONG_PTR InitialStack)
828 {
829     CODE_SEG("INIT") DECLSPEC_NORETURN VOID NTAPI KiSystemStartupBootStack(VOID);
830 
831     /* We have to switch to a new stack before continuing kernel initialization */
832 #ifdef __GNUC__
833     __asm__
834     (
835         "movl %0, %%esp\n\t"
836         "subl %1, %%esp\n\t"
837         "pushl %2\n\t"
838         "jmp _KiSystemStartupBootStack@0"
839         :
840         : "c"(InitialStack),
841           "i"(NPX_FRAME_LENGTH + KTRAP_FRAME_ALIGN + KTRAP_FRAME_LENGTH),
842           "i"(CR0_EM | CR0_TS | CR0_MP),
843           "p"(KiSystemStartupBootStack)
844         : "%esp"
845     );
846 #elif defined(_MSC_VER)
847     __asm
848     {
849         mov esp, InitialStack
850         sub esp, (NPX_FRAME_LENGTH + KTRAP_FRAME_ALIGN + KTRAP_FRAME_LENGTH)
851         push (CR0_EM | CR0_TS | CR0_MP)
852         jmp KiSystemStartupBootStack
853     }
854 #else
855 #error Unknown Compiler
856 #endif
857 
858     UNREACHABLE;
859 }
860 
861 //
862 // Emits the iret instruction for C code
863 //
864 FORCEINLINE
865 DECLSPEC_NORETURN
866 VOID
867 KiIret(VOID)
868 {
869 #if defined(__GNUC__)
870     __asm__ __volatile__
871     (
872         "iret"
873     );
874 #elif defined(_MSC_VER)
875     __asm
876     {
877         iretd
878     }
879 #else
880 #error Unsupported compiler
881 #endif
882     UNREACHABLE;
883 }
884 
885 //
886 // Normally this is done by the HAL, but on x86 as an optimization, the kernel
887 // initiates the end by calling back into the HAL and exiting the trap here.
888 //
889 FORCEINLINE
890 VOID
891 KiEndInterrupt(IN KIRQL Irql,
892                IN PKTRAP_FRAME TrapFrame)
893 {
894     /* Disable interrupts and end the interrupt */
895     _disable();
896     HalEndSystemInterrupt(Irql, TrapFrame);
897 
898     /* Exit the interrupt */
899     KiEoiHelper(TrapFrame);
900 }
901 
902 //
903 // PERF Code
904 //
905 FORCEINLINE
906 VOID
907 Ki386PerfEnd(VOID)
908 {
909     extern ULONGLONG BootCyclesEnd, BootCycles;
910     BootCyclesEnd = __rdtsc();
911     DbgPrint("Boot took %I64u cycles!\n", BootCyclesEnd - BootCycles);
912     DbgPrint("Interrupts: %u System Calls: %u Context Switches: %u\n",
913              KeGetCurrentPrcb()->InterruptCount,
914              KeGetCurrentPrcb()->KeSystemCalls,
915              KeGetContextSwitches(KeGetCurrentPrcb()));
916 }
917 
918 FORCEINLINE
919 PULONG
920 KiGetUserModeStackAddress(void)
921 {
922     return &(KeGetCurrentThread()->TrapFrame->HardwareEsp);
923 }
924 
925 #ifdef __cplusplus
926 } // extern "C"
927 #endif
928 
929 #endif
930