xref: /reactos/ntoskrnl/include/internal/i386/ke.h (revision 7353af1e) 
1 #pragma once
2 
3 #ifndef __ASM__
4 
5 #include "intrin_i.h"
6 
7 #ifdef __cplusplus
8 extern "C"
9 {
10 #endif
11 
12 //
13 // Thread Dispatcher Header DebugActive Mask
14 //
15 #define DR_MASK(x)                              (1 << (x))
16 #define DR_REG_MASK                             0x4F
17 
18 //
19 // INT3 is 1 byte long
20 //
21 #define KD_BREAKPOINT_TYPE        UCHAR
22 #define KD_BREAKPOINT_SIZE        sizeof(UCHAR)
23 #define KD_BREAKPOINT_VALUE       0xCC
24 
25 //
26 // One-liners for getting and setting special purpose registers in portable code
27 //
28 FORCEINLINE
29 ULONG_PTR
30 KeGetContextPc(PCONTEXT Context)
31 {
32     return Context->Eip;
33 }
34 
35 FORCEINLINE
36 VOID
37 KeSetContextPc(PCONTEXT Context, ULONG_PTR ProgramCounter)
38 {
39     Context->Eip = ProgramCounter;
40 }
41 
42 FORCEINLINE
43 ULONG_PTR
44 KeGetContextReturnRegister(PCONTEXT Context)
45 {
46     return Context->Eax;
47 }
48 
49 FORCEINLINE
50 VOID
51 KeSetContextReturnRegister(PCONTEXT Context, ULONG_PTR ReturnValue)
52 {
53     Context->Eax = ReturnValue;
54 }
55 
56 FORCEINLINE
57 ULONG_PTR
58 KeGetContextFrameRegister(PCONTEXT Context)
59 {
60     return Context->Ebp;
61 }
62 
63 FORCEINLINE
64 VOID
65 KeSetContextFrameRegister(PCONTEXT Context, ULONG_PTR Frame)
66 {
67     Context->Ebp = Frame;
68 }
69 
70 FORCEINLINE
71 ULONG_PTR
72 KeGetTrapFramePc(PKTRAP_FRAME TrapFrame)
73 {
74     return TrapFrame->Eip;
75 }
76 
77 FORCEINLINE
78 PKTRAP_FRAME
79 KiGetLinkedTrapFrame(PKTRAP_FRAME TrapFrame)
80 {
81     return (PKTRAP_FRAME)TrapFrame->Edx;
82 }
83 
84 
85 FORCEINLINE
86 ULONG_PTR
87 KeGetTrapFrameStackRegister(PKTRAP_FRAME TrapFrame)
88 {
89     if (TrapFrame->PreviousPreviousMode == KernelMode)
90         return TrapFrame->TempEsp;
91     return TrapFrame->HardwareEsp;
92 }
93 
94 FORCEINLINE
95 ULONG_PTR
96 KeGetTrapFrameFrameRegister(PKTRAP_FRAME TrapFrame)
97 {
98     return TrapFrame->Ebp;
99 }
100 
101 //
102 // Macro to get trap and exception frame from a thread stack
103 //
104 #define KeGetTrapFrame(Thread) \
105     (PKTRAP_FRAME)((ULONG_PTR)((Thread)->InitialStack) - \
106                    sizeof(KTRAP_FRAME) - \
107                    sizeof(FX_SAVE_AREA))
108 
109 #define KeGetExceptionFrame(Thread) \
110     NULL
111 
112 //
113 // Macro to get context switches from the PRCB
114 // All architectures but x86 have it in the PRCB's KeContextSwitches
115 //
116 #define KeGetContextSwitches(Prcb)  \
117     CONTAINING_RECORD(Prcb, KIPCR, PrcbData)->ContextSwitches
118 
119 //
120 // Macro to get the second level cache size field name which differs between
121 // CISC and RISC architectures, as the former has unified I/D cache
122 //
123 #define KiGetSecondLevelDCacheSize() ((PKIPCR)KeGetPcr())->SecondLevelCacheSize
124 
125 //
126 // Returns the Interrupt State from a Trap Frame.
127 // ON = TRUE, OFF = FALSE
128 //
129 #define KeGetTrapFrameInterruptState(TrapFrame) \
130         BooleanFlagOn((TrapFrame)->EFlags, EFLAGS_INTERRUPT_MASK)
131 
132 //
133 // Flags for exiting a trap
134 //
135 #define KTE_SKIP_PM_BIT  (((KTRAP_EXIT_SKIP_BITS) { { .SkipPreviousMode = TRUE } }).Bits)
136 #define KTE_SKIP_SEG_BIT (((KTRAP_EXIT_SKIP_BITS) { { .SkipSegments  = TRUE } }).Bits)
137 #define KTE_SKIP_VOL_BIT (((KTRAP_EXIT_SKIP_BITS) { { .SkipVolatiles = TRUE } }).Bits)
138 
139 typedef union _KTRAP_EXIT_SKIP_BITS
140 {
141     struct
142     {
143         UCHAR SkipPreviousMode:1;
144         UCHAR SkipSegments:1;
145         UCHAR SkipVolatiles:1;
146         UCHAR Reserved:5;
147     };
148     UCHAR Bits;
149 } KTRAP_EXIT_SKIP_BITS, *PKTRAP_EXIT_SKIP_BITS;
150 
151 
152 //
153 // Flags used by the VDM/V8086 emulation engine for determining instruction prefixes
154 //
155 #define PFX_FLAG_ES                0x00000100
156 #define PFX_FLAG_CS                0x00000200
157 #define PFX_FLAG_SS                0x00000400
158 #define PFX_FLAG_DS                0x00000800
159 #define PFX_FLAG_FS                0x00001000
160 #define PFX_FLAG_GS                0x00002000
161 #define PFX_FLAG_OPER32            0x00004000
162 #define PFX_FLAG_ADDR32            0x00008000
163 #define PFX_FLAG_LOCK              0x00010000
164 #define PFX_FLAG_REPNE             0x00020000
165 #define PFX_FLAG_REP               0x00040000
166 
167 //
168 // VDM Helper Macros
169 //
170 // All VDM/V8086 opcode emulators have the same FASTCALL function definition.
171 // We need to keep 2 parameters while the original ASM implementation uses 4:
172 // TrapFrame, PrefixFlags, Eip, InstructionSize;
173 //
174 // We pass the trap frame, and prefix flags, in our two parameters.
175 //
176 // We then realize that since the smallest prefix flag is 0x100, this gives us
177 // a count of up to 0xFF. So we OR in the instruction size with the prefix flags
178 //
179 // We further realize that we always have access to EIP from the trap frame, and
180 // that if we want the *current instruction* EIP, we simply have to add the
181 // instruction size *MINUS ONE*, and that gives us the EIP we should be looking
182 // at now, so we don't need to use the stack to push this parameter.
183 //
184 // We actually only care about the *current instruction* EIP in one location,
185 // so although it may be slightly more expensive to re-calculate the EIP one
186 // more time, this way we don't redefine ALL opcode handlers to have 3 parameters,
187 // which would be forcing stack usage in all other scenarios.
188 //
189 #define KiVdmSetVdmEFlags(x)        InterlockedOr((PLONG)KiNtVdmState, (x));
190 #define KiVdmClearVdmEFlags(x)      InterlockedAnd((PLONG)KiNtVdmState, ~(x))
191 #define KiCallVdmHandler(x)         KiVdmOpcode##x(TrapFrame, Flags)
192 #define KiCallVdmPrefixHandler(x)   KiVdmOpcodePrefix(TrapFrame, Flags | x)
193 #define KiVdmUnhandledOpcode(x)                     \
194     BOOLEAN                                         \
195     FASTCALL                                        \
196     KiVdmOpcode##x(IN PKTRAP_FRAME TrapFrame,       \
197                    IN ULONG Flags)                  \
198     {                                               \
199         /* Not yet handled */                       \
200         UNIMPLEMENTED_DBGBREAK();                   \
201         return FALSE;                               \
202     }
203 
204 C_ASSERT(NPX_FRAME_LENGTH == sizeof(FX_SAVE_AREA));
205 
206 //
207 // Local parameters
208 //
209 typedef struct _KV86_FRAME
210 {
211     PVOID ThreadStack;
212     PVOID ThreadTeb;
213     PVOID PcrTeb;
214 } KV86_FRAME, *PKV86_FRAME;
215 
216 //
217 // Virtual Stack Frame
218 //
219 typedef struct _KV8086_STACK_FRAME
220 {
221     KTRAP_FRAME TrapFrame;
222     FX_SAVE_AREA NpxArea;
223     KV86_FRAME V86Frame;
224 } KV8086_STACK_FRAME, *PKV8086_STACK_FRAME;
225 
226 //
227 // Large Pages Support
228 //
229 typedef struct _LARGE_IDENTITY_MAP
230 {
231     PHARDWARE_PTE TopLevelDirectory;
232     ULONG Cr3;
233     ULONG_PTR StartAddress;
234     ULONG PagesCount;
235     PVOID PagesList[30];
236 } LARGE_IDENTITY_MAP, *PLARGE_IDENTITY_MAP;
237 
238 /* Diable interrupts and return whether they were enabled before */
239 FORCEINLINE
240 BOOLEAN
241 KeDisableInterrupts(VOID)
242 {
243     ULONG Flags;
244     BOOLEAN Return;
245 
246     /* Get EFLAGS and check if the interrupt bit is set */
247     Flags = __readeflags();
248     Return = (Flags & EFLAGS_INTERRUPT_MASK) ? TRUE: FALSE;
249 
250     /* Disable interrupts */
251     _disable();
252     return Return;
253 }
254 
255 /* Restore previous interrupt state */
256 FORCEINLINE
257 VOID
258 KeRestoreInterrupts(BOOLEAN WereEnabled)
259 {
260     if (WereEnabled) _enable();
261 }
262 
263 //
264 // Registers an interrupt handler with an IDT vector
265 //
266 FORCEINLINE
267 VOID
268 KeRegisterInterruptHandler(IN ULONG Vector,
269                            IN PVOID Handler)
270 {
271     UCHAR Entry;
272     ULONG_PTR Address;
273     PKIPCR Pcr = (PKIPCR)KeGetPcr();
274 
275     //
276     // Get the entry from the HAL
277     //
278     Entry = HalVectorToIDTEntry(Vector);
279     Address = PtrToUlong(Handler);
280 
281     //
282     // Now set the data
283     //
284     Pcr->IDT[Entry].ExtendedOffset = (USHORT)(Address >> 16);
285     Pcr->IDT[Entry].Offset = (USHORT)Address;
286 }
287 
288 //
289 // Returns the registered interrupt handler for a given IDT vector
290 //
291 FORCEINLINE
292 PVOID
293 KeQueryInterruptHandler(IN ULONG Vector)
294 {
295     PKIPCR Pcr = (PKIPCR)KeGetPcr();
296     UCHAR Entry;
297 
298     //
299     // Get the entry from the HAL
300     //
301     Entry = HalVectorToIDTEntry(Vector);
302 
303     //
304     // Read the entry from the IDT
305     //
306     return (PVOID)(((Pcr->IDT[Entry].ExtendedOffset << 16) & 0xFFFF0000) |
307                     (Pcr->IDT[Entry].Offset & 0xFFFF));
308 }
309 
310 //
311 // Invalidates the TLB entry for a specified address
312 //
313 FORCEINLINE
314 VOID
315 KeInvalidateTlbEntry(IN PVOID Address)
316 {
317     /* Invalidate the TLB entry for this address */
318     __invlpg(Address);
319 }
320 
321 FORCEINLINE
322 VOID
323 KeFlushProcessTb(VOID)
324 {
325     /* Flush the TLB by resetting CR3 */
326     __writecr3(__readcr3());
327 }
328 
329 FORCEINLINE
330 VOID
331 KeSweepICache(IN PVOID BaseAddress,
332               IN SIZE_T FlushSize)
333 {
334     //
335     // Always sweep the whole cache
336     //
337     UNREFERENCED_PARAMETER(BaseAddress);
338     UNREFERENCED_PARAMETER(FlushSize);
339     __wbinvd();
340 }
341 
342 FORCEINLINE
343 PRKTHREAD
344 KeGetCurrentThread(VOID)
345 {
346     /* Return the current thread */
347     return ((PKIPCR)KeGetPcr())->PrcbData.CurrentThread;
348 }
349 
350 FORCEINLINE
351 VOID
352 KiRundownThread(IN PKTHREAD Thread)
353 {
354 #ifndef CONFIG_SMP
355     /* Check if this is the NPX Thread */
356     if (KeGetCurrentPrcb()->NpxThread == Thread)
357     {
358         /* Clear it */
359         KeGetCurrentPrcb()->NpxThread = NULL;
360         Ke386FnInit();
361     }
362 #else
363     /* Nothing to do */
364 #endif
365 }
366 
367 FORCEINLINE
368 VOID
369 Ke386SetGdtEntryBase(PKGDTENTRY GdtEntry, PVOID BaseAddress)
370 {
371     GdtEntry->BaseLow = (USHORT)((ULONG_PTR)BaseAddress & 0xFFFF);
372     GdtEntry->HighWord.Bytes.BaseMid = (UCHAR)((ULONG_PTR)BaseAddress >> 16);
373     GdtEntry->HighWord.Bytes.BaseHi = (UCHAR)((ULONG_PTR)BaseAddress >> 24);
374 }
375 
376 FORCEINLINE
377 VOID
378 KiSetTebBase(PKPCR Pcr, PNT_TIB TebAddress)
379 {
380     Pcr->NtTib.Self = TebAddress;
381     Ke386SetGdtEntryBase(&Pcr->GDT[KGDT_R3_TEB / sizeof(KGDTENTRY)], TebAddress);
382 }
383 
384 CODE_SEG("INIT")
385 VOID
386 FASTCALL
387 Ki386InitializeTss(
388     IN PKTSS Tss,
389     IN PKIDTENTRY Idt,
390     IN PKGDTENTRY Gdt
391 );
392 
393 CODE_SEG("INIT")
394 VOID
395 NTAPI
396 KiSetCR0Bits(VOID);
397 
398 CODE_SEG("INIT")
399 VOID
400 NTAPI
401 KiGetCacheInformation(VOID);
402 
403 CODE_SEG("INIT")
404 BOOLEAN
405 NTAPI
406 KiIsNpxPresent(
407     VOID
408 );
409 
410 CODE_SEG("INIT")
411 BOOLEAN
412 NTAPI
413 KiIsNpxErrataPresent(
414     VOID
415 );
416 
417 CODE_SEG("INIT")
418 VOID
419 NTAPI
420 KiSetProcessorType(VOID);
421 
422 CODE_SEG("INIT")
423 ULONG
424 NTAPI
425 KiGetFeatureBits(VOID);
426 
427 VOID
428 NTAPI
429 KiThreadStartup(VOID);
430 
431 NTSTATUS
432 NTAPI
433 Ke386GetGdtEntryThread(
434     IN PKTHREAD Thread,
435     IN ULONG Offset,
436     IN PKGDTENTRY Descriptor
437 );
438 
439 VOID
440 NTAPI
441 KiFlushNPXState(
442     IN FLOATING_SAVE_AREA *SaveArea
443 );
444 
445 VOID
446 NTAPI
447 Ki386AdjustEsp0(
448     IN PKTRAP_FRAME TrapFrame
449 );
450 
451 VOID
452 NTAPI
453 Ki386SetupAndExitToV86Mode(
454     OUT PTEB VdmTeb
455 );
456 
457 CODE_SEG("INIT")
458 VOID
459 NTAPI
460 KeI386VdmInitialize(
461     VOID
462 );
463 
464 CODE_SEG("INIT")
465 ULONG_PTR
466 NTAPI
467 Ki386EnableGlobalPage(
468     IN ULONG_PTR Context
469 );
470 
471 CODE_SEG("INIT")
472 ULONG_PTR
473 NTAPI
474 Ki386EnableTargetLargePage(
475     IN ULONG_PTR Context
476 );
477 
478 BOOLEAN
479 NTAPI
480 Ki386CreateIdentityMap(
481     IN PLARGE_IDENTITY_MAP IdentityMap,
482     IN PVOID StartPtr,
483     IN ULONG Length
484 );
485 
486 VOID
487 NTAPI
488 Ki386FreeIdentityMap(
489     IN PLARGE_IDENTITY_MAP IdentityMap
490 );
491 
492 VOID
493 NTAPI
494 Ki386EnableCurrentLargePage(
495     IN ULONG_PTR StartAddress,
496     IN ULONG Cr3
497 );
498 
499 CODE_SEG("INIT")
500 VOID
501 NTAPI
502 KiI386PentiumLockErrataFixup(
503     VOID
504 );
505 
506 CODE_SEG("INIT")
507 VOID
508 NTAPI
509 KiInitializePAT(
510     VOID
511 );
512 
513 CODE_SEG("INIT")
514 VOID
515 NTAPI
516 KiInitializeMTRR(
517     IN BOOLEAN FinalCpu
518 );
519 
520 CODE_SEG("INIT")
521 VOID
522 NTAPI
523 KiAmdK6InitializeMTRR(
524     VOID
525 );
526 
527 CODE_SEG("INIT")
528 VOID
529 NTAPI
530 KiRestoreFastSyscallReturnState(
531     VOID
532 );
533 
534 CODE_SEG("INIT")
535 ULONG_PTR
536 NTAPI
537 Ki386EnableDE(
538     IN ULONG_PTR Context
539 );
540 
541 CODE_SEG("INIT")
542 ULONG_PTR
543 NTAPI
544 Ki386EnableFxsr(
545     IN ULONG_PTR Context
546 );
547 
548 CODE_SEG("INIT")
549 ULONG_PTR
550 NTAPI
551 Ki386EnableXMMIExceptions(
552     IN ULONG_PTR Context
553 );
554 
555 BOOLEAN
556 NTAPI
557 VdmDispatchBop(
558     IN PKTRAP_FRAME TrapFrame
559 );
560 
561 BOOLEAN
562 NTAPI
563 VdmDispatchPageFault(
564     _In_ PKTRAP_FRAME TrapFrame
565 );
566 
567 BOOLEAN
568 FASTCALL
569 KiVdmOpcodePrefix(
570     IN PKTRAP_FRAME TrapFrame,
571     IN ULONG Flags
572 );
573 
574 BOOLEAN
575 FASTCALL
576 Ki386HandleOpcodeV86(
577     IN PKTRAP_FRAME TrapFrame
578 );
579 
580 DECLSPEC_NORETURN
581 VOID
582 FASTCALL
583 KiEoiHelper(
584     IN PKTRAP_FRAME TrapFrame
585 );
586 
587 VOID
588 FASTCALL
589 Ki386BiosCallReturnAddress(
590     IN PKTRAP_FRAME TrapFrame
591 );
592 
593 ULONG_PTR
594 FASTCALL
595 KiExitV86Mode(
596     IN PKTRAP_FRAME TrapFrame
597 );
598 
599 DECLSPEC_NORETURN
600 VOID
601 NTAPI
602 KiDispatchExceptionFromTrapFrame(
603     IN NTSTATUS Code,
604     IN ULONG Flags,
605     IN ULONG_PTR Address,
606     IN ULONG ParameterCount,
607     IN ULONG_PTR Parameter1,
608     IN ULONG_PTR Parameter2,
609     IN ULONG_PTR Parameter3,
610     IN PKTRAP_FRAME TrapFrame
611 );
612 
613 NTSTATUS
614 NTAPI
615 KiConvertToGuiThread(
616     VOID
617 );
618 
619 //
620 // Global x86 only Kernel data
621 //
622 extern PVOID Ki386IopmSaveArea;
623 extern ULONG KeI386EFlagsAndMaskV86;
624 extern ULONG KeI386EFlagsOrMaskV86;
625 extern BOOLEAN KeI386VirtualIntExtensions;
626 extern KIDTENTRY KiIdt[MAXIMUM_IDTVECTOR+1];
627 extern KDESCRIPTOR KiIdtDescriptor;
628 extern BOOLEAN KiI386PentiumLockErrataPresent;
629 extern ULONG KeI386NpxPresent;
630 extern ULONG KeI386XMMIPresent;
631 extern ULONG KeI386FxsrPresent;
632 extern ULONG KiMXCsrMask;
633 extern ULONG KeI386CpuType;
634 extern ULONG KeI386CpuStep;
635 extern ULONG KiFastSystemCallDisable;
636 extern UCHAR KiDebugRegisterTrapOffsets[9];
637 extern UCHAR KiDebugRegisterContextOffsets[9];
638 extern VOID __cdecl KiTrap02(VOID);
639 extern VOID __cdecl KiTrap08(VOID);
640 extern VOID __cdecl KiTrap13(VOID);
641 extern VOID __cdecl KiFastCallEntry(VOID);
642 extern VOID NTAPI ExpInterlockedPopEntrySListFault(VOID);
643 extern VOID NTAPI ExpInterlockedPopEntrySListResume(VOID);
644 extern VOID __cdecl CopyParams(VOID);
645 extern VOID __cdecl ReadBatch(VOID);
646 extern CHAR KiSystemCallExitBranch[];
647 extern CHAR KiSystemCallExit[];
648 extern CHAR KiSystemCallExit2[];
649 
650 //
651 // Trap Macros
652 //
653 #include "trap_x.h"
654 
655 //
656 // Returns a thread's FPU save area
657 //
658 FORCEINLINE
659 PFX_SAVE_AREA
660 KiGetThreadNpxArea(IN PKTHREAD Thread)
661 {
662     ASSERT((ULONG_PTR)Thread->InitialStack % 16 == 0);
663     return (PFX_SAVE_AREA)((ULONG_PTR)Thread->InitialStack - sizeof(FX_SAVE_AREA));
664 }
665 
666 //
667 // Sanitizes a selector
668 //
669 FORCEINLINE
670 ULONG
671 Ke386SanitizeSeg(IN ULONG Cs,
672                 IN KPROCESSOR_MODE Mode)
673 {
674     //
675     // Check if we're in kernel-mode, and force CPL 0 if so.
676     // Otherwise, force CPL 3.
677     //
678     return ((Mode == KernelMode) ?
679             (Cs & (0xFFFF & ~RPL_MASK)) :
680             (RPL_MASK | (Cs & 0xFFFF)));
681 }
682 
683 //
684 // Sanitizes EFLAGS
685 //
686 FORCEINLINE
687 ULONG
688 Ke386SanitizeFlags(IN ULONG Eflags,
689                    IN KPROCESSOR_MODE Mode)
690 {
691     //
692     // Check if we're in kernel-mode, and sanitize EFLAGS if so.
693     // Otherwise, also force interrupt mask on.
694     //
695     return ((Mode == KernelMode) ?
696             (Eflags & (EFLAGS_USER_SANITIZE | EFLAGS_INTERRUPT_MASK)) :
697             (EFLAGS_INTERRUPT_MASK | (Eflags & EFLAGS_USER_SANITIZE)));
698 }
699 
700 //
701 // Sanitizes a Debug Register
702 //
703 FORCEINLINE
704 PVOID
705 Ke386SanitizeDr(IN PVOID DrAddress,
706                 IN KPROCESSOR_MODE Mode)
707 {
708     //
709     // Check if we're in kernel-mode, and return the address directly if so.
710     // Otherwise, make sure it's not inside the kernel-mode address space.
711     // If it is, then clear the address.
712     //
713     return ((Mode == KernelMode) ? DrAddress :
714             (DrAddress <= MM_HIGHEST_USER_ADDRESS) ? DrAddress : 0);
715 }
716 
717 //
718 // Exception with no arguments
719 //
720 FORCEINLINE
721 DECLSPEC_NORETURN
722 VOID
723 KiDispatchException0Args(IN NTSTATUS Code,
724                          IN ULONG_PTR Address,
725                          IN PKTRAP_FRAME TrapFrame)
726 {
727     /* Helper for exceptions with no arguments */
728     KiDispatchExceptionFromTrapFrame(Code, 0, Address, 0, 0, 0, 0, TrapFrame);
729 }
730 
731 //
732 // Exception with one argument
733 //
734 FORCEINLINE
735 DECLSPEC_NORETURN
736 VOID
737 KiDispatchException1Args(IN NTSTATUS Code,
738                          IN ULONG_PTR Address,
739                          IN ULONG P1,
740                          IN PKTRAP_FRAME TrapFrame)
741 {
742     /* Helper for exceptions with no arguments */
743     KiDispatchExceptionFromTrapFrame(Code, 0, Address, 1, P1, 0, 0, TrapFrame);
744 }
745 
746 //
747 // Exception with two arguments
748 //
749 FORCEINLINE
750 DECLSPEC_NORETURN
751 VOID
752 KiDispatchException2Args(IN NTSTATUS Code,
753                          IN ULONG_PTR Address,
754                          IN ULONG P1,
755                          IN ULONG P2,
756                          IN PKTRAP_FRAME TrapFrame)
757 {
758     /* Helper for exceptions with no arguments */
759     KiDispatchExceptionFromTrapFrame(Code, 0, Address, 2, P1, P2, 0, TrapFrame);
760 }
761 
762 //
763 // Performs a system call
764 //
765 NTSTATUS
766 NTAPI
767 KiSystemCallTrampoline(_In_ PVOID Handler,
768                        _In_ PVOID Arguments,
769                        _In_ ULONG StackBytes);
770 
771 
772 //
773 // Checks for pending APCs
774 //
775 FORCEINLINE
776 VOID
777 KiCheckForApcDelivery(IN PKTRAP_FRAME TrapFrame)
778 {
779     PKTHREAD Thread;
780     KIRQL OldIrql;
781 
782     /* Check for V8086 or user-mode trap */
783     if ((TrapFrame->EFlags & EFLAGS_V86_MASK) || (KiUserTrap(TrapFrame)))
784     {
785         /* Get the thread */
786         Thread = KeGetCurrentThread();
787         while (TRUE)
788         {
789             /* Turn off the alerted state for kernel mode */
790             Thread->Alerted[KernelMode] = FALSE;
791 
792             /* Are there pending user APCs? */
793             if (!Thread->ApcState.UserApcPending) break;
794 
795             /* Raise to APC level and enable interrupts */
796             OldIrql = KfRaiseIrql(APC_LEVEL);
797             _enable();
798 
799             /* Deliver APCs */
800             KiDeliverApc(UserMode, NULL, TrapFrame);
801 
802             /* Restore IRQL and disable interrupts once again */
803             KfLowerIrql(OldIrql);
804             _disable();
805         }
806     }
807 }
808 
809 //
810 // Switches from boot loader to initial kernel stack
811 //
812 CODE_SEG("INIT")
813 FORCEINLINE
814 DECLSPEC_NORETURN
815 VOID
816 KiSwitchToBootStack(IN ULONG_PTR InitialStack)
817 {
818     CODE_SEG("INIT") DECLSPEC_NORETURN VOID NTAPI KiSystemStartupBootStack(VOID);
819 
820     /* We have to switch to a new stack before continuing kernel initialization */
821 #ifdef __GNUC__
822     __asm__
823     (
824         "movl %0, %%esp\n\t"
825         "subl %1, %%esp\n\t"
826         "pushl %2\n\t"
827         "jmp _KiSystemStartupBootStack@0"
828         :
829         : "c"(InitialStack),
830           "i"(NPX_FRAME_LENGTH + KTRAP_FRAME_ALIGN + KTRAP_FRAME_LENGTH),
831           "i"(CR0_EM | CR0_TS | CR0_MP),
832           "p"(KiSystemStartupBootStack)
833         : "%esp"
834     );
835 #elif defined(_MSC_VER)
836     __asm
837     {
838         mov esp, InitialStack
839         sub esp, (NPX_FRAME_LENGTH + KTRAP_FRAME_ALIGN + KTRAP_FRAME_LENGTH)
840         push (CR0_EM | CR0_TS | CR0_MP)
841         jmp KiSystemStartupBootStack
842     }
843 #else
844 #error Unknown Compiler
845 #endif
846 
847     UNREACHABLE;
848 }
849 
850 //
851 // Emits the iret instruction for C code
852 //
853 FORCEINLINE
854 DECLSPEC_NORETURN
855 VOID
856 KiIret(VOID)
857 {
858 #if defined(__GNUC__)
859     __asm__ __volatile__
860     (
861         "iret"
862     );
863 #elif defined(_MSC_VER)
864     __asm
865     {
866         iretd
867     }
868 #else
869 #error Unsupported compiler
870 #endif
871     UNREACHABLE;
872 }
873 
874 //
875 // Normally this is done by the HAL, but on x86 as an optimization, the kernel
876 // initiates the end by calling back into the HAL and exiting the trap here.
877 //
878 FORCEINLINE
879 VOID
880 KiEndInterrupt(IN KIRQL Irql,
881                IN PKTRAP_FRAME TrapFrame)
882 {
883     /* Disable interrupts and end the interrupt */
884     _disable();
885     HalEndSystemInterrupt(Irql, TrapFrame);
886 
887     /* Exit the interrupt */
888     KiEoiHelper(TrapFrame);
889 }
890 
891 //
892 // PERF Code
893 //
894 FORCEINLINE
895 VOID
896 Ki386PerfEnd(VOID)
897 {
898     extern ULONGLONG BootCyclesEnd, BootCycles;
899     BootCyclesEnd = __rdtsc();
900     DbgPrint("Boot took %I64u cycles!\n", BootCyclesEnd - BootCycles);
901     DbgPrint("Interrupts: %u System Calls: %u Context Switches: %u\n",
902              KeGetCurrentPrcb()->InterruptCount,
903              KeGetCurrentPrcb()->KeSystemCalls,
904              KeGetContextSwitches(KeGetCurrentPrcb()));
905 }
906 
907 FORCEINLINE
908 PULONG
909 KiGetUserModeStackAddress(void)
910 {
911     return &(KeGetCurrentThread()->TrapFrame->HardwareEsp);
912 }
913 
914 #ifdef __cplusplus
915 } // extern "C"
916 #endif
917 
918 #endif
919