1 #pragma once
2
3 #ifndef __ASM__
4
5 #include "intrin_i.h"
6
7 #ifdef __cplusplus
8 extern "C"
9 {
10 #endif
11
12 //
13 // Thread Dispatcher Header DebugActive Mask
14 //
15 #define DR_MASK(x) (1 << (x))
16 #define DR_REG_MASK 0x4F
17
18 //
19 // INT3 is 1 byte long
20 //
21 #define KD_BREAKPOINT_TYPE UCHAR
22 #define KD_BREAKPOINT_SIZE sizeof(UCHAR)
23 #define KD_BREAKPOINT_VALUE 0xCC
24
25 /* CPUID 1 - EDX flags */
26 #define X86_FEATURE_FPU 0x00000001 /* x87 FPU is present */
27 #define X86_FEATURE_VME 0x00000002 /* Virtual 8086 Extensions are present */
28 #define X86_FEATURE_DBG 0x00000004 /* Debugging extensions are present */
29 #define X86_FEATURE_PSE 0x00000008 /* Page Size Extension is present */
30 #define X86_FEATURE_TSC 0x00000010 /* Time Stamp Counters are present */
31 #define X86_FEATURE_PAE 0x00000040 /* Physical Address Extension is present */
32 #define X86_FEATURE_CX8 0x00000100 /* CMPXCHG8B instruction present */
33 #define X86_FEATURE_APIC 0x00000200 /* APIC is present */
34 #define X86_FEATURE_SYSCALL 0x00000800 /* SYSCALL/SYSRET support present */
35 #define X86_FEATURE_MTTR 0x00001000 /* Memory type range registers are present */
36 #define X86_FEATURE_PGE 0x00002000 /* Page Global Enable */
37 #define X86_FEATURE_CMOV 0x00008000 /* "Conditional move" instruction supported */
38 #define X86_FEATURE_PAT 0x00010000 /* Page Attribute Table is supported */
39 #define X86_FEATURE_DS 0x00200000 /* Debug Store is present */
40 #define X86_FEATURE_MMX 0x00800000 /* MMX extension present */
41 #define X86_FEATURE_FXSR 0x01000000 /* FXSAVE/FXRSTOR instructions present */
42 #define X86_FEATURE_SSE 0x02000000 /* SSE extension present */
43 #define X86_FEATURE_SSE2 0x04000000 /* SSE2 extension present */
44 #define X86_FEATURE_HT 0x10000000 /* Hyper-Threading present */
45
46 /* CPUID 0x80000001 - EDX extended flags */
47 #define X86_FEATURE_NX 0x00100000 /* NX support present */
48
49 //
50 // One-liners for getting and setting special purpose registers in portable code
51 //
52 FORCEINLINE
53 ULONG_PTR
KeGetContextPc(PCONTEXT Context)54 KeGetContextPc(PCONTEXT Context)
55 {
56 return Context->Eip;
57 }
58
59 FORCEINLINE
60 VOID
KeSetContextPc(PCONTEXT Context,ULONG_PTR ProgramCounter)61 KeSetContextPc(PCONTEXT Context, ULONG_PTR ProgramCounter)
62 {
63 Context->Eip = ProgramCounter;
64 }
65
66 FORCEINLINE
67 ULONG_PTR
KeGetContextReturnRegister(PCONTEXT Context)68 KeGetContextReturnRegister(PCONTEXT Context)
69 {
70 return Context->Eax;
71 }
72
73 FORCEINLINE
74 VOID
KeSetContextReturnRegister(PCONTEXT Context,ULONG_PTR ReturnValue)75 KeSetContextReturnRegister(PCONTEXT Context, ULONG_PTR ReturnValue)
76 {
77 Context->Eax = ReturnValue;
78 }
79
80 FORCEINLINE
81 ULONG_PTR
KeGetContextFrameRegister(PCONTEXT Context)82 KeGetContextFrameRegister(PCONTEXT Context)
83 {
84 return Context->Ebp;
85 }
86
87 FORCEINLINE
88 VOID
KeSetContextFrameRegister(PCONTEXT Context,ULONG_PTR Frame)89 KeSetContextFrameRegister(PCONTEXT Context, ULONG_PTR Frame)
90 {
91 Context->Ebp = Frame;
92 }
93
94 FORCEINLINE
95 ULONG_PTR
KeGetTrapFramePc(PKTRAP_FRAME TrapFrame)96 KeGetTrapFramePc(PKTRAP_FRAME TrapFrame)
97 {
98 return TrapFrame->Eip;
99 }
100
101 FORCEINLINE
102 PKTRAP_FRAME
KiGetLinkedTrapFrame(PKTRAP_FRAME TrapFrame)103 KiGetLinkedTrapFrame(PKTRAP_FRAME TrapFrame)
104 {
105 return (PKTRAP_FRAME)TrapFrame->Edx;
106 }
107
108
109 FORCEINLINE
110 ULONG_PTR
KeGetTrapFrameStackRegister(PKTRAP_FRAME TrapFrame)111 KeGetTrapFrameStackRegister(PKTRAP_FRAME TrapFrame)
112 {
113 if (TrapFrame->PreviousPreviousMode == KernelMode)
114 return TrapFrame->TempEsp;
115 return TrapFrame->HardwareEsp;
116 }
117
118 FORCEINLINE
119 ULONG_PTR
KeGetTrapFrameFrameRegister(PKTRAP_FRAME TrapFrame)120 KeGetTrapFrameFrameRegister(PKTRAP_FRAME TrapFrame)
121 {
122 return TrapFrame->Ebp;
123 }
124
125 //
126 // Macro to get trap and exception frame from a thread stack
127 //
128 #define KeGetTrapFrame(Thread) \
129 (PKTRAP_FRAME)((ULONG_PTR)((Thread)->InitialStack) - \
130 sizeof(KTRAP_FRAME) - \
131 sizeof(FX_SAVE_AREA))
132
133 #define KeGetExceptionFrame(Thread) \
134 NULL
135
136 //
137 // Macro to get context switches from the PRCB
138 // All architectures but x86 have it in the PRCB's KeContextSwitches
139 //
140 #define KeGetContextSwitches(Prcb) \
141 CONTAINING_RECORD(Prcb, KIPCR, PrcbData)->ContextSwitches
142
143 //
144 // Macro to get the second level cache size field name which differs between
145 // CISC and RISC architectures, as the former has unified I/D cache
146 //
147 #define KiGetSecondLevelDCacheSize() ((PKIPCR)KeGetPcr())->SecondLevelCacheSize
148
149 //
150 // Returns the Interrupt State from a Trap Frame.
151 // ON = TRUE, OFF = FALSE
152 //
153 #define KeGetTrapFrameInterruptState(TrapFrame) \
154 BooleanFlagOn((TrapFrame)->EFlags, EFLAGS_INTERRUPT_MASK)
155
156 //
157 // Flags for exiting a trap
158 //
159 #define KTE_SKIP_PM_BIT (((KTRAP_EXIT_SKIP_BITS) { { .SkipPreviousMode = TRUE } }).Bits)
160 #define KTE_SKIP_SEG_BIT (((KTRAP_EXIT_SKIP_BITS) { { .SkipSegments = TRUE } }).Bits)
161 #define KTE_SKIP_VOL_BIT (((KTRAP_EXIT_SKIP_BITS) { { .SkipVolatiles = TRUE } }).Bits)
162
163 typedef union _KTRAP_EXIT_SKIP_BITS
164 {
165 struct
166 {
167 UCHAR SkipPreviousMode:1;
168 UCHAR SkipSegments:1;
169 UCHAR SkipVolatiles:1;
170 UCHAR Reserved:5;
171 };
172 UCHAR Bits;
173 } KTRAP_EXIT_SKIP_BITS, *PKTRAP_EXIT_SKIP_BITS;
174
175
176 //
177 // Flags used by the VDM/V8086 emulation engine for determining instruction prefixes
178 //
179 #define PFX_FLAG_ES 0x00000100
180 #define PFX_FLAG_CS 0x00000200
181 #define PFX_FLAG_SS 0x00000400
182 #define PFX_FLAG_DS 0x00000800
183 #define PFX_FLAG_FS 0x00001000
184 #define PFX_FLAG_GS 0x00002000
185 #define PFX_FLAG_OPER32 0x00004000
186 #define PFX_FLAG_ADDR32 0x00008000
187 #define PFX_FLAG_LOCK 0x00010000
188 #define PFX_FLAG_REPNE 0x00020000
189 #define PFX_FLAG_REP 0x00040000
190
191 //
192 // VDM Helper Macros
193 //
194 // All VDM/V8086 opcode emulators have the same FASTCALL function definition.
195 // We need to keep 2 parameters while the original ASM implementation uses 4:
196 // TrapFrame, PrefixFlags, Eip, InstructionSize;
197 //
198 // We pass the trap frame, and prefix flags, in our two parameters.
199 //
200 // We then realize that since the smallest prefix flag is 0x100, this gives us
201 // a count of up to 0xFF. So we OR in the instruction size with the prefix flags
202 //
203 // We further realize that we always have access to EIP from the trap frame, and
204 // that if we want the *current instruction* EIP, we simply have to add the
205 // instruction size *MINUS ONE*, and that gives us the EIP we should be looking
206 // at now, so we don't need to use the stack to push this parameter.
207 //
208 // We actually only care about the *current instruction* EIP in one location,
209 // so although it may be slightly more expensive to re-calculate the EIP one
210 // more time, this way we don't redefine ALL opcode handlers to have 3 parameters,
211 // which would be forcing stack usage in all other scenarios.
212 //
213 #define KiVdmSetVdmEFlags(x) InterlockedOr((PLONG)KiNtVdmState, (x));
214 #define KiVdmClearVdmEFlags(x) InterlockedAnd((PLONG)KiNtVdmState, ~(x))
215 #define KiCallVdmHandler(x) KiVdmOpcode##x(TrapFrame, Flags)
216 #define KiCallVdmPrefixHandler(x) KiVdmOpcodePrefix(TrapFrame, Flags | x)
217 #define KiVdmUnhandledOpcode(x) \
218 BOOLEAN \
219 FASTCALL \
220 KiVdmOpcode##x(IN PKTRAP_FRAME TrapFrame, \
221 IN ULONG Flags) \
222 { \
223 /* Not yet handled */ \
224 UNIMPLEMENTED_DBGBREAK(); \
225 return FALSE; \
226 }
227
228 C_ASSERT(NPX_FRAME_LENGTH == sizeof(FX_SAVE_AREA));
229
230 //
231 // Local parameters
232 //
233 typedef struct _KV86_FRAME
234 {
235 PVOID ThreadStack;
236 PVOID ThreadTeb;
237 PVOID PcrTeb;
238 } KV86_FRAME, *PKV86_FRAME;
239
240 //
241 // Virtual Stack Frame
242 //
243 typedef struct _KV8086_STACK_FRAME
244 {
245 KTRAP_FRAME TrapFrame;
246 FX_SAVE_AREA NpxArea;
247 KV86_FRAME V86Frame;
248 } KV8086_STACK_FRAME, *PKV8086_STACK_FRAME;
249
250 //
251 // Large Pages Support
252 //
253 typedef struct _LARGE_IDENTITY_MAP
254 {
255 PHARDWARE_PTE TopLevelDirectory;
256 ULONG Cr3;
257 ULONG_PTR StartAddress;
258 ULONG PagesCount;
259 PVOID PagesList[30];
260 } LARGE_IDENTITY_MAP, *PLARGE_IDENTITY_MAP;
261
262 //
263 // Floating Point Internal Context Structure
264 //
265 typedef struct _FLOATING_SAVE_CONTEXT
266 {
267 PKTHREAD CurrentThread;
268 KIRQL OldNpxIrql;
269 PFX_SAVE_AREA Buffer;
270 PFX_SAVE_AREA PfxSaveArea;
271 } FLOATING_SAVE_CONTEXT, *PFLOATING_SAVE_CONTEXT;
272
273 /* Diable interrupts and return whether they were enabled before */
274 FORCEINLINE
275 BOOLEAN
KeDisableInterrupts(VOID)276 KeDisableInterrupts(VOID)
277 {
278 ULONG Flags;
279 BOOLEAN Return;
280
281 /* Get EFLAGS and check if the interrupt bit is set */
282 Flags = __readeflags();
283 Return = (Flags & EFLAGS_INTERRUPT_MASK) ? TRUE: FALSE;
284
285 /* Disable interrupts */
286 _disable();
287 return Return;
288 }
289
290 /* Restore previous interrupt state */
291 FORCEINLINE
292 VOID
KeRestoreInterrupts(BOOLEAN WereEnabled)293 KeRestoreInterrupts(BOOLEAN WereEnabled)
294 {
295 if (WereEnabled) _enable();
296 }
297
298 //
299 // Registers an interrupt handler with an IDT vector
300 //
301 FORCEINLINE
302 VOID
KeRegisterInterruptHandler(IN ULONG Vector,IN PVOID Handler)303 KeRegisterInterruptHandler(IN ULONG Vector,
304 IN PVOID Handler)
305 {
306 UCHAR Entry;
307 ULONG_PTR Address;
308 PKIPCR Pcr = (PKIPCR)KeGetPcr();
309
310 //
311 // Get the entry from the HAL
312 //
313 Entry = HalVectorToIDTEntry(Vector);
314 Address = PtrToUlong(Handler);
315
316 //
317 // Now set the data
318 //
319 Pcr->IDT[Entry].ExtendedOffset = (USHORT)(Address >> 16);
320 Pcr->IDT[Entry].Offset = (USHORT)Address;
321 }
322
323 //
324 // Returns the registered interrupt handler for a given IDT vector
325 //
326 FORCEINLINE
327 PVOID
KeQueryInterruptHandler(IN ULONG Vector)328 KeQueryInterruptHandler(IN ULONG Vector)
329 {
330 PKIPCR Pcr = (PKIPCR)KeGetPcr();
331 UCHAR Entry;
332
333 //
334 // Get the entry from the HAL
335 //
336 Entry = HalVectorToIDTEntry(Vector);
337
338 //
339 // Read the entry from the IDT
340 //
341 return (PVOID)(((Pcr->IDT[Entry].ExtendedOffset << 16) & 0xFFFF0000) |
342 (Pcr->IDT[Entry].Offset & 0xFFFF));
343 }
344
345 //
346 // Invalidates the TLB entry for a specified address
347 //
348 FORCEINLINE
349 VOID
KeInvalidateTlbEntry(IN PVOID Address)350 KeInvalidateTlbEntry(IN PVOID Address)
351 {
352 /* Invalidate the TLB entry for this address */
353 __invlpg(Address);
354 }
355
356 FORCEINLINE
357 VOID
KeFlushProcessTb(VOID)358 KeFlushProcessTb(VOID)
359 {
360 /* Flush the TLB by resetting CR3 */
361 __writecr3(__readcr3());
362 }
363
364 FORCEINLINE
365 VOID
KeSweepICache(IN PVOID BaseAddress,IN SIZE_T FlushSize)366 KeSweepICache(IN PVOID BaseAddress,
367 IN SIZE_T FlushSize)
368 {
369 //
370 // Always sweep the whole cache
371 //
372 UNREFERENCED_PARAMETER(BaseAddress);
373 UNREFERENCED_PARAMETER(FlushSize);
374 __wbinvd();
375 }
376
377 FORCEINLINE
378 PRKTHREAD
KeGetCurrentThread(VOID)379 KeGetCurrentThread(VOID)
380 {
381 /* Return the current thread */
382 return ((PKIPCR)KeGetPcr())->PrcbData.CurrentThread;
383 }
384
385 FORCEINLINE
386 VOID
KiRundownThread(IN PKTHREAD Thread)387 KiRundownThread(IN PKTHREAD Thread)
388 {
389 /* Check if this is the NPX Thread */
390 if (KeGetCurrentPrcb()->NpxThread == Thread)
391 {
392 /* Clear it */
393 KeGetCurrentPrcb()->NpxThread = NULL;
394 Ke386FnInit();
395 }
396 }
397
398 CODE_SEG("INIT")
399 VOID
400 NTAPI
401 KiInitializePcr(IN ULONG ProcessorNumber,
402 IN PKIPCR Pcr,
403 IN PKIDTENTRY Idt,
404 IN PKGDTENTRY Gdt,
405 IN PKTSS Tss,
406 IN PKTHREAD IdleThread,
407 IN PVOID DpcStack);
408
409 FORCEINLINE
410 VOID
Ke386SetGdtEntryBase(PKGDTENTRY GdtEntry,PVOID BaseAddress)411 Ke386SetGdtEntryBase(PKGDTENTRY GdtEntry, PVOID BaseAddress)
412 {
413 GdtEntry->BaseLow = (USHORT)((ULONG_PTR)BaseAddress & 0xFFFF);
414 GdtEntry->HighWord.Bytes.BaseMid = (UCHAR)((ULONG_PTR)BaseAddress >> 16);
415 GdtEntry->HighWord.Bytes.BaseHi = (UCHAR)((ULONG_PTR)BaseAddress >> 24);
416 }
417
418 FORCEINLINE
419 VOID
KiSetTebBase(PKPCR Pcr,PNT_TIB TebAddress)420 KiSetTebBase(PKPCR Pcr, PNT_TIB TebAddress)
421 {
422 Pcr->NtTib.Self = TebAddress;
423 Ke386SetGdtEntryBase(&Pcr->GDT[KGDT_R3_TEB / sizeof(KGDTENTRY)], TebAddress);
424 }
425
426 CODE_SEG("INIT")
427 VOID
428 FASTCALL
429 Ki386InitializeTss(
430 IN PKTSS Tss,
431 IN PKIDTENTRY Idt,
432 IN PKGDTENTRY Gdt
433 );
434
435 CODE_SEG("INIT")
436 VOID
437 NTAPI
438 KiSetCR0Bits(VOID);
439
440 CODE_SEG("INIT")
441 VOID
442 NTAPI
443 KiGetCacheInformation(VOID);
444
445 CODE_SEG("INIT")
446 BOOLEAN
447 NTAPI
448 KiIsNpxPresent(
449 VOID
450 );
451
452 CODE_SEG("INIT")
453 BOOLEAN
454 NTAPI
455 KiIsNpxErrataPresent(
456 VOID
457 );
458
459 CODE_SEG("INIT")
460 VOID
461 NTAPI
462 KiSetProcessorType(VOID);
463
464 CODE_SEG("INIT")
465 ULONG
466 NTAPI
467 KiGetFeatureBits(VOID);
468
469 #if DBG
470 CODE_SEG("INIT")
471 VOID
472 KiReportCpuFeatures(VOID);
473 #endif
474
475 VOID
476 NTAPI
477 KiThreadStartup(VOID);
478
479 NTSTATUS
480 NTAPI
481 Ke386GetGdtEntryThread(
482 IN PKTHREAD Thread,
483 IN ULONG Offset,
484 IN PKGDTENTRY Descriptor
485 );
486
487 VOID
488 NTAPI
489 KiFlushNPXState(
490 IN FLOATING_SAVE_AREA *SaveArea
491 );
492
493 VOID
494 NTAPI
495 Ki386AdjustEsp0(
496 IN PKTRAP_FRAME TrapFrame
497 );
498
499 VOID
500 NTAPI
501 Ki386SetupAndExitToV86Mode(
502 OUT PTEB VdmTeb
503 );
504
505 CODE_SEG("INIT")
506 VOID
507 NTAPI
508 KeI386VdmInitialize(
509 VOID
510 );
511
512 CODE_SEG("INIT")
513 ULONG_PTR
514 NTAPI
515 Ki386EnableGlobalPage(
516 IN ULONG_PTR Context
517 );
518
519 CODE_SEG("INIT")
520 ULONG_PTR
521 NTAPI
522 Ki386EnableTargetLargePage(
523 IN ULONG_PTR Context
524 );
525
526 BOOLEAN
527 NTAPI
528 Ki386CreateIdentityMap(
529 IN PLARGE_IDENTITY_MAP IdentityMap,
530 IN PVOID StartPtr,
531 IN ULONG Length
532 );
533
534 VOID
535 NTAPI
536 Ki386FreeIdentityMap(
537 IN PLARGE_IDENTITY_MAP IdentityMap
538 );
539
540 VOID
541 NTAPI
542 Ki386EnableCurrentLargePage(
543 IN ULONG_PTR StartAddress,
544 IN ULONG Cr3
545 );
546
547 CODE_SEG("INIT")
548 VOID
549 NTAPI
550 KiI386PentiumLockErrataFixup(
551 VOID
552 );
553
554 CODE_SEG("INIT")
555 VOID
556 NTAPI
557 KiInitializePAT(
558 VOID
559 );
560
561 CODE_SEG("INIT")
562 VOID
563 NTAPI
564 KiInitializeMTRR(
565 IN BOOLEAN FinalCpu
566 );
567
568 CODE_SEG("INIT")
569 VOID
570 NTAPI
571 KiAmdK6InitializeMTRR(
572 VOID
573 );
574
575 CODE_SEG("INIT")
576 VOID
577 NTAPI
578 KiRestoreFastSyscallReturnState(
579 VOID
580 );
581
582 CODE_SEG("INIT")
583 ULONG_PTR
584 NTAPI
585 Ki386EnableDE(
586 IN ULONG_PTR Context
587 );
588
589 CODE_SEG("INIT")
590 ULONG_PTR
591 NTAPI
592 Ki386EnableFxsr(
593 IN ULONG_PTR Context
594 );
595
596 CODE_SEG("INIT")
597 ULONG_PTR
598 NTAPI
599 Ki386EnableXMMIExceptions(
600 IN ULONG_PTR Context
601 );
602
603 BOOLEAN
604 NTAPI
605 VdmDispatchBop(
606 IN PKTRAP_FRAME TrapFrame
607 );
608
609 BOOLEAN
610 NTAPI
611 VdmDispatchPageFault(
612 _In_ PKTRAP_FRAME TrapFrame
613 );
614
615 BOOLEAN
616 FASTCALL
617 KiVdmOpcodePrefix(
618 IN PKTRAP_FRAME TrapFrame,
619 IN ULONG Flags
620 );
621
622 BOOLEAN
623 FASTCALL
624 Ki386HandleOpcodeV86(
625 IN PKTRAP_FRAME TrapFrame
626 );
627
628 DECLSPEC_NORETURN
629 VOID
630 FASTCALL
631 KiEoiHelper(
632 IN PKTRAP_FRAME TrapFrame
633 );
634
635 VOID
636 FASTCALL
637 Ki386BiosCallReturnAddress(
638 IN PKTRAP_FRAME TrapFrame
639 );
640
641 ULONG_PTR
642 FASTCALL
643 KiExitV86Mode(
644 IN PKTRAP_FRAME TrapFrame
645 );
646
647 DECLSPEC_NORETURN
648 VOID
649 NTAPI
650 KiDispatchExceptionFromTrapFrame(
651 IN NTSTATUS Code,
652 IN ULONG Flags,
653 IN ULONG_PTR Address,
654 IN ULONG ParameterCount,
655 IN ULONG_PTR Parameter1,
656 IN ULONG_PTR Parameter2,
657 IN ULONG_PTR Parameter3,
658 IN PKTRAP_FRAME TrapFrame
659 );
660
661 NTSTATUS
662 NTAPI
663 KiConvertToGuiThread(
664 VOID
665 );
666
667 DECLSPEC_NORETURN
668 VOID
669 FASTCALL
670 KiServiceExit(
671 IN PKTRAP_FRAME TrapFrame,
672 IN NTSTATUS Status
673 );
674
675 DECLSPEC_NORETURN
676 VOID
677 FASTCALL
678 KiServiceExit2(
679 IN PKTRAP_FRAME TrapFrame
680 );
681
682 FORCEINLINE
683 DECLSPEC_NORETURN
684 VOID
KiExceptionExit(_In_ PKTRAP_FRAME TrapFrame,_In_ PKEXCEPTION_FRAME ExceptionFrame)685 KiExceptionExit(
686 _In_ PKTRAP_FRAME TrapFrame,
687 _In_ PKEXCEPTION_FRAME ExceptionFrame)
688 {
689 KiServiceExit2(TrapFrame);
690 }
691
692 //
693 // Global x86 only Kernel data
694 //
695 extern PVOID Ki386IopmSaveArea;
696 extern ULONG KeI386EFlagsAndMaskV86;
697 extern ULONG KeI386EFlagsOrMaskV86;
698 extern BOOLEAN KeI386VirtualIntExtensions;
699 extern KIDTENTRY KiIdt[MAXIMUM_IDTVECTOR+1];
700 extern KDESCRIPTOR KiIdtDescriptor;
701 extern BOOLEAN KiI386PentiumLockErrataPresent;
702 extern ULONG KeI386NpxPresent;
703 extern ULONG KeI386XMMIPresent;
704 extern ULONG KeI386FxsrPresent;
705 extern ULONG KiMXCsrMask;
706 extern ULONG KeI386CpuType;
707 extern ULONG KeI386CpuStep;
708 extern ULONG KiFastSystemCallDisable;
709 extern UCHAR KiDebugRegisterTrapOffsets[9];
710 extern UCHAR KiDebugRegisterContextOffsets[9];
711 extern VOID __cdecl KiTrap02(VOID);
712 extern VOID __cdecl KiTrap08(VOID);
713 extern VOID __cdecl KiTrap13(VOID);
714 extern VOID __cdecl KiFastCallEntry(VOID);
715 extern VOID NTAPI ExpInterlockedPopEntrySListFault(VOID);
716 extern VOID NTAPI ExpInterlockedPopEntrySListResume(VOID);
717 extern VOID __cdecl CopyParams(VOID);
718 extern VOID __cdecl ReadBatch(VOID);
719 extern CHAR KiSystemCallExitBranch[];
720 extern CHAR KiSystemCallExit[];
721 extern CHAR KiSystemCallExit2[];
722
723 //
724 // Trap Macros
725 //
726 #include "trap_x.h"
727
728 //
729 // Returns a thread's FPU save area
730 //
731 FORCEINLINE
732 PFX_SAVE_AREA
KiGetThreadNpxArea(IN PKTHREAD Thread)733 KiGetThreadNpxArea(IN PKTHREAD Thread)
734 {
735 ASSERT((ULONG_PTR)Thread->InitialStack % 16 == 0);
736 return (PFX_SAVE_AREA)((ULONG_PTR)Thread->InitialStack - sizeof(FX_SAVE_AREA));
737 }
738
739 //
740 // Sanitizes a selector
741 //
742 FORCEINLINE
743 ULONG
Ke386SanitizeSeg(IN ULONG Cs,IN KPROCESSOR_MODE Mode)744 Ke386SanitizeSeg(IN ULONG Cs,
745 IN KPROCESSOR_MODE Mode)
746 {
747 //
748 // Check if we're in kernel-mode, and force CPL 0 if so.
749 // Otherwise, force CPL 3.
750 //
751 return ((Mode == KernelMode) ?
752 (Cs & (0xFFFF & ~RPL_MASK)) :
753 (RPL_MASK | (Cs & 0xFFFF)));
754 }
755
756 //
757 // Sanitizes EFLAGS
758 //
759 FORCEINLINE
760 ULONG
Ke386SanitizeFlags(IN ULONG Eflags,IN KPROCESSOR_MODE Mode)761 Ke386SanitizeFlags(IN ULONG Eflags,
762 IN KPROCESSOR_MODE Mode)
763 {
764 //
765 // Check if we're in kernel-mode, and sanitize EFLAGS if so.
766 // Otherwise, also force interrupt mask on.
767 //
768 return ((Mode == KernelMode) ?
769 (Eflags & (EFLAGS_USER_SANITIZE | EFLAGS_INTERRUPT_MASK)) :
770 (EFLAGS_INTERRUPT_MASK | (Eflags & EFLAGS_USER_SANITIZE)));
771 }
772
773 //
774 // Sanitizes a Debug Register
775 //
776 FORCEINLINE
777 PVOID
Ke386SanitizeDr(IN PVOID DrAddress,IN KPROCESSOR_MODE Mode)778 Ke386SanitizeDr(IN PVOID DrAddress,
779 IN KPROCESSOR_MODE Mode)
780 {
781 //
782 // Check if we're in kernel-mode, and return the address directly if so.
783 // Otherwise, make sure it's not inside the kernel-mode address space.
784 // If it is, then clear the address.
785 //
786 return ((Mode == KernelMode) ? DrAddress :
787 (DrAddress <= MM_HIGHEST_USER_ADDRESS) ? DrAddress : 0);
788 }
789
790 //
791 // Exception with no arguments
792 //
793 FORCEINLINE
794 DECLSPEC_NORETURN
795 VOID
KiDispatchException0Args(IN NTSTATUS Code,IN ULONG_PTR Address,IN PKTRAP_FRAME TrapFrame)796 KiDispatchException0Args(IN NTSTATUS Code,
797 IN ULONG_PTR Address,
798 IN PKTRAP_FRAME TrapFrame)
799 {
800 /* Helper for exceptions with no arguments */
801 KiDispatchExceptionFromTrapFrame(Code, 0, Address, 0, 0, 0, 0, TrapFrame);
802 }
803
804 //
805 // Exception with one argument
806 //
807 FORCEINLINE
808 DECLSPEC_NORETURN
809 VOID
KiDispatchException1Args(IN NTSTATUS Code,IN ULONG_PTR Address,IN ULONG P1,IN PKTRAP_FRAME TrapFrame)810 KiDispatchException1Args(IN NTSTATUS Code,
811 IN ULONG_PTR Address,
812 IN ULONG P1,
813 IN PKTRAP_FRAME TrapFrame)
814 {
815 /* Helper for exceptions with no arguments */
816 KiDispatchExceptionFromTrapFrame(Code, 0, Address, 1, P1, 0, 0, TrapFrame);
817 }
818
819 //
820 // Exception with two arguments
821 //
822 FORCEINLINE
823 DECLSPEC_NORETURN
824 VOID
KiDispatchException2Args(IN NTSTATUS Code,IN ULONG_PTR Address,IN ULONG P1,IN ULONG P2,IN PKTRAP_FRAME TrapFrame)825 KiDispatchException2Args(IN NTSTATUS Code,
826 IN ULONG_PTR Address,
827 IN ULONG P1,
828 IN ULONG P2,
829 IN PKTRAP_FRAME TrapFrame)
830 {
831 /* Helper for exceptions with no arguments */
832 KiDispatchExceptionFromTrapFrame(Code, 0, Address, 2, P1, P2, 0, TrapFrame);
833 }
834
835 //
836 // Performs a system call
837 //
838 NTSTATUS
839 NTAPI
840 KiSystemCallTrampoline(_In_ PVOID Handler,
841 _In_ PVOID Arguments,
842 _In_ ULONG StackBytes);
843
844
845 //
846 // Checks for pending APCs
847 //
848 FORCEINLINE
849 VOID
KiCheckForApcDelivery(IN PKTRAP_FRAME TrapFrame)850 KiCheckForApcDelivery(IN PKTRAP_FRAME TrapFrame)
851 {
852 PKTHREAD Thread;
853 KIRQL OldIrql;
854
855 /* Check for V8086 or user-mode trap */
856 if ((TrapFrame->EFlags & EFLAGS_V86_MASK) || (KiUserTrap(TrapFrame)))
857 {
858 /* Get the thread */
859 Thread = KeGetCurrentThread();
860 while (TRUE)
861 {
862 /* Turn off the alerted state for kernel mode */
863 Thread->Alerted[KernelMode] = FALSE;
864
865 /* Are there pending user APCs? */
866 if (!Thread->ApcState.UserApcPending) break;
867
868 /* Raise to APC level and enable interrupts */
869 OldIrql = KfRaiseIrql(APC_LEVEL);
870 _enable();
871
872 /* Deliver APCs */
873 KiDeliverApc(UserMode, NULL, TrapFrame);
874
875 /* Restore IRQL and disable interrupts once again */
876 KfLowerIrql(OldIrql);
877 _disable();
878 }
879 }
880 }
881
882 //
883 // Switches from boot loader to initial kernel stack
884 //
885 CODE_SEG("INIT")
886 FORCEINLINE
887 DECLSPEC_NORETURN
888 VOID
KiSwitchToBootStack(IN ULONG_PTR InitialStack)889 KiSwitchToBootStack(IN ULONG_PTR InitialStack)
890 {
891 CODE_SEG("INIT") DECLSPEC_NORETURN VOID NTAPI KiSystemStartupBootStack(VOID);
892
893 /* We have to switch to a new stack before continuing kernel initialization */
894 #ifdef __GNUC__
895 __asm__
896 (
897 "movl %0, %%esp\n\t"
898 "subl %1, %%esp\n\t"
899 "pushl %2\n\t"
900 "jmp _KiSystemStartupBootStack@0"
901 :
902 : "c"(InitialStack),
903 "i"(NPX_FRAME_LENGTH + KTRAP_FRAME_ALIGN + KTRAP_FRAME_LENGTH),
904 "i"(CR0_EM | CR0_TS | CR0_MP),
905 "p"(KiSystemStartupBootStack)
906 : "%esp"
907 );
908 #elif defined(_MSC_VER)
909 __asm
910 {
911 mov esp, InitialStack
912 sub esp, (NPX_FRAME_LENGTH + KTRAP_FRAME_ALIGN + KTRAP_FRAME_LENGTH)
913 push (CR0_EM | CR0_TS | CR0_MP)
914 jmp KiSystemStartupBootStack
915 }
916 #else
917 #error Unknown Compiler
918 #endif
919
920 UNREACHABLE;
921 }
922
923 //
924 // Emits the iret instruction for C code
925 //
926 FORCEINLINE
927 DECLSPEC_NORETURN
928 VOID
KiIret(VOID)929 KiIret(VOID)
930 {
931 #if defined(__GNUC__)
932 __asm__ __volatile__
933 (
934 "iret"
935 );
936 #elif defined(_MSC_VER)
937 __asm
938 {
939 iretd
940 }
941 #else
942 #error Unsupported compiler
943 #endif
944 UNREACHABLE;
945 }
946
947 //
948 // Normally this is done by the HAL, but on x86 as an optimization, the kernel
949 // initiates the end by calling back into the HAL and exiting the trap here.
950 //
951 FORCEINLINE
952 VOID
KiEndInterrupt(IN KIRQL Irql,IN PKTRAP_FRAME TrapFrame)953 KiEndInterrupt(IN KIRQL Irql,
954 IN PKTRAP_FRAME TrapFrame)
955 {
956 /* Disable interrupts and end the interrupt */
957 _disable();
958 HalEndSystemInterrupt(Irql, TrapFrame);
959
960 /* Exit the interrupt */
961 KiEoiHelper(TrapFrame);
962 }
963
964 //
965 // PERF Code
966 //
967 FORCEINLINE
968 VOID
Ki386PerfEnd(VOID)969 Ki386PerfEnd(VOID)
970 {
971 extern ULONGLONG BootCyclesEnd, BootCycles;
972 BootCyclesEnd = __rdtsc();
973 DbgPrint("Boot took %I64u cycles!\n", BootCyclesEnd - BootCycles);
974 DbgPrint("Interrupts: %u System Calls: %u Context Switches: %u\n",
975 KeGetCurrentPrcb()->InterruptCount,
976 KeGetCurrentPrcb()->KeSystemCalls,
977 KeGetContextSwitches(KeGetCurrentPrcb()));
978 }
979
980 FORCEINLINE
981 PULONG
KiGetUserModeStackAddress(void)982 KiGetUserModeStackAddress(void)
983 {
984 return &(KeGetCurrentThread()->TrapFrame->HardwareEsp);
985 }
986
987 #ifdef __cplusplus
988 } // extern "C"
989 #endif
990
991 #endif
992