1 #ifndef __NTOSKRNL_INCLUDE_INTERNAL_AMD64_KE_H
2 #define __NTOSKRNL_INCLUDE_INTERNAL_AMD64_KE_H
3
4 #ifdef __cplusplus
5 extern "C" {
6 #endif
7
8 #define X86_EFLAGS_TF 0x00000100 /* Trap flag */
9 #define X86_EFLAGS_IF 0x00000200 /* Interrupt Enable flag */
10 #define X86_EFLAGS_IOPL 0x00003000 /* I/O Privilege Level bits */
11 #define X86_EFLAGS_NT 0x00004000 /* Nested Task flag */
12 #define X86_EFLAGS_RF 0x00010000 /* Resume flag */
13 #define X86_EFLAGS_VM 0x00020000 /* Virtual Mode */
14 #define X86_EFLAGS_ID 0x00200000 /* CPUID detection flag */
15
16 #define X86_CR0_PE 0x00000001 /* enable Protected Mode */
17 #define X86_CR0_NE 0x00000020 /* enable native FPU error reporting */
18 #define X86_CR0_TS 0x00000008 /* enable exception on FPU instruction for task switch */
19 #define X86_CR0_EM 0x00000004 /* enable FPU emulation (disable FPU) */
20 #define X86_CR0_MP 0x00000002 /* enable FPU monitoring */
21 #define X86_CR0_WP 0x00010000 /* enable Write Protect (copy on write) */
22 #define X86_CR0_PG 0x80000000 /* enable Paging */
23
24 #define X86_CR4_PAE 0x00000020 /* enable physical address extensions */
25 #define X86_CR4_PGE 0x00000080 /* enable global pages */
26 #define X86_CR4_OSFXSR 0x00000200 /* enable FXSAVE/FXRSTOR instructions */
27 #define X86_CR4_OSXMMEXCPT 0x00000400 /* enable #XF exception */
28
29 /* EDX flags */
30 #define X86_FEATURE_FPU 0x00000001 /* x87 FPU is present */
31 #define X86_FEATURE_VME 0x00000002 /* Virtual 8086 Extensions are present */
32 #define X86_FEATURE_DBG 0x00000004 /* Debugging extensions are present */
33 #define X86_FEATURE_PSE 0x00000008 /* Page Size Extension is present */
34 #define X86_FEATURE_TSC 0x00000010 /* time stamp counters are present */
35 #define X86_FEATURE_PAE 0x00000040 /* physical address extension is present */
36 #define X86_FEATURE_CX8 0x00000100 /* CMPXCHG8B instruction present */
37 #define X86_FEATURE_SYSCALL 0x00000800 /* SYSCALL/SYSRET support present */
38 #define X86_FEATURE_MTTR 0x00001000 /* Memory type range registers are present */
39 #define X86_FEATURE_PGE 0x00002000 /* Page Global Enable */
40 #define X86_FEATURE_CMOV 0x00008000 /* "Conditional move" instruction supported */
41 #define X86_FEATURE_PAT 0x00010000 /* Page Attribute Table is supported */
42 #define X86_FEATURE_DS 0x00200000 /* Debug Store is present */
43 #define X86_FEATURE_MMX 0x00800000 /* MMX extension present */
44 #define X86_FEATURE_FXSR 0x01000000 /* FXSAVE/FXRSTOR instructions present */
45 #define X86_FEATURE_SSE 0x02000000 /* SSE extension present */
46 #define X86_FEATURE_SSE2 0x04000000 /* SSE2 extension present */
47 #define X86_FEATURE_HT 0x10000000 /* Hyper-Threading present */
48
49 /* ECX flags */
50 #define X86_FEATURE_SSE3 0x00000001 /* SSE3 is supported */
51 #define X86_FEATURE_MONITOR 0x00000008 /* SSE3 Monitor instructions supported */
52 #define X86_FEATURE_VMX 0x00000020 /* Virtual Machine eXtensions are available */
53 #define X86_FEATURE_SSSE3 0x00000200 /* Supplemental SSE3 are available */
54 #define X86_FEATURE_FMA3 0x00001000 /* Fused multiple-add supported */
55 #define X86_FEATURE_CX16 0x00002000 /* CMPXCHG16B instruction are available */
56 #define X86_FEATURE_PCID 0x00020000 /* Process Context IDentifiers are supported */
57 #define X86_FEATURE_SSE41 0x00080000 /* SSE 4.1 is supported */
58 #define X86_FEATURE_SSE42 0x00100000 /* SSE 4.2 is supported */
59 #define X86_FEATURE_POPCNT 0x00800000 /* POPCNT instruction is available */
60 #define X86_FEATURE_XSAVE 0x04000000 /* XSAVE family are available */
61
62 /* EDX extended flags */
63 #define X86_FEATURE_NX 0x00100000 /* NX support present */
64
65 #define X86_EXT_FEATURE_SSE3 0x00000001 /* SSE3 extension present */
66 #define X86_EXT_FEATURE_3DNOW 0x40000000 /* 3DNOW! extension present */
67
68 #define FRAME_EDITED 0xFFF8
69
70 #define X86_MSR_GSBASE 0xC0000101
71 #define X86_MSR_KERNEL_GSBASE 0xC0000102
72 #define X86_MSR_EFER 0xC0000080
73 #define X86_MSR_STAR 0xC0000081
74 #define X86_MSR_LSTAR 0xC0000082
75 #define X86_MSR_CSTAR 0xC0000083
76 #define X86_MSR_SFMASK 0xC0000084
77
78 #define EFER_SCE 0x0001
79 #define EFER_LME 0x0100
80 #define EFER_LMA 0x0400
81 #define EFER_NXE 0x0800
82 #define EFER_SVME 0x1000
83 #define EFER_FFXSR 0x4000
84
85 #define AMD64_TSS 9
86
87 #define APIC_EOI_REGISTER 0xFFFFFFFFFFFE00B0ULL
88
89 #ifndef __ASM__
90
91 #include "intrin_i.h"
92
93 typedef struct _KIDT_INIT
94 {
95 UCHAR InterruptId;
96 UCHAR Dpl;
97 UCHAR IstIndex;
98 PVOID ServiceRoutine;
99 } KIDT_INIT, *PKIDT_INIT;
100
101 #include <pshpack1.h>
102 typedef struct _KI_INTERRUPT_DISPATCH_ENTRY
103 {
104 UCHAR _Op_nop;
105 UCHAR _Op_push;
106 UCHAR _Vector;
107 UCHAR _Op_jmp;
108 ULONG RelativeAddress;
109 } KI_INTERRUPT_DISPATCH_ENTRY, *PKI_INTERRUPT_DISPATCH_ENTRY;
110 #include <poppack.h>
111
112 extern ULONG KeI386NpxPresent;
113 extern ULONG KeI386XMMIPresent;
114 extern ULONG KeI386FxsrPresent;
115 extern ULONG KeI386CpuType;
116 extern ULONG KeI386CpuStep;
117
118 //
119 // INT3 is 1 byte long
120 //
121 #define KD_BREAKPOINT_TYPE UCHAR
122 #define KD_BREAKPOINT_SIZE sizeof(UCHAR)
123 #define KD_BREAKPOINT_VALUE 0xCC
124
125 //
126 // One-liners for getting and setting special purpose registers in portable code
127 //
128 FORCEINLINE
129 ULONG_PTR
KeGetContextPc(PCONTEXT Context)130 KeGetContextPc(PCONTEXT Context)
131 {
132 return Context->Rip;
133 }
134
135 FORCEINLINE
136 VOID
KeSetContextPc(PCONTEXT Context,ULONG_PTR ProgramCounter)137 KeSetContextPc(PCONTEXT Context, ULONG_PTR ProgramCounter)
138 {
139 Context->Rip = ProgramCounter;
140 }
141
142 FORCEINLINE
143 ULONG_PTR
KeGetContextReturnRegister(PCONTEXT Context)144 KeGetContextReturnRegister(PCONTEXT Context)
145 {
146 return Context->Rax;
147 }
148
149 FORCEINLINE
150 VOID
KeSetContextReturnRegister(PCONTEXT Context,ULONG_PTR ReturnValue)151 KeSetContextReturnRegister(PCONTEXT Context, ULONG_PTR ReturnValue)
152 {
153 Context->Rax = ReturnValue;
154 }
155
156 FORCEINLINE
157 ULONG_PTR
KeGetContextStackRegister(PCONTEXT Context)158 KeGetContextStackRegister(PCONTEXT Context)
159 {
160 return Context->Rsp;
161 }
162
163 FORCEINLINE
164 ULONG_PTR
KeGetContextFrameRegister(PCONTEXT Context)165 KeGetContextFrameRegister(PCONTEXT Context)
166 {
167 return Context->Rbp;
168 }
169
170 FORCEINLINE
171 VOID
KeSetContextFrameRegister(PCONTEXT Context,ULONG_PTR Frame)172 KeSetContextFrameRegister(PCONTEXT Context, ULONG_PTR Frame)
173 {
174 Context->Rbp = Frame;
175 }
176
177 FORCEINLINE
178 ULONG_PTR
KeGetTrapFramePc(PKTRAP_FRAME TrapFrame)179 KeGetTrapFramePc(PKTRAP_FRAME TrapFrame)
180 {
181 return TrapFrame->Rip;
182 }
183
184 FORCEINLINE
185 PKTRAP_FRAME
KiGetLinkedTrapFrame(PKTRAP_FRAME TrapFrame)186 KiGetLinkedTrapFrame(PKTRAP_FRAME TrapFrame)
187 {
188 return (PKTRAP_FRAME)TrapFrame->TrapFrame;
189 }
190
191 FORCEINLINE
192 ULONG_PTR
KeGetTrapFrameStackRegister(PKTRAP_FRAME TrapFrame)193 KeGetTrapFrameStackRegister(PKTRAP_FRAME TrapFrame)
194 {
195 return TrapFrame->Rsp;
196 }
197
198 FORCEINLINE
199 ULONG_PTR
KeGetTrapFrameFrameRegister(PKTRAP_FRAME TrapFrame)200 KeGetTrapFrameFrameRegister(PKTRAP_FRAME TrapFrame)
201 {
202 return TrapFrame->Rbp;
203 }
204
205 //
206 // Macro to get trap and exception frame from a thread stack
207 //
208 #define KeGetTrapFrame(Thread) \
209 ((PKTRAP_FRAME)((ULONG_PTR)((Thread)->InitialStack) - \
210 sizeof(KTRAP_FRAME)))
211
212 //
213 // Macro to get context switches from the PRCB
214 // All architectures but x86 have it in the PRCB's KeContextSwitches
215 //
216 #define KeGetContextSwitches(Prcb) \
217 (Prcb->KeContextSwitches)
218
219 //
220 // Macro to get the second level cache size field name which differs between
221 // CISC and RISC architectures, as the former has unified I/D cache
222 //
223 #define KiGetSecondLevelDCacheSize() ((PKIPCR)KeGetPcr())->SecondLevelCacheSize
224
225 #define KeGetExceptionFrame(Thread) \
226 (PKEXCEPTION_FRAME)((ULONG_PTR)KeGetTrapFrame(Thread) - \
227 sizeof(KEXCEPTION_FRAME))
228
229 //
230 // Returns the Interrupt State from a Trap Frame.
231 // ON = TRUE, OFF = FALSE
232 //
233 #define KeGetTrapFrameInterruptState(TrapFrame) \
234 BooleanFlagOn((TrapFrame)->EFlags, EFLAGS_INTERRUPT_MASK)
235
236 /* Diable interrupts and return whether they were enabled before */
237 FORCEINLINE
238 BOOLEAN
KeDisableInterrupts(VOID)239 KeDisableInterrupts(VOID)
240 {
241 ULONG_PTR Flags;
242
243 /* Get EFLAGS and check if the interrupt bit is set */
244 Flags = __readeflags();
245
246 /* Disable interrupts */
247 _disable();
248 return (Flags & EFLAGS_INTERRUPT_MASK) ? TRUE : FALSE;
249 }
250
251 /* Restore previous interrupt state */
252 FORCEINLINE
253 VOID
KeRestoreInterrupts(BOOLEAN WereEnabled)254 KeRestoreInterrupts(BOOLEAN WereEnabled)
255 {
256 if (WereEnabled) _enable();
257 }
258
259 //
260 // Invalidates the TLB entry for a specified address
261 //
262 FORCEINLINE
263 VOID
KeInvalidateTlbEntry(IN PVOID Address)264 KeInvalidateTlbEntry(IN PVOID Address)
265 {
266 /* Invalidate the TLB entry for this address */
267 __invlpg(Address);
268 }
269
270 FORCEINLINE
271 VOID
KeFlushProcessTb(VOID)272 KeFlushProcessTb(VOID)
273 {
274 /* Flush the TLB by resetting CR3 */
275 __writecr3(__readcr3());
276 }
277
278 FORCEINLINE
279 VOID
KeSweepICache(IN PVOID BaseAddress,IN SIZE_T FlushSize)280 KeSweepICache(IN PVOID BaseAddress,
281 IN SIZE_T FlushSize)
282 {
283 //
284 // Always sweep the whole cache
285 //
286 UNREFERENCED_PARAMETER(BaseAddress);
287 UNREFERENCED_PARAMETER(FlushSize);
288 __wbinvd();
289 }
290
291 FORCEINLINE
292 VOID
KiRundownThread(IN PKTHREAD Thread)293 KiRundownThread(IN PKTHREAD Thread)
294 {
295 /* Nothing to do */
296 }
297
298 /* Registers an interrupt handler with an IDT vector */
299 FORCEINLINE
300 VOID
KeRegisterInterruptHandler(IN ULONG Vector,IN PVOID Handler)301 KeRegisterInterruptHandler(IN ULONG Vector,
302 IN PVOID Handler)
303 {
304 UCHAR Entry;
305 PKIDTENTRY64 Idt;
306
307 /* Get the entry from the HAL */
308 Entry = HalVectorToIDTEntry(Vector);
309
310 /* Now set the data */
311 Idt = &KeGetPcr()->IdtBase[Entry];
312 Idt->OffsetLow = (ULONG_PTR)Handler & 0xffff;
313 Idt->OffsetMiddle = ((ULONG_PTR)Handler >> 16) & 0xffff;
314 Idt->OffsetHigh = (ULONG_PTR)Handler >> 32;
315 Idt->Selector = KGDT64_R0_CODE;
316 Idt->IstIndex = 0;
317 Idt->Type = 0x0e;
318 Idt->Dpl = 0;
319 Idt->Present = 1;
320 Idt->Reserved0 = 0;
321 Idt->Reserved1 = 0;
322 }
323
324 /* Returns the registered interrupt handler for a given IDT vector */
325 FORCEINLINE
326 PVOID
KeQueryInterruptHandler(IN ULONG Vector)327 KeQueryInterruptHandler(IN ULONG Vector)
328 {
329 UCHAR Entry;
330 PKIDTENTRY64 Idt;
331
332 /* Get the entry from the HAL */
333 Entry = HalVectorToIDTEntry(Vector);
334
335 /* Get the IDT entry */
336 Idt = &KeGetPcr()->IdtBase[Entry];
337
338 /* Return the address */
339 return (PVOID)((ULONG64)Idt->OffsetHigh << 32 |
340 (ULONG64)Idt->OffsetMiddle << 16 |
341 (ULONG64)Idt->OffsetLow);
342 }
343
344 FORCEINLINE
345 VOID
KiSendEOI(VOID)346 KiSendEOI(VOID)
347 {
348 /* Write 0 to the apic EOI register */
349 *((volatile ULONG*)APIC_EOI_REGISTER) = 0;
350 }
351
352 FORCEINLINE
353 VOID
KiEndInterrupt(IN KIRQL Irql,IN PKTRAP_FRAME TrapFrame)354 KiEndInterrupt(IN KIRQL Irql,
355 IN PKTRAP_FRAME TrapFrame)
356 {
357 /* Make sure this is from the clock handler */
358 ASSERT(TrapFrame->ErrorCode == 0xc10c4);
359 //KeLowerIrql(Irql);
360 }
361
362 FORCEINLINE
363 BOOLEAN
KiUserTrap(IN PKTRAP_FRAME TrapFrame)364 KiUserTrap(IN PKTRAP_FRAME TrapFrame)
365 {
366 /* Anything else but Ring 0 is Ring 3 */
367 return !!(TrapFrame->SegCs & MODE_MASK);
368 }
369
370 //
371 // PERF Code
372 //
373 FORCEINLINE
374 VOID
Ki386PerfEnd(VOID)375 Ki386PerfEnd(VOID)
376 {
377 extern ULONGLONG BootCyclesEnd, BootCycles;
378 BootCyclesEnd = __rdtsc();
379 DbgPrint("Boot took %I64u cycles!\n", BootCyclesEnd - BootCycles);
380 DbgPrint("Interrupts: %u System Calls: %u Context Switches: %u\n",
381 KeGetCurrentPrcb()->InterruptCount,
382 KeGetCurrentPrcb()->KeSystemCalls,
383 KeGetContextSwitches(KeGetCurrentPrcb()));
384 }
385
386 struct _KPCR;
387
388 DECLSPEC_NORETURN VOID KiSwitchToBootStack(IN ULONG_PTR InitialStack);
389 VOID KiDivideErrorFault(VOID);
390 VOID KiDebugTrapOrFault(VOID);
391 VOID KiNmiInterrupt(VOID);
392 VOID KiBreakpointTrap(VOID);
393 VOID KiOverflowTrap(VOID);
394 VOID KiBoundFault(VOID);
395 VOID KiInvalidOpcodeFault(VOID);
396 VOID KiNpxNotAvailableFault(VOID);
397 VOID KiDoubleFaultAbort(VOID);
398 VOID KiNpxSegmentOverrunAbort(VOID);
399 VOID KiInvalidTssFault(VOID);
400 VOID KiSegmentNotPresentFault(VOID);
401 VOID KiStackFault(VOID);
402 VOID KiGeneralProtectionFault(VOID);
403 VOID KiPageFault(VOID);
404 VOID KiFloatingErrorFault(VOID);
405 VOID KiAlignmentFault(VOID);
406 VOID KiMcheckAbort(VOID);
407 VOID KiXmmException(VOID);
408 VOID KiApcInterrupt(VOID);
409 VOID KiRaiseAssertion(VOID);
410 VOID KiDebugServiceTrap(VOID);
411 VOID KiDpcInterrupt(VOID);
412 VOID KiIpiInterrupt(VOID);
413
414 VOID KiGdtPrepareForApplicationProcessorInit(ULONG Id);
415 VOID Ki386InitializeLdt(VOID);
416 VOID Ki386SetProcessorFeatures(VOID);
417 VOID KiGetCacheInformation(VOID);
418 VOID KiSetProcessorType(VOID);
419 ULONG64 KiGetFeatureBits(VOID);
420 VOID KiInitializeCpuFeatures(VOID);
421 #if DBG
422 VOID KiReportCpuFeatures(IN PKPRCB Prcb);
423 #endif
424
425 ULONG KeAllocateGdtSelector(ULONG Desc[2]);
426 VOID KeFreeGdtSelector(ULONG Entry);
427 VOID NtEarlyInitVdm(VOID);
428 VOID KeApplicationProcessorInitDispatcher(VOID);
429 VOID KeCreateApplicationProcessorIdleThread(ULONG Id);
430
431 VOID
432 Ke386InitThreadWithContext(PKTHREAD Thread,
433 PKSYSTEM_ROUTINE SystemRoutine,
434 PKSTART_ROUTINE StartRoutine,
435 PVOID StartContext,
436 PCONTEXT Context);
437 #define KeArchInitThreadWithContext(Thread,SystemRoutine,StartRoutine,StartContext,Context) \
438 Ke386InitThreadWithContext(Thread,SystemRoutine,StartRoutine,StartContext,Context)
439
440 #ifdef _NTOSKRNL_ /* FIXME: Move flags above to NDK instead of here */
441 VOID
442 KiThreadStartup(PKSYSTEM_ROUTINE SystemRoutine,
443 PKSTART_ROUTINE StartRoutine,
444 PVOID StartContext,
445 BOOLEAN UserThread,
446 KTRAP_FRAME TrapFrame);
447 #endif
448
449 #endif /* __ASM__ */
450
451 // HACK
452 extern NTKERNELAPI volatile KSYSTEM_TIME KeTickCount;
453
454 // win64 uses DMA macros, this one is not defined
455 NTHALAPI
456 NTSTATUS
457 NTAPI
458 HalAllocateAdapterChannel(
459 IN PADAPTER_OBJECT AdapterObject,
460 IN PWAIT_CONTEXT_BLOCK Wcb,
461 IN ULONG NumberOfMapRegisters,
462 IN PDRIVER_CONTROL ExecutionRoutine);
463
464 FORCEINLINE
465 PULONG_PTR
KiGetUserModeStackAddress(void)466 KiGetUserModeStackAddress(void)
467 {
468 return &PsGetCurrentThread()->Tcb.TrapFrame->Rsp;
469 }
470
471 VOID
472 KiSetTrapContext(
473 _Out_ PKTRAP_FRAME TrapFrame,
474 _In_ PCONTEXT Context,
475 _In_ KPROCESSOR_MODE RequestorMode);
476
477 // Exits to user mode, only restores the trap frame, zeroes the non-volatile registers
478 DECLSPEC_NORETURN
479 VOID
480 KiUserCallbackExit(
481 _In_ PKTRAP_FRAME TrapFrame);
482
483 DECLSPEC_NORETURN
484 VOID
485 KiExceptionExit(
486 _In_ PKTRAP_FRAME TrapFrame,
487 _In_ PKEXCEPTION_FRAME ExceptionFrame);
488
489 BOOLEAN
490 KiProcessorFreezeHandler(
491 _In_ PKTRAP_FRAME TrapFrame,
492 _In_ PKEXCEPTION_FRAME ExceptionFrame);
493
494 #ifdef __cplusplus
495 } // extern "C"
496 #endif
497
498 #endif /* __NTOSKRNL_INCLUDE_INTERNAL_AMD64_KE_H */
499
500 /* EOF */
501