1 #ifndef __NTOSKRNL_INCLUDE_INTERNAL_AMD64_KE_H 2 #define __NTOSKRNL_INCLUDE_INTERNAL_AMD64_KE_H 3 4 #ifdef __cplusplus 5 extern "C" { 6 #endif 7 8 #define X86_EFLAGS_TF 0x00000100 /* Trap flag */ 9 #define X86_EFLAGS_IF 0x00000200 /* Interrupt Enable flag */ 10 #define X86_EFLAGS_IOPL 0x00003000 /* I/O Privilege Level bits */ 11 #define X86_EFLAGS_NT 0x00004000 /* Nested Task flag */ 12 #define X86_EFLAGS_RF 0x00010000 /* Resume flag */ 13 #define X86_EFLAGS_VM 0x00020000 /* Virtual Mode */ 14 #define X86_EFLAGS_ID 0x00200000 /* CPUID detection flag */ 15 16 #define X86_CR0_PE 0x00000001 /* enable Protected Mode */ 17 #define X86_CR0_NE 0x00000020 /* enable native FPU error reporting */ 18 #define X86_CR0_TS 0x00000008 /* enable exception on FPU instruction for task switch */ 19 #define X86_CR0_EM 0x00000004 /* enable FPU emulation (disable FPU) */ 20 #define X86_CR0_MP 0x00000002 /* enable FPU monitoring */ 21 #define X86_CR0_WP 0x00010000 /* enable Write Protect (copy on write) */ 22 #define X86_CR0_PG 0x80000000 /* enable Paging */ 23 24 #define X86_CR4_PAE 0x00000020 /* enable physical address extensions */ 25 #define X86_CR4_PGE 0x00000080 /* enable global pages */ 26 #define X86_CR4_OSFXSR 0x00000200 /* enable FXSAVE/FXRSTOR instructions */ 27 #define X86_CR4_OSXMMEXCPT 0x00000400 /* enable #XF exception */ 28 29 /* EDX flags */ 30 #define X86_FEATURE_FPU 0x00000001 /* x87 FPU is present */ 31 #define X86_FEATURE_VME 0x00000002 /* Virtual 8086 Extensions are present */ 32 #define X86_FEATURE_DBG 0x00000004 /* Debugging extensions are present */ 33 #define X86_FEATURE_PSE 0x00000008 /* Page Size Extension is present */ 34 #define X86_FEATURE_TSC 0x00000010 /* time stamp counters are present */ 35 #define X86_FEATURE_PAE 0x00000040 /* physical address extension is present */ 36 #define X86_FEATURE_CX8 0x00000100 /* CMPXCHG8B instruction present */ 37 #define X86_FEATURE_SYSCALL 0x00000800 /* SYSCALL/SYSRET support present */ 38 #define X86_FEATURE_MTTR 0x00001000 /* Memory type range registers are present */ 39 #define X86_FEATURE_PGE 0x00002000 /* Page Global Enable */ 40 #define X86_FEATURE_CMOV 0x00008000 /* "Conditional move" instruction supported */ 41 #define X86_FEATURE_PAT 0x00010000 /* Page Attribute Table is supported */ 42 #define X86_FEATURE_DS 0x00200000 /* Debug Store is present */ 43 #define X86_FEATURE_MMX 0x00800000 /* MMX extension present */ 44 #define X86_FEATURE_FXSR 0x01000000 /* FXSAVE/FXRSTOR instructions present */ 45 #define X86_FEATURE_SSE 0x02000000 /* SSE extension present */ 46 #define X86_FEATURE_SSE2 0x04000000 /* SSE2 extension present */ 47 #define X86_FEATURE_HT 0x10000000 /* Hyper-Threading present */ 48 49 /* ECX flags */ 50 #define X86_FEATURE_SSE3 0x00000001 /* SSE3 is supported */ 51 #define X86_FEATURE_MONITOR 0x00000008 /* SSE3 Monitor instructions supported */ 52 #define X86_FEATURE_VMX 0x00000020 /* Virtual Machine eXtensions are available */ 53 #define X86_FEATURE_SSSE3 0x00000200 /* Supplemental SSE3 are available */ 54 #define X86_FEATURE_FMA3 0x00001000 /* Fused multiple-add supported */ 55 #define X86_FEATURE_CX16 0x00002000 /* CMPXCHG16B instruction are available */ 56 #define X86_FEATURE_PCID 0x00020000 /* Process Context IDentifiers are supported */ 57 #define X86_FEATURE_SSE41 0x00080000 /* SSE 4.1 is supported */ 58 #define X86_FEATURE_SSE42 0x00100000 /* SSE 4.2 is supported */ 59 #define X86_FEATURE_POPCNT 0x00800000 /* POPCNT instruction is available */ 60 #define X86_FEATURE_XSAVE 0x04000000 /* XSAVE family are available */ 61 62 /* EDX extended flags */ 63 #define X86_FEATURE_NX 0x00100000 /* NX support present */ 64 65 #define X86_EXT_FEATURE_SSE3 0x00000001 /* SSE3 extension present */ 66 #define X86_EXT_FEATURE_3DNOW 0x40000000 /* 3DNOW! extension present */ 67 68 #define FRAME_EDITED 0xFFF8 69 70 #define X86_MSR_GSBASE 0xC0000101 71 #define X86_MSR_KERNEL_GSBASE 0xC0000102 72 #define X86_MSR_EFER 0xC0000080 73 #define X86_MSR_STAR 0xC0000081 74 #define X86_MSR_LSTAR 0xC0000082 75 #define X86_MSR_CSTAR 0xC0000083 76 #define X86_MSR_SFMASK 0xC0000084 77 78 #define EFER_SCE 0x0001 79 #define EFER_LME 0x0100 80 #define EFER_LMA 0x0400 81 #define EFER_NXE 0x0800 82 #define EFER_SVME 0x1000 83 #define EFER_FFXSR 0x4000 84 85 #define AMD64_TSS 9 86 87 #define APIC_EOI_REGISTER 0xFFFFFFFFFFFE00B0ULL 88 89 #ifndef __ASM__ 90 91 #include "intrin_i.h" 92 93 typedef struct _KIDT_INIT 94 { 95 UCHAR InterruptId; 96 UCHAR Dpl; 97 UCHAR IstIndex; 98 PVOID ServiceRoutine; 99 } KIDT_INIT, *PKIDT_INIT; 100 101 #include <pshpack1.h> 102 typedef struct _KI_INTERRUPT_DISPATCH_ENTRY 103 { 104 UCHAR _Op_nop; 105 UCHAR _Op_push; 106 UCHAR _Vector; 107 UCHAR _Op_jmp; 108 ULONG RelativeAddress; 109 } KI_INTERRUPT_DISPATCH_ENTRY, *PKI_INTERRUPT_DISPATCH_ENTRY; 110 #include <poppack.h> 111 112 extern ULONG KeI386NpxPresent; 113 extern ULONG KeI386XMMIPresent; 114 extern ULONG KeI386FxsrPresent; 115 extern ULONG KeI386CpuType; 116 extern ULONG KeI386CpuStep; 117 118 // 119 // INT3 is 1 byte long 120 // 121 #define KD_BREAKPOINT_TYPE UCHAR 122 #define KD_BREAKPOINT_SIZE sizeof(UCHAR) 123 #define KD_BREAKPOINT_VALUE 0xCC 124 125 // 126 // One-liners for getting and setting special purpose registers in portable code 127 // 128 FORCEINLINE 129 ULONG_PTR 130 KeGetContextPc(PCONTEXT Context) 131 { 132 return Context->Rip; 133 } 134 135 FORCEINLINE 136 VOID 137 KeSetContextPc(PCONTEXT Context, ULONG_PTR ProgramCounter) 138 { 139 Context->Rip = ProgramCounter; 140 } 141 142 FORCEINLINE 143 ULONG_PTR 144 KeGetContextReturnRegister(PCONTEXT Context) 145 { 146 return Context->Rax; 147 } 148 149 FORCEINLINE 150 VOID 151 KeSetContextReturnRegister(PCONTEXT Context, ULONG_PTR ReturnValue) 152 { 153 Context->Rax = ReturnValue; 154 } 155 156 FORCEINLINE 157 ULONG_PTR 158 KeGetContextStackRegister(PCONTEXT Context) 159 { 160 return Context->Rsp; 161 } 162 163 FORCEINLINE 164 ULONG_PTR 165 KeGetContextFrameRegister(PCONTEXT Context) 166 { 167 return Context->Rbp; 168 } 169 170 FORCEINLINE 171 VOID 172 KeSetContextFrameRegister(PCONTEXT Context, ULONG_PTR Frame) 173 { 174 Context->Rbp = Frame; 175 } 176 177 FORCEINLINE 178 ULONG_PTR 179 KeGetTrapFramePc(PKTRAP_FRAME TrapFrame) 180 { 181 return TrapFrame->Rip; 182 } 183 184 FORCEINLINE 185 PKTRAP_FRAME 186 KiGetLinkedTrapFrame(PKTRAP_FRAME TrapFrame) 187 { 188 return (PKTRAP_FRAME)TrapFrame->TrapFrame; 189 } 190 191 FORCEINLINE 192 ULONG_PTR 193 KeGetTrapFrameStackRegister(PKTRAP_FRAME TrapFrame) 194 { 195 return TrapFrame->Rsp; 196 } 197 198 FORCEINLINE 199 ULONG_PTR 200 KeGetTrapFrameFrameRegister(PKTRAP_FRAME TrapFrame) 201 { 202 return TrapFrame->Rbp; 203 } 204 205 // 206 // Macro to get trap and exception frame from a thread stack 207 // 208 #define KeGetTrapFrame(Thread) \ 209 (PKTRAP_FRAME)((ULONG_PTR)((Thread)->InitialStack) - \ 210 sizeof(KTRAP_FRAME)) 211 212 // 213 // Macro to get context switches from the PRCB 214 // All architectures but x86 have it in the PRCB's KeContextSwitches 215 // 216 #define KeGetContextSwitches(Prcb) \ 217 (Prcb->KeContextSwitches) 218 219 // 220 // Macro to get the second level cache size field name which differs between 221 // CISC and RISC architectures, as the former has unified I/D cache 222 // 223 #define KiGetSecondLevelDCacheSize() ((PKIPCR)KeGetPcr())->SecondLevelCacheSize 224 225 #define KeGetExceptionFrame(Thread) \ 226 (PKEXCEPTION_FRAME)((ULONG_PTR)KeGetTrapFrame(Thread) - \ 227 sizeof(KEXCEPTION_FRAME)) 228 229 // 230 // Returns the Interrupt State from a Trap Frame. 231 // ON = TRUE, OFF = FALSE 232 // 233 #define KeGetTrapFrameInterruptState(TrapFrame) \ 234 BooleanFlagOn((TrapFrame)->EFlags, EFLAGS_INTERRUPT_MASK) 235 236 /* Diable interrupts and return whether they were enabled before */ 237 FORCEINLINE 238 BOOLEAN 239 KeDisableInterrupts(VOID) 240 { 241 ULONG_PTR Flags; 242 243 /* Get EFLAGS and check if the interrupt bit is set */ 244 Flags = __readeflags(); 245 246 /* Disable interrupts */ 247 _disable(); 248 return (Flags & EFLAGS_INTERRUPT_MASK) ? TRUE : FALSE; 249 } 250 251 /* Restore previous interrupt state */ 252 FORCEINLINE 253 VOID 254 KeRestoreInterrupts(BOOLEAN WereEnabled) 255 { 256 if (WereEnabled) _enable(); 257 } 258 259 // 260 // Invalidates the TLB entry for a specified address 261 // 262 FORCEINLINE 263 VOID 264 KeInvalidateTlbEntry(IN PVOID Address) 265 { 266 /* Invalidate the TLB entry for this address */ 267 __invlpg(Address); 268 } 269 270 FORCEINLINE 271 VOID 272 KeFlushProcessTb(VOID) 273 { 274 /* Flush the TLB by resetting CR3 */ 275 __writecr3(__readcr3()); 276 } 277 278 FORCEINLINE 279 VOID 280 KeSweepICache(IN PVOID BaseAddress, 281 IN SIZE_T FlushSize) 282 { 283 // 284 // Always sweep the whole cache 285 // 286 UNREFERENCED_PARAMETER(BaseAddress); 287 UNREFERENCED_PARAMETER(FlushSize); 288 __wbinvd(); 289 } 290 291 FORCEINLINE 292 VOID 293 KiRundownThread(IN PKTHREAD Thread) 294 { 295 /* Nothing to do */ 296 } 297 298 /* Registers an interrupt handler with an IDT vector */ 299 FORCEINLINE 300 VOID 301 KeRegisterInterruptHandler(IN ULONG Vector, 302 IN PVOID Handler) 303 { 304 UCHAR Entry; 305 PKIDTENTRY64 Idt; 306 307 /* Get the entry from the HAL */ 308 Entry = HalVectorToIDTEntry(Vector); 309 310 /* Now set the data */ 311 Idt = &KeGetPcr()->IdtBase[Entry]; 312 Idt->OffsetLow = (ULONG_PTR)Handler & 0xffff; 313 Idt->OffsetMiddle = ((ULONG_PTR)Handler >> 16) & 0xffff; 314 Idt->OffsetHigh = (ULONG_PTR)Handler >> 32; 315 Idt->Selector = KGDT64_R0_CODE; 316 Idt->IstIndex = 0; 317 Idt->Type = 0x0e; 318 Idt->Dpl = 0; 319 Idt->Present = 1; 320 Idt->Reserved0 = 0; 321 Idt->Reserved1 = 0; 322 } 323 324 /* Returns the registered interrupt handler for a given IDT vector */ 325 FORCEINLINE 326 PVOID 327 KeQueryInterruptHandler(IN ULONG Vector) 328 { 329 UCHAR Entry; 330 PKIDTENTRY64 Idt; 331 332 /* Get the entry from the HAL */ 333 Entry = HalVectorToIDTEntry(Vector); 334 335 /* Get the IDT entry */ 336 Idt = &KeGetPcr()->IdtBase[Entry]; 337 338 /* Return the address */ 339 return (PVOID)((ULONG64)Idt->OffsetHigh << 32 | 340 (ULONG64)Idt->OffsetMiddle << 16 | 341 (ULONG64)Idt->OffsetLow); 342 } 343 344 FORCEINLINE 345 VOID 346 KiSendEOI(VOID) 347 { 348 /* Write 0 to the apic EOI register */ 349 *((volatile ULONG*)APIC_EOI_REGISTER) = 0; 350 } 351 352 FORCEINLINE 353 VOID 354 KiEndInterrupt(IN KIRQL Irql, 355 IN PKTRAP_FRAME TrapFrame) 356 { 357 /* Make sure this is from the clock handler */ 358 ASSERT(TrapFrame->ErrorCode == 0xc10c4); 359 //KeLowerIrql(Irql); 360 } 361 362 FORCEINLINE 363 BOOLEAN 364 KiUserTrap(IN PKTRAP_FRAME TrapFrame) 365 { 366 /* Anything else but Ring 0 is Ring 3 */ 367 return !!(TrapFrame->SegCs & MODE_MASK); 368 } 369 370 // 371 // PERF Code 372 // 373 FORCEINLINE 374 VOID 375 Ki386PerfEnd(VOID) 376 { 377 extern ULONGLONG BootCyclesEnd, BootCycles; 378 BootCyclesEnd = __rdtsc(); 379 DbgPrint("Boot took %I64u cycles!\n", BootCyclesEnd - BootCycles); 380 DbgPrint("Interrupts: %u System Calls: %u Context Switches: %u\n", 381 KeGetCurrentPrcb()->InterruptCount, 382 KeGetCurrentPrcb()->KeSystemCalls, 383 KeGetContextSwitches(KeGetCurrentPrcb())); 384 } 385 386 struct _KPCR; 387 388 //VOID KiInitializeTss(IN PKTSS Tss, IN UINT64 Stack); 389 390 DECLSPEC_NORETURN VOID KiSwitchToBootStack(IN ULONG_PTR InitialStack); 391 VOID KiDivideErrorFault(VOID); 392 VOID KiDebugTrapOrFault(VOID); 393 VOID KiNmiInterrupt(VOID); 394 VOID KiBreakpointTrap(VOID); 395 VOID KiOverflowTrap(VOID); 396 VOID KiBoundFault(VOID); 397 VOID KiInvalidOpcodeFault(VOID); 398 VOID KiNpxNotAvailableFault(VOID); 399 VOID KiDoubleFaultAbort(VOID); 400 VOID KiNpxSegmentOverrunAbort(VOID); 401 VOID KiInvalidTssFault(VOID); 402 VOID KiSegmentNotPresentFault(VOID); 403 VOID KiStackFault(VOID); 404 VOID KiGeneralProtectionFault(VOID); 405 VOID KiPageFault(VOID); 406 VOID KiFloatingErrorFault(VOID); 407 VOID KiAlignmentFault(VOID); 408 VOID KiMcheckAbort(VOID); 409 VOID KiXmmException(VOID); 410 VOID KiApcInterrupt(VOID); 411 VOID KiRaiseAssertion(VOID); 412 VOID KiDebugServiceTrap(VOID); 413 VOID KiDpcInterrupt(VOID); 414 VOID KiIpiInterrupt(VOID); 415 416 VOID KiGdtPrepareForApplicationProcessorInit(ULONG Id); 417 VOID Ki386InitializeLdt(VOID); 418 VOID Ki386SetProcessorFeatures(VOID); 419 VOID KiGetCacheInformation(VOID); 420 VOID KiSetProcessorType(VOID); 421 ULONG64 KiGetFeatureBits(VOID); 422 VOID KiInitializeCpuFeatures(VOID); 423 #if DBG 424 VOID KiReportCpuFeatures(IN PKPRCB Prcb); 425 #endif 426 427 ULONG KeAllocateGdtSelector(ULONG Desc[2]); 428 VOID KeFreeGdtSelector(ULONG Entry); 429 VOID NtEarlyInitVdm(VOID); 430 VOID KeApplicationProcessorInitDispatcher(VOID); 431 VOID KeCreateApplicationProcessorIdleThread(ULONG Id); 432 433 VOID 434 Ke386InitThreadWithContext(PKTHREAD Thread, 435 PKSYSTEM_ROUTINE SystemRoutine, 436 PKSTART_ROUTINE StartRoutine, 437 PVOID StartContext, 438 PCONTEXT Context); 439 #define KeArchInitThreadWithContext(Thread,SystemRoutine,StartRoutine,StartContext,Context) \ 440 Ke386InitThreadWithContext(Thread,SystemRoutine,StartRoutine,StartContext,Context) 441 442 #ifdef _NTOSKRNL_ /* FIXME: Move flags above to NDK instead of here */ 443 VOID 444 KiThreadStartup(PKSYSTEM_ROUTINE SystemRoutine, 445 PKSTART_ROUTINE StartRoutine, 446 PVOID StartContext, 447 BOOLEAN UserThread, 448 KTRAP_FRAME TrapFrame); 449 #endif 450 451 #endif /* __ASM__ */ 452 453 // HACK 454 extern NTKERNELAPI volatile KSYSTEM_TIME KeTickCount; 455 456 // win64 uses DMA macros, this one is not defined 457 NTHALAPI 458 NTSTATUS 459 NTAPI 460 HalAllocateAdapterChannel( 461 IN PADAPTER_OBJECT AdapterObject, 462 IN PWAIT_CONTEXT_BLOCK Wcb, 463 IN ULONG NumberOfMapRegisters, 464 IN PDRIVER_CONTROL ExecutionRoutine); 465 466 FORCEINLINE 467 PULONG_PTR 468 KiGetUserModeStackAddress(void) 469 { 470 return &PsGetCurrentThread()->Tcb.TrapFrame->Rsp; 471 } 472 473 VOID 474 KiSetTrapContext( 475 _Out_ PKTRAP_FRAME TrapFrame, 476 _In_ PCONTEXT Context, 477 _In_ KPROCESSOR_MODE RequestorMode); 478 479 #ifdef __cplusplus 480 } // extern "C" 481 #endif 482 483 #endif /* __NTOSKRNL_INCLUDE_INTERNAL_AMD64_KE_H */ 484 485 /* EOF */ 486