1 #ifndef __NTOSKRNL_INCLUDE_INTERNAL_AMD64_KE_H 2 #define __NTOSKRNL_INCLUDE_INTERNAL_AMD64_KE_H 3 4 #define X86_EFLAGS_TF 0x00000100 /* Trap flag */ 5 #define X86_EFLAGS_IF 0x00000200 /* Interrupt Enable flag */ 6 #define X86_EFLAGS_IOPL 0x00003000 /* I/O Privilege Level bits */ 7 #define X86_EFLAGS_NT 0x00004000 /* Nested Task flag */ 8 #define X86_EFLAGS_RF 0x00010000 /* Resume flag */ 9 #define X86_EFLAGS_VM 0x00020000 /* Virtual Mode */ 10 #define X86_EFLAGS_ID 0x00200000 /* CPUID detection flag */ 11 12 #define X86_CR0_PE 0x00000001 /* enable Protected Mode */ 13 #define X86_CR0_NE 0x00000020 /* enable native FPU error reporting */ 14 #define X86_CR0_TS 0x00000008 /* enable exception on FPU instruction for task switch */ 15 #define X86_CR0_EM 0x00000004 /* enable FPU emulation (disable FPU) */ 16 #define X86_CR0_MP 0x00000002 /* enable FPU monitoring */ 17 #define X86_CR0_WP 0x00010000 /* enable Write Protect (copy on write) */ 18 #define X86_CR0_PG 0x80000000 /* enable Paging */ 19 20 #define X86_CR4_PAE 0x00000020 /* enable physical address extensions */ 21 #define X86_CR4_PGE 0x00000080 /* enable global pages */ 22 #define X86_CR4_OSFXSR 0x00000200 /* enable FXSAVE/FXRSTOR instructions */ 23 #define X86_CR4_OSXMMEXCPT 0x00000400 /* enable #XF exception */ 24 25 /* EDX flags */ 26 #define X86_FEATURE_FPU 0x00000001 /* x87 FPU is present */ 27 #define X86_FEATURE_VME 0x00000002 /* Virtual 8086 Extensions are present */ 28 #define X86_FEATURE_DBG 0x00000004 /* Debugging extensions are present */ 29 #define X86_FEATURE_PSE 0x00000008 /* Page Size Extension is present */ 30 #define X86_FEATURE_TSC 0x00000010 /* time stamp counters are present */ 31 #define X86_FEATURE_PAE 0x00000040 /* physical address extension is present */ 32 #define X86_FEATURE_CX8 0x00000100 /* CMPXCHG8B instruction present */ 33 #define X86_FEATURE_SYSCALL 0x00000800 /* SYSCALL/SYSRET support present */ 34 #define X86_FEATURE_MTTR 0x00001000 /* Memory type range registers are present */ 35 #define X86_FEATURE_PGE 0x00002000 /* Page Global Enable */ 36 #define X86_FEATURE_CMOV 0x00008000 /* "Conditional move" instruction supported */ 37 #define X86_FEATURE_PAT 0x00010000 /* Page Attribute Table is supported */ 38 #define X86_FEATURE_DS 0x00200000 /* Debug Store is present */ 39 #define X86_FEATURE_MMX 0x00800000 /* MMX extension present */ 40 #define X86_FEATURE_FXSR 0x01000000 /* FXSAVE/FXRSTOR instructions present */ 41 #define X86_FEATURE_SSE 0x02000000 /* SSE extension present */ 42 #define X86_FEATURE_SSE2 0x04000000 /* SSE2 extension present */ 43 #define X86_FEATURE_HT 0x10000000 /* Hyper-Threading present */ 44 45 /* ECX flags */ 46 #define X86_FEATURE_SSE3 0x00000001 /* SSE3 is supported */ 47 #define X86_FEATURE_MONITOR 0x00000008 /* SSE3 Monitor instructions supported */ 48 #define X86_FEATURE_VMX 0x00000020 /* Virtual Machine eXtensions are available */ 49 #define X86_FEATURE_SSSE3 0x00000200 /* Supplemental SSE3 are available */ 50 #define X86_FEATURE_FMA3 0x00001000 /* Fused multiple-add supported */ 51 #define X86_FEATURE_CX16 0x00002000 /* CMPXCHG16B instruction are available */ 52 #define X86_FEATURE_PCID 0x00020000 /* Process Context IDentifiers are supported */ 53 #define X86_FEATURE_SSE41 0x00080000 /* SSE 4.1 is supported */ 54 #define X86_FEATURE_SSE42 0x00100000 /* SSE 4.2 is supported */ 55 #define X86_FEATURE_POPCNT 0x00800000 /* POPCNT instruction is available */ 56 #define X86_FEATURE_XSAVE 0x04000000 /* XSAVE family are available */ 57 58 /* EDX extended flags */ 59 #define X86_FEATURE_NX 0x00100000 /* NX support present */ 60 61 #define X86_EXT_FEATURE_SSE3 0x00000001 /* SSE3 extension present */ 62 #define X86_EXT_FEATURE_3DNOW 0x40000000 /* 3DNOW! extension present */ 63 64 #define FRAME_EDITED 0xFFF8 65 66 #define X86_MSR_GSBASE 0xC0000101 67 #define X86_MSR_KERNEL_GSBASE 0xC0000102 68 #define X86_MSR_EFER 0xC0000080 69 #define X86_MSR_STAR 0xC0000081 70 #define X86_MSR_LSTAR 0xC0000082 71 #define X86_MSR_CSTAR 0xC0000083 72 #define X86_MSR_SFMASK 0xC0000084 73 74 #define EFER_SCE 0x0001 75 #define EFER_LME 0x0100 76 #define EFER_LMA 0x0400 77 #define EFER_NXE 0x0800 78 #define EFER_SVME 0x1000 79 #define EFER_FFXSR 0x4000 80 81 #define AMD64_TSS 9 82 83 #define APIC_EOI_REGISTER 0xFFFFFFFFFFFE00B0ULL 84 85 #ifndef __ASM__ 86 87 #include "intrin_i.h" 88 89 typedef struct _KIDT_INIT 90 { 91 UCHAR InterruptId; 92 UCHAR Dpl; 93 UCHAR IstIndex; 94 PVOID ServiceRoutine; 95 } KIDT_INIT, *PKIDT_INIT; 96 97 #include <pshpack1.h> 98 typedef struct _KI_INTERRUPT_DISPATCH_ENTRY 99 { 100 UCHAR _Op_nop; 101 UCHAR _Op_push; 102 UCHAR _Vector; 103 UCHAR _Op_jmp; 104 ULONG RelativeAddress; 105 } KI_INTERRUPT_DISPATCH_ENTRY, *PKI_INTERRUPT_DISPATCH_ENTRY; 106 #include <poppack.h> 107 108 extern ULONG KeI386NpxPresent; 109 extern ULONG KeI386XMMIPresent; 110 extern ULONG KeI386FxsrPresent; 111 extern ULONG KeI386CpuType; 112 extern ULONG KeI386CpuStep; 113 114 // 115 // INT3 is 1 byte long 116 // 117 #define KD_BREAKPOINT_TYPE UCHAR 118 #define KD_BREAKPOINT_SIZE sizeof(UCHAR) 119 #define KD_BREAKPOINT_VALUE 0xCC 120 121 // 122 // One-liners for getting and setting special purpose registers in portable code 123 // 124 FORCEINLINE 125 ULONG_PTR 126 KeGetContextPc(PCONTEXT Context) 127 { 128 return Context->Rip; 129 } 130 131 FORCEINLINE 132 VOID 133 KeSetContextPc(PCONTEXT Context, ULONG_PTR ProgramCounter) 134 { 135 Context->Rip = ProgramCounter; 136 } 137 138 FORCEINLINE 139 ULONG_PTR 140 KeGetContextReturnRegister(PCONTEXT Context) 141 { 142 return Context->Rax; 143 } 144 145 FORCEINLINE 146 VOID 147 KeSetContextReturnRegister(PCONTEXT Context, ULONG_PTR ReturnValue) 148 { 149 Context->Rax = ReturnValue; 150 } 151 152 FORCEINLINE 153 ULONG_PTR 154 KeGetContextStackRegister(PCONTEXT Context) 155 { 156 return Context->Rsp; 157 } 158 159 FORCEINLINE 160 ULONG_PTR 161 KeGetContextFrameRegister(PCONTEXT Context) 162 { 163 return Context->Rbp; 164 } 165 166 FORCEINLINE 167 VOID 168 KeSetContextFrameRegister(PCONTEXT Context, ULONG_PTR Frame) 169 { 170 Context->Rbp = Frame; 171 } 172 173 FORCEINLINE 174 ULONG_PTR 175 KeGetTrapFramePc(PKTRAP_FRAME TrapFrame) 176 { 177 return TrapFrame->Rip; 178 } 179 180 FORCEINLINE 181 PKTRAP_FRAME 182 KiGetLinkedTrapFrame(PKTRAP_FRAME TrapFrame) 183 { 184 return (PKTRAP_FRAME)TrapFrame->TrapFrame; 185 } 186 187 FORCEINLINE 188 ULONG_PTR 189 KeGetTrapFrameStackRegister(PKTRAP_FRAME TrapFrame) 190 { 191 return TrapFrame->Rsp; 192 } 193 194 FORCEINLINE 195 ULONG_PTR 196 KeGetTrapFrameFrameRegister(PKTRAP_FRAME TrapFrame) 197 { 198 return TrapFrame->Rbp; 199 } 200 201 // 202 // Macro to get trap and exception frame from a thread stack 203 // 204 #define KeGetTrapFrame(Thread) \ 205 (PKTRAP_FRAME)((ULONG_PTR)((Thread)->InitialStack) - \ 206 sizeof(KTRAP_FRAME)) 207 208 // 209 // Macro to get context switches from the PRCB 210 // All architectures but x86 have it in the PRCB's KeContextSwitches 211 // 212 #define KeGetContextSwitches(Prcb) \ 213 (Prcb->KeContextSwitches) 214 215 // 216 // Macro to get the second level cache size field name which differs between 217 // CISC and RISC architectures, as the former has unified I/D cache 218 // 219 #define KiGetSecondLevelDCacheSize() ((PKIPCR)KeGetPcr())->SecondLevelCacheSize 220 221 #define KeGetExceptionFrame(Thread) \ 222 (PKEXCEPTION_FRAME)((ULONG_PTR)KeGetTrapFrame(Thread) - \ 223 sizeof(KEXCEPTION_FRAME)) 224 225 // 226 // Returns the Interrupt State from a Trap Frame. 227 // ON = TRUE, OFF = FALSE 228 // 229 #define KeGetTrapFrameInterruptState(TrapFrame) \ 230 BooleanFlagOn((TrapFrame)->EFlags, EFLAGS_INTERRUPT_MASK) 231 232 /* Diable interrupts and return whether they were enabled before */ 233 FORCEINLINE 234 BOOLEAN 235 KeDisableInterrupts(VOID) 236 { 237 ULONG_PTR Flags; 238 239 /* Get EFLAGS and check if the interrupt bit is set */ 240 Flags = __readeflags(); 241 242 /* Disable interrupts */ 243 _disable(); 244 return (Flags & EFLAGS_INTERRUPT_MASK) ? TRUE : FALSE; 245 } 246 247 /* Restore previous interrupt state */ 248 FORCEINLINE 249 VOID 250 KeRestoreInterrupts(BOOLEAN WereEnabled) 251 { 252 if (WereEnabled) _enable(); 253 } 254 255 // 256 // Invalidates the TLB entry for a specified address 257 // 258 FORCEINLINE 259 VOID 260 KeInvalidateTlbEntry(IN PVOID Address) 261 { 262 /* Invalidate the TLB entry for this address */ 263 __invlpg(Address); 264 } 265 266 FORCEINLINE 267 VOID 268 KeFlushProcessTb(VOID) 269 { 270 /* Flush the TLB by resetting CR3 */ 271 __writecr3(__readcr3()); 272 } 273 274 FORCEINLINE 275 VOID 276 KeSweepICache(IN PVOID BaseAddress, 277 IN SIZE_T FlushSize) 278 { 279 // 280 // Always sweep the whole cache 281 // 282 UNREFERENCED_PARAMETER(BaseAddress); 283 UNREFERENCED_PARAMETER(FlushSize); 284 __wbinvd(); 285 } 286 287 FORCEINLINE 288 VOID 289 KiRundownThread(IN PKTHREAD Thread) 290 { 291 /* Nothing to do */ 292 } 293 294 /* Registers an interrupt handler with an IDT vector */ 295 FORCEINLINE 296 VOID 297 KeRegisterInterruptHandler(IN ULONG Vector, 298 IN PVOID Handler) 299 { 300 UCHAR Entry; 301 PKIDTENTRY64 Idt; 302 303 /* Get the entry from the HAL */ 304 Entry = HalVectorToIDTEntry(Vector); 305 306 /* Now set the data */ 307 Idt = &KeGetPcr()->IdtBase[Entry]; 308 Idt->OffsetLow = (ULONG_PTR)Handler & 0xffff; 309 Idt->OffsetMiddle = ((ULONG_PTR)Handler >> 16) & 0xffff; 310 Idt->OffsetHigh = (ULONG_PTR)Handler >> 32; 311 Idt->Selector = KGDT64_R0_CODE; 312 Idt->IstIndex = 0; 313 Idt->Type = 0x0e; 314 Idt->Dpl = 0; 315 Idt->Present = 1; 316 Idt->Reserved0 = 0; 317 Idt->Reserved1 = 0; 318 } 319 320 /* Returns the registered interrupt handler for a given IDT vector */ 321 FORCEINLINE 322 PVOID 323 KeQueryInterruptHandler(IN ULONG Vector) 324 { 325 UCHAR Entry; 326 PKIDTENTRY64 Idt; 327 328 /* Get the entry from the HAL */ 329 Entry = HalVectorToIDTEntry(Vector); 330 331 /* Get the IDT entry */ 332 Idt = &KeGetPcr()->IdtBase[Entry]; 333 334 /* Return the address */ 335 return (PVOID)((ULONG64)Idt->OffsetHigh << 32 | 336 (ULONG64)Idt->OffsetMiddle << 16 | 337 (ULONG64)Idt->OffsetLow); 338 } 339 340 FORCEINLINE 341 VOID 342 KiSendEOI(VOID) 343 { 344 /* Write 0 to the apic EOI register */ 345 *((volatile ULONG*)APIC_EOI_REGISTER) = 0; 346 } 347 348 FORCEINLINE 349 VOID 350 KiEndInterrupt(IN KIRQL Irql, 351 IN PKTRAP_FRAME TrapFrame) 352 { 353 /* Make sure this is from the clock handler */ 354 ASSERT(TrapFrame->ErrorCode == 0xc10c4); 355 //KeLowerIrql(Irql); 356 } 357 358 FORCEINLINE 359 BOOLEAN 360 KiUserTrap(IN PKTRAP_FRAME TrapFrame) 361 { 362 /* Anything else but Ring 0 is Ring 3 */ 363 return !!(TrapFrame->SegCs & MODE_MASK); 364 } 365 366 #define Ki386PerfEnd() 367 368 struct _KPCR; 369 370 //VOID KiInitializeTss(IN PKTSS Tss, IN UINT64 Stack); 371 372 DECLSPEC_NORETURN VOID KiSwitchToBootStack(IN ULONG_PTR InitialStack); 373 VOID KiDivideErrorFault(VOID); 374 VOID KiDebugTrapOrFault(VOID); 375 VOID KiNmiInterrupt(VOID); 376 VOID KiBreakpointTrap(VOID); 377 VOID KiOverflowTrap(VOID); 378 VOID KiBoundFault(VOID); 379 VOID KiInvalidOpcodeFault(VOID); 380 VOID KiNpxNotAvailableFault(VOID); 381 VOID KiDoubleFaultAbort(VOID); 382 VOID KiNpxSegmentOverrunAbort(VOID); 383 VOID KiInvalidTssFault(VOID); 384 VOID KiSegmentNotPresentFault(VOID); 385 VOID KiStackFault(VOID); 386 VOID KiGeneralProtectionFault(VOID); 387 VOID KiPageFault(VOID); 388 VOID KiFloatingErrorFault(VOID); 389 VOID KiAlignmentFault(VOID); 390 VOID KiMcheckAbort(VOID); 391 VOID KiXmmException(VOID); 392 VOID KiApcInterrupt(VOID); 393 VOID KiRaiseAssertion(VOID); 394 VOID KiDebugServiceTrap(VOID); 395 VOID KiDpcInterrupt(VOID); 396 VOID KiIpiInterrupt(VOID); 397 398 VOID KiGdtPrepareForApplicationProcessorInit(ULONG Id); 399 VOID Ki386InitializeLdt(VOID); 400 VOID Ki386SetProcessorFeatures(VOID); 401 VOID KiGetCacheInformation(VOID); 402 VOID KiSetProcessorType(VOID); 403 ULONG KiGetFeatureBits(VOID); 404 VOID KiInitializeCpuFeatures(VOID); 405 406 ULONG KeAllocateGdtSelector(ULONG Desc[2]); 407 VOID KeFreeGdtSelector(ULONG Entry); 408 VOID NtEarlyInitVdm(VOID); 409 VOID KeApplicationProcessorInitDispatcher(VOID); 410 VOID KeCreateApplicationProcessorIdleThread(ULONG Id); 411 412 VOID 413 Ke386InitThreadWithContext(PKTHREAD Thread, 414 PKSYSTEM_ROUTINE SystemRoutine, 415 PKSTART_ROUTINE StartRoutine, 416 PVOID StartContext, 417 PCONTEXT Context); 418 #define KeArchInitThreadWithContext(Thread,SystemRoutine,StartRoutine,StartContext,Context) \ 419 Ke386InitThreadWithContext(Thread,SystemRoutine,StartRoutine,StartContext,Context) 420 421 #ifdef _NTOSKRNL_ /* FIXME: Move flags above to NDK instead of here */ 422 VOID 423 KiThreadStartup(PKSYSTEM_ROUTINE SystemRoutine, 424 PKSTART_ROUTINE StartRoutine, 425 PVOID StartContext, 426 BOOLEAN UserThread, 427 KTRAP_FRAME TrapFrame); 428 #endif 429 430 #endif /* __ASM__ */ 431 432 // HACK 433 extern NTKERNELAPI volatile KSYSTEM_TIME KeTickCount; 434 435 // win64 uses DMA macros, this one is not defined 436 NTHALAPI 437 NTSTATUS 438 NTAPI 439 HalAllocateAdapterChannel( 440 IN PADAPTER_OBJECT AdapterObject, 441 IN PWAIT_CONTEXT_BLOCK Wcb, 442 IN ULONG NumberOfMapRegisters, 443 IN PDRIVER_CONTROL ExecutionRoutine); 444 445 FORCEINLINE 446 PULONG_PTR 447 KiGetUserModeStackAddress(void) 448 { 449 return &PsGetCurrentThread()->Tcb.TrapFrame->Rsp; 450 } 451 452 VOID 453 KiSetTrapContext( 454 _Out_ PKTRAP_FRAME TrapFrame, 455 _In_ PCONTEXT Context, 456 _In_ KPROCESSOR_MODE RequestorMode); 457 458 #endif /* __NTOSKRNL_INCLUDE_INTERNAL_AMD64_KE_H */ 459 460 /* EOF */ 461