1 #ifndef __NTOSKRNL_INCLUDE_INTERNAL_AMD64_KE_H 2 #define __NTOSKRNL_INCLUDE_INTERNAL_AMD64_KE_H 3 4 #define X86_EFLAGS_TF 0x00000100 /* Trap flag */ 5 #define X86_EFLAGS_IF 0x00000200 /* Interrupt Enable flag */ 6 #define X86_EFLAGS_IOPL 0x00003000 /* I/O Privilege Level bits */ 7 #define X86_EFLAGS_NT 0x00004000 /* Nested Task flag */ 8 #define X86_EFLAGS_RF 0x00010000 /* Resume flag */ 9 #define X86_EFLAGS_VM 0x00020000 /* Virtual Mode */ 10 #define X86_EFLAGS_ID 0x00200000 /* CPUID detection flag */ 11 12 #define X86_CR0_PE 0x00000001 /* enable Protected Mode */ 13 #define X86_CR0_NE 0x00000020 /* enable native FPU error reporting */ 14 #define X86_CR0_TS 0x00000008 /* enable exception on FPU instruction for task switch */ 15 #define X86_CR0_EM 0x00000004 /* enable FPU emulation (disable FPU) */ 16 #define X86_CR0_MP 0x00000002 /* enable FPU monitoring */ 17 #define X86_CR0_WP 0x00010000 /* enable Write Protect (copy on write) */ 18 #define X86_CR0_PG 0x80000000 /* enable Paging */ 19 20 #define X86_CR4_PAE 0x00000020 /* enable physical address extensions */ 21 #define X86_CR4_PGE 0x00000080 /* enable global pages */ 22 #define X86_CR4_OSFXSR 0x00000200 /* enable FXSAVE/FXRSTOR instructions */ 23 #define X86_CR4_OSXMMEXCPT 0x00000400 /* enable #XF exception */ 24 25 /* EDX flags */ 26 #define X86_FEATURE_FPU 0x00000001 /* x87 FPU is present */ 27 #define X86_FEATURE_VME 0x00000002 /* Virtual 8086 Extensions are present */ 28 #define X86_FEATURE_DBG 0x00000004 /* Debugging extensions are present */ 29 #define X86_FEATURE_PSE 0x00000008 /* Page Size Extension is present */ 30 #define X86_FEATURE_TSC 0x00000010 /* time stamp counters are present */ 31 #define X86_FEATURE_PAE 0x00000040 /* physical address extension is present */ 32 #define X86_FEATURE_CX8 0x00000100 /* CMPXCHG8B instruction present */ 33 #define X86_FEATURE_SYSCALL 0x00000800 /* SYSCALL/SYSRET support present */ 34 #define X86_FEATURE_MTTR 0x00001000 /* Memory type range registers are present */ 35 #define X86_FEATURE_PGE 0x00002000 /* Page Global Enable */ 36 #define X86_FEATURE_CMOV 0x00008000 /* "Conditional move" instruction supported */ 37 #define X86_FEATURE_PAT 0x00010000 /* Page Attribute Table is supported */ 38 #define X86_FEATURE_DS 0x00200000 /* Debug Store is present */ 39 #define X86_FEATURE_MMX 0x00800000 /* MMX extension present */ 40 #define X86_FEATURE_FXSR 0x01000000 /* FXSAVE/FXRSTOR instructions present */ 41 #define X86_FEATURE_SSE 0x02000000 /* SSE extension present */ 42 #define X86_FEATURE_SSE2 0x04000000 /* SSE2 extension present */ 43 #define X86_FEATURE_HT 0x10000000 /* Hyper-Threading present */ 44 45 /* ECX flags */ 46 #define X86_FEATURE_SSE3 0x00000001 /* SSE3 is supported */ 47 #define X86_FEATURE_MONITOR 0x00000008 /* SSE3 Monitor instructions supported */ 48 #define X86_FEATURE_VMX 0x00000020 /* Virtual Machine eXtensions are available */ 49 #define X86_FEATURE_SSSE3 0x00000200 /* Supplemental SSE3 are available */ 50 #define X86_FEATURE_FMA3 0x00001000 /* Fused multiple-add supported */ 51 #define X86_FEATURE_CX16 0x00002000 /* CMPXCHG16B instruction are available */ 52 #define X86_FEATURE_PCID 0x00020000 /* Process Context IDentifiers are supported */ 53 #define X86_FEATURE_SSE41 0x00080000 /* SSE 4.1 is supported */ 54 #define X86_FEATURE_SSE42 0x00100000 /* SSE 4.2 is supported */ 55 #define X86_FEATURE_POPCNT 0x00800000 /* POPCNT instruction is available */ 56 #define X86_FEATURE_XSAVE 0x04000000 /* XSAVE family are available */ 57 58 /* EDX extended flags */ 59 #define X86_FEATURE_NX 0x00100000 /* NX support present */ 60 61 #define X86_EXT_FEATURE_SSE3 0x00000001 /* SSE3 extension present */ 62 #define X86_EXT_FEATURE_3DNOW 0x40000000 /* 3DNOW! extension present */ 63 64 #define FRAME_EDITED 0xFFF8 65 66 #define X86_MSR_GSBASE 0xC0000101 67 #define X86_MSR_KERNEL_GSBASE 0xC0000102 68 #define X86_MSR_EFER 0xC0000080 69 #define X86_MSR_STAR 0xC0000081 70 #define X86_MSR_LSTAR 0xC0000082 71 #define X86_MSR_CSTAR 0xC0000083 72 #define X86_MSR_SFMASK 0xC0000084 73 74 #define EFER_SCE 0x0001 75 #define EFER_LME 0x0100 76 #define EFER_LMA 0x0400 77 #define EFER_NXE 0x0800 78 #define EFER_SVME 0x1000 79 #define EFER_FFXSR 0x4000 80 81 #define AMD64_TSS 9 82 83 #define APIC_EOI_REGISTER 0xFFFFFFFFFFFE00B0ULL 84 85 #ifndef __ASM__ 86 87 #include "intrin_i.h" 88 89 typedef struct _KIDT_INIT 90 { 91 UCHAR InterruptId; 92 UCHAR Dpl; 93 UCHAR IstIndex; 94 PVOID ServiceRoutine; 95 } KIDT_INIT, *PKIDT_INIT; 96 97 #include <pshpack1.h> 98 typedef struct _KI_INTERRUPT_DISPATCH_ENTRY 99 { 100 UCHAR _Op_nop; 101 UCHAR _Op_push; 102 UCHAR _Vector; 103 UCHAR _Op_jmp; 104 ULONG RelativeAddress; 105 } KI_INTERRUPT_DISPATCH_ENTRY, *PKI_INTERRUPT_DISPATCH_ENTRY; 106 #include <poppack.h> 107 108 extern ULONG KeI386NpxPresent; 109 extern ULONG KeI386XMMIPresent; 110 extern ULONG KeI386FxsrPresent; 111 extern ULONG KeI386CpuType; 112 extern ULONG KeI386CpuStep; 113 114 // 115 // INT3 is 1 byte long 116 // 117 #define KD_BREAKPOINT_TYPE UCHAR 118 #define KD_BREAKPOINT_SIZE sizeof(UCHAR) 119 #define KD_BREAKPOINT_VALUE 0xCC 120 121 // 122 // Macros for getting and setting special purpose registers in portable code 123 // 124 #define KeGetContextPc(Context) \ 125 ((Context)->Rip) 126 127 #define KeSetContextPc(Context, ProgramCounter) \ 128 ((Context)->Rip = (ProgramCounter)) 129 130 #define KeGetTrapFramePc(TrapFrame) \ 131 ((TrapFrame)->Rip) 132 133 #define KiGetLinkedTrapFrame(x) \ 134 (PKTRAP_FRAME)((x)->TrapFrame) 135 136 #define KeGetContextReturnRegister(Context) \ 137 ((Context)->Rax) 138 139 #define KeSetContextReturnRegister(Context, ReturnValue) \ 140 ((Context)->Rax = (ReturnValue)) 141 142 // 143 // Macro to get trap and exception frame from a thread stack 144 // 145 #define KeGetTrapFrame(Thread) \ 146 (PKTRAP_FRAME)((ULONG_PTR)((Thread)->InitialStack) - \ 147 sizeof(KTRAP_FRAME)) 148 149 // 150 // Macro to get context switches from the PRCB 151 // All architectures but x86 have it in the PRCB's KeContextSwitches 152 // 153 #define KeGetContextSwitches(Prcb) \ 154 (Prcb->KeContextSwitches) 155 156 // 157 // Macro to get the second level cache size field name which differs between 158 // CISC and RISC architectures, as the former has unified I/D cache 159 // 160 #define KiGetSecondLevelDCacheSize() ((PKIPCR)KeGetPcr())->SecondLevelCacheSize 161 162 #define KeGetExceptionFrame(Thread) \ 163 (PKEXCEPTION_FRAME)((ULONG_PTR)KeGetTrapFrame(Thread) - \ 164 sizeof(KEXCEPTION_FRAME)) 165 166 // 167 // Returns the Interrupt State from a Trap Frame. 168 // ON = TRUE, OFF = FALSE 169 // 170 #define KeGetTrapFrameInterruptState(TrapFrame) \ 171 BooleanFlagOn((TrapFrame)->EFlags, EFLAGS_INTERRUPT_MASK) 172 173 /* Diable interrupts and return whether they were enabled before */ 174 FORCEINLINE 175 BOOLEAN 176 KeDisableInterrupts(VOID) 177 { 178 ULONG_PTR Flags; 179 180 /* Get EFLAGS and check if the interrupt bit is set */ 181 Flags = __readeflags(); 182 183 /* Disable interrupts */ 184 _disable(); 185 return (Flags & EFLAGS_INTERRUPT_MASK) ? TRUE : FALSE; 186 } 187 188 /* Restore previous interrupt state */ 189 FORCEINLINE 190 VOID 191 KeRestoreInterrupts(BOOLEAN WereEnabled) 192 { 193 if (WereEnabled) _enable(); 194 } 195 196 // 197 // Invalidates the TLB entry for a specified address 198 // 199 FORCEINLINE 200 VOID 201 KeInvalidateTlbEntry(IN PVOID Address) 202 { 203 /* Invalidate the TLB entry for this address */ 204 __invlpg(Address); 205 } 206 207 FORCEINLINE 208 VOID 209 KeFlushProcessTb(VOID) 210 { 211 /* Flush the TLB by resetting CR3 */ 212 __writecr3(__readcr3()); 213 } 214 215 FORCEINLINE 216 VOID 217 KeSweepICache(IN PVOID BaseAddress, 218 IN SIZE_T FlushSize) 219 { 220 // 221 // Always sweep the whole cache 222 // 223 UNREFERENCED_PARAMETER(BaseAddress); 224 UNREFERENCED_PARAMETER(FlushSize); 225 __wbinvd(); 226 } 227 228 FORCEINLINE 229 VOID 230 KiRundownThread(IN PKTHREAD Thread) 231 { 232 #ifndef CONFIG_SMP 233 DbgPrint("KiRundownThread is unimplemented\n"); 234 #else 235 /* Nothing to do */ 236 #endif 237 } 238 239 /* Registers an interrupt handler with an IDT vector */ 240 FORCEINLINE 241 VOID 242 KeRegisterInterruptHandler(IN ULONG Vector, 243 IN PVOID Handler) 244 { 245 UCHAR Entry; 246 PKIDTENTRY64 Idt; 247 248 /* Get the entry from the HAL */ 249 Entry = HalVectorToIDTEntry(Vector); 250 251 /* Now set the data */ 252 Idt = &KeGetPcr()->IdtBase[Entry]; 253 Idt->OffsetLow = (ULONG_PTR)Handler & 0xffff; 254 Idt->OffsetMiddle = ((ULONG_PTR)Handler >> 16) & 0xffff; 255 Idt->OffsetHigh = (ULONG_PTR)Handler >> 32; 256 Idt->Selector = KGDT64_R0_CODE; 257 Idt->IstIndex = 0; 258 Idt->Type = 0x0e; 259 Idt->Dpl = 0; 260 Idt->Present = 1; 261 Idt->Reserved0 = 0; 262 Idt->Reserved1 = 0; 263 } 264 265 /* Returns the registered interrupt handler for a given IDT vector */ 266 FORCEINLINE 267 PVOID 268 KeQueryInterruptHandler(IN ULONG Vector) 269 { 270 UCHAR Entry; 271 PKIDTENTRY64 Idt; 272 273 /* Get the entry from the HAL */ 274 Entry = HalVectorToIDTEntry(Vector); 275 276 /* Get the IDT entry */ 277 Idt = &KeGetPcr()->IdtBase[Entry]; 278 279 /* Return the address */ 280 return (PVOID)((ULONG64)Idt->OffsetHigh << 32 | 281 (ULONG64)Idt->OffsetMiddle << 16 | 282 (ULONG64)Idt->OffsetLow); 283 } 284 285 FORCEINLINE 286 VOID 287 KiSendEOI(VOID) 288 { 289 /* Write 0 to the apic EOI register */ 290 *((volatile ULONG*)APIC_EOI_REGISTER) = 0; 291 } 292 293 FORCEINLINE 294 VOID 295 KiEndInterrupt(IN KIRQL Irql, 296 IN PKTRAP_FRAME TrapFrame) 297 { 298 /* Make sure this is from the clock handler */ 299 ASSERT(TrapFrame->ErrorCode == 0xc10c4); 300 //KeLowerIrql(Irql); 301 } 302 303 FORCEINLINE 304 BOOLEAN 305 KiUserTrap(IN PKTRAP_FRAME TrapFrame) 306 { 307 /* Anything else but Ring 0 is Ring 3 */ 308 return !!(TrapFrame->SegCs & MODE_MASK); 309 } 310 311 #define Ki386PerfEnd() 312 313 struct _KPCR; 314 315 //VOID KiInitializeTss(IN PKTSS Tss, IN UINT64 Stack); 316 317 VOID KiSwitchToBootStack(IN ULONG_PTR InitialStack); 318 VOID KiDivideErrorFault(VOID); 319 VOID KiDebugTrapOrFault(VOID); 320 VOID KiNmiInterrupt(VOID); 321 VOID KiBreakpointTrap(VOID); 322 VOID KiOverflowTrap(VOID); 323 VOID KiBoundFault(VOID); 324 VOID KiInvalidOpcodeFault(VOID); 325 VOID KiNpxNotAvailableFault(VOID); 326 VOID KiDoubleFaultAbort(VOID); 327 VOID KiNpxSegmentOverrunAbort(VOID); 328 VOID KiInvalidTssFault(VOID); 329 VOID KiSegmentNotPresentFault(VOID); 330 VOID KiStackFault(VOID); 331 VOID KiGeneralProtectionFault(VOID); 332 VOID KiPageFault(VOID); 333 VOID KiFloatingErrorFault(VOID); 334 VOID KiAlignmentFault(VOID); 335 VOID KiMcheckAbort(VOID); 336 VOID KiXmmException(VOID); 337 VOID KiApcInterrupt(VOID); 338 VOID KiRaiseAssertion(VOID); 339 VOID KiDebugServiceTrap(VOID); 340 VOID KiDpcInterrupt(VOID); 341 VOID KiIpiInterrupt(VOID); 342 343 VOID KiGdtPrepareForApplicationProcessorInit(ULONG Id); 344 VOID Ki386InitializeLdt(VOID); 345 VOID Ki386SetProcessorFeatures(VOID); 346 VOID KiGetCacheInformation(VOID); 347 VOID KiSetProcessorType(VOID); 348 ULONG KiGetFeatureBits(VOID); 349 VOID KiInitializeCpuFeatures(VOID); 350 351 ULONG KeAllocateGdtSelector(ULONG Desc[2]); 352 VOID KeFreeGdtSelector(ULONG Entry); 353 VOID NtEarlyInitVdm(VOID); 354 VOID KeApplicationProcessorInitDispatcher(VOID); 355 VOID KeCreateApplicationProcessorIdleThread(ULONG Id); 356 357 VOID 358 Ke386InitThreadWithContext(PKTHREAD Thread, 359 PKSYSTEM_ROUTINE SystemRoutine, 360 PKSTART_ROUTINE StartRoutine, 361 PVOID StartContext, 362 PCONTEXT Context); 363 #define KeArchInitThreadWithContext(Thread,SystemRoutine,StartRoutine,StartContext,Context) \ 364 Ke386InitThreadWithContext(Thread,SystemRoutine,StartRoutine,StartContext,Context) 365 366 #ifdef _NTOSKRNL_ /* FIXME: Move flags above to NDK instead of here */ 367 VOID 368 KiThreadStartup(PKSYSTEM_ROUTINE SystemRoutine, 369 PKSTART_ROUTINE StartRoutine, 370 PVOID StartContext, 371 BOOLEAN UserThread, 372 KTRAP_FRAME TrapFrame); 373 #endif 374 375 #endif /* __ASM__ */ 376 377 // HACK 378 extern NTKERNELAPI volatile KSYSTEM_TIME KeTickCount; 379 380 // win64 uses DMA macros, this one is not defined 381 NTHALAPI 382 NTSTATUS 383 NTAPI 384 HalAllocateAdapterChannel( 385 IN PADAPTER_OBJECT AdapterObject, 386 IN PWAIT_CONTEXT_BLOCK Wcb, 387 IN ULONG NumberOfMapRegisters, 388 IN PDRIVER_CONTROL ExecutionRoutine); 389 390 #endif /* __NTOSKRNL_INCLUDE_INTERNAL_AMD64_KE_H */ 391 392 /* EOF */ 393