1 /* 2 * PROJECT: ReactOS Kernel 3 * LICENSE: GPL - See COPYING in the top level directory 4 * FILE: ntoskrnl/ke/amd64/kiinit.c 5 * PURPOSE: Kernel Initialization for x86 CPUs 6 * PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org) 7 * Timo Kreuzer (timo.kreuzer@reactos.org) 8 */ 9 10 /* INCLUDES *****************************************************************/ 11 12 #include <ntoskrnl.h> 13 #define NDEBUG 14 #include <debug.h> 15 16 #define REQUIRED_FEATURE_BITS (KF_RDTSC|KF_CR4|KF_CMPXCHG8B|KF_XMMI|KF_XMMI64| \ 17 KF_LARGE_PAGE|KF_FAST_SYSCALL|KF_GLOBAL_PAGE| \ 18 KF_CMOV|KF_PAT|KF_MMX|KF_FXSR|KF_NX_BIT|KF_MTRR) 19 20 /* GLOBALS *******************************************************************/ 21 22 /* Function pointer for early debug prints */ 23 ULONG (*FrLdrDbgPrint)(const char *Format, ...); 24 25 /* Spinlocks used only on X86 */ 26 KSPIN_LOCK KiFreezeExecutionLock; 27 28 29 KIPCR KiInitialPcr; 30 31 /* Boot and double-fault/NMI/DPC stack */ 32 UCHAR DECLSPEC_ALIGN(16) P0BootStackData[KERNEL_STACK_SIZE] = {0}; 33 UCHAR DECLSPEC_ALIGN(16) KiDoubleFaultStackData[KERNEL_STACK_SIZE] = {0}; 34 ULONG_PTR P0BootStack = (ULONG_PTR)&P0BootStackData[KERNEL_STACK_SIZE]; 35 ULONG_PTR KiDoubleFaultStack = (ULONG_PTR)&KiDoubleFaultStackData[KERNEL_STACK_SIZE]; 36 37 void KiInitializeSegments(); 38 void KiSystemCallEntry64(); 39 void KiSystemCallEntry32(); 40 41 /* FUNCTIONS *****************************************************************/ 42 43 CODE_SEG("INIT") 44 VOID 45 NTAPI 46 KiInitMachineDependent(VOID) 47 { 48 /* Check for large page support */ 49 if (KeFeatureBits & KF_LARGE_PAGE) 50 { 51 /* FIXME: Support this */ 52 DPRINT("Large Page support detected but not yet taken advantage of!\n"); 53 } 54 55 /* Check for global page support */ 56 if (KeFeatureBits & KF_GLOBAL_PAGE) 57 { 58 /* FIXME: Support this */ 59 DPRINT("Global Page support detected but not yet taken advantage of!\n"); 60 } 61 62 /* Check if we have MTRR */ 63 if (KeFeatureBits & KF_MTRR) 64 { 65 /* FIXME: Support this */ 66 DPRINT("MTRR support detected but not yet taken advantage of!\n"); 67 } 68 69 /* Check for PAT and/or MTRR support */ 70 if (KeFeatureBits & KF_PAT) 71 { 72 /* FIXME: Support this */ 73 DPRINT("PAT support detected but not yet taken advantage of!\n"); 74 } 75 76 // /* Allocate the IOPM save area */ 77 // Ki386IopmSaveArea = ExAllocatePoolWithTag(PagedPool, 78 // IOPM_SIZE, 79 // ' eK'); 80 // if (!Ki386IopmSaveArea) 81 // { 82 // /* Bugcheck. We need this for V86/VDM support. */ 83 // KeBugCheckEx(NO_PAGES_AVAILABLE, 2, IOPM_SIZE, 0, 0); 84 // } 85 86 } 87 88 VOID 89 NTAPI 90 KiInitializePcr(IN PKIPCR Pcr, 91 IN ULONG ProcessorNumber, 92 IN PKTHREAD IdleThread, 93 IN PVOID DpcStack) 94 { 95 KDESCRIPTOR GdtDescriptor = {{0},0,0}, IdtDescriptor = {{0},0,0}; 96 PKGDTENTRY64 TssEntry; 97 USHORT Tr = 0; 98 99 /* Zero out the PCR */ 100 RtlZeroMemory(Pcr, sizeof(KIPCR)); 101 102 /* Set pointers to ourselves */ 103 Pcr->Self = (PKPCR)Pcr; 104 Pcr->CurrentPrcb = &Pcr->Prcb; 105 106 /* Set the PCR Version */ 107 Pcr->MajorVersion = PCR_MAJOR_VERSION; 108 Pcr->MinorVersion = PCR_MINOR_VERSION; 109 110 /* Set the PRCB Version */ 111 Pcr->Prcb.MajorVersion = 1; 112 Pcr->Prcb.MinorVersion = 1; 113 114 /* Set the Build Type */ 115 Pcr->Prcb.BuildType = 0; 116 #ifndef CONFIG_SMP 117 Pcr->Prcb.BuildType |= PRCB_BUILD_UNIPROCESSOR; 118 #endif 119 #if DBG 120 Pcr->Prcb.BuildType |= PRCB_BUILD_DEBUG; 121 #endif 122 123 /* Set the Processor Number and current Processor Mask */ 124 Pcr->Prcb.Number = (UCHAR)ProcessorNumber; 125 Pcr->Prcb.SetMember = 1ULL << ProcessorNumber; 126 127 /* Get GDT and IDT descriptors */ 128 __sgdt(&GdtDescriptor.Limit); 129 __sidt(&IdtDescriptor.Limit); 130 Pcr->GdtBase = (PVOID)GdtDescriptor.Base; 131 Pcr->IdtBase = (PKIDTENTRY)IdtDescriptor.Base; 132 133 /* Get TSS Selector */ 134 __str(&Tr); 135 ASSERT(Tr == KGDT64_SYS_TSS); 136 137 /* Get TSS Entry */ 138 TssEntry = KiGetGdtEntry(Pcr->GdtBase, Tr); 139 140 /* Get the KTSS itself */ 141 Pcr->TssBase = KiGetGdtDescriptorBase(TssEntry); 142 143 Pcr->Prcb.RspBase = Pcr->TssBase->Rsp0; // FIXME 144 145 /* Set DPC Stack */ 146 Pcr->Prcb.DpcStack = DpcStack; 147 148 /* Setup the processor set */ 149 Pcr->Prcb.MultiThreadProcessorSet = Pcr->Prcb.SetMember; 150 151 /* Clear DR6/7 to cleanup bootloader debugging */ 152 Pcr->Prcb.ProcessorState.SpecialRegisters.KernelDr6 = 0; 153 Pcr->Prcb.ProcessorState.SpecialRegisters.KernelDr7 = 0; 154 155 /* Set the Current Thread */ 156 Pcr->Prcb.CurrentThread = IdleThread; 157 158 /* Start us out at PASSIVE_LEVEL */ 159 Pcr->Irql = PASSIVE_LEVEL; 160 KeSetCurrentIrql(PASSIVE_LEVEL); 161 } 162 163 VOID 164 NTAPI 165 KiInitializeCpu(PKIPCR Pcr) 166 { 167 ULONG64 Pat; 168 ULONG FeatureBits; 169 170 /* Initialize gs */ 171 KiInitializeSegments(); 172 173 /* Set GS base */ 174 __writemsr(MSR_GS_BASE, (ULONG64)Pcr); 175 __writemsr(MSR_GS_SWAP, (ULONG64)Pcr); 176 177 /* Detect and set the CPU Type */ 178 KiSetProcessorType(); 179 180 /* Get the processor features for this CPU */ 181 FeatureBits = KiGetFeatureBits(); 182 183 /* Check if we support all needed features */ 184 if ((FeatureBits & REQUIRED_FEATURE_BITS) != REQUIRED_FEATURE_BITS) 185 { 186 /* If not, bugcheck system */ 187 FrLdrDbgPrint("CPU doesn't have needed features! Has: 0x%x, required: 0x%x\n", 188 FeatureBits, REQUIRED_FEATURE_BITS); 189 KeBugCheck(0); 190 } 191 192 /* Set DEP to always on */ 193 SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSON; 194 FeatureBits |= KF_NX_ENABLED; 195 196 /* Save feature bits */ 197 Pcr->Prcb.FeatureBits = FeatureBits; 198 199 /* Enable fx save restore support */ 200 __writecr4(__readcr4() | CR4_FXSR); 201 202 /* Enable XMMI exceptions */ 203 __writecr4(__readcr4() | CR4_XMMEXCPT); 204 205 /* Enable Write-Protection */ 206 __writecr0(__readcr0() | CR0_WP); 207 208 /* Disable fpu monitoring */ 209 __writecr0(__readcr0() & ~CR0_MP); 210 211 /* Disable x87 fpu exceptions */ 212 __writecr0(__readcr0() & ~CR0_NE); 213 214 /* LDT is unused */ 215 __lldt(0); 216 217 /* Set the systemcall entry points */ 218 __writemsr(MSR_LSTAR, (ULONG64)KiSystemCallEntry64); 219 __writemsr(MSR_CSTAR, (ULONG64)KiSystemCallEntry32); 220 221 __writemsr(MSR_STAR, ((ULONG64)KGDT64_R0_CODE << 32) | 222 ((ULONG64)(KGDT64_R3_CMCODE|RPL_MASK) << 48)); 223 224 /* Set the flags to be cleared when doing a syscall */ 225 __writemsr(MSR_SYSCALL_MASK, EFLAGS_IF_MASK | EFLAGS_TF | EFLAGS_DF); 226 227 /* Enable syscall instruction and no-execute support */ 228 __writemsr(MSR_EFER, __readmsr(MSR_EFER) | MSR_SCE | MSR_NXE); 229 230 /* Initialize the PAT */ 231 Pat = (PAT_WB << 0) | (PAT_WC << 8) | (PAT_UCM << 16) | (PAT_UC << 24) | 232 (PAT_WB << 32) | (PAT_WC << 40) | (PAT_UCM << 48) | (PAT_UC << 56); 233 __writemsr(MSR_PAT, Pat); 234 } 235 236 VOID 237 FASTCALL 238 KiInitializeTss(IN PKTSS64 Tss, 239 IN UINT64 Stack) 240 { 241 PKGDTENTRY64 TssEntry; 242 243 /* Get pointer to the GDT entry */ 244 TssEntry = KiGetGdtEntry(KeGetPcr()->GdtBase, KGDT64_SYS_TSS); 245 246 /* Initialize the GDT entry */ 247 KiInitGdtEntry(TssEntry, (ULONG64)Tss, sizeof(KTSS64), AMD64_TSS, 0); 248 249 /* Zero out the TSS */ 250 RtlZeroMemory(Tss, sizeof(KTSS64)); 251 252 /* FIXME: I/O Map? */ 253 Tss->IoMapBase = 0x68; 254 255 /* Setup ring 0 stack pointer */ 256 Tss->Rsp0 = Stack; 257 258 /* Setup a stack for Double Fault Traps */ 259 Tss->Ist[1] = (ULONG64)KiDoubleFaultStack; 260 261 /* Setup a stack for CheckAbort Traps */ 262 Tss->Ist[2] = (ULONG64)KiDoubleFaultStack; 263 264 /* Setup a stack for NMI Traps */ 265 Tss->Ist[3] = (ULONG64)KiDoubleFaultStack; 266 267 /* Load the task register */ 268 __ltr(KGDT64_SYS_TSS); 269 } 270 271 CODE_SEG("INIT") 272 VOID 273 NTAPI 274 KiInitializeKernelMachineDependent( 275 IN PKPRCB Prcb, 276 IN PLOADER_PARAMETER_BLOCK LoaderBlock) 277 { 278 /* Set boot-level flags */ 279 KeI386CpuType = Prcb->CpuType; 280 KeI386CpuStep = Prcb->CpuStep; 281 KeProcessorArchitecture = PROCESSOR_ARCHITECTURE_AMD64; 282 KeProcessorLevel = (USHORT)Prcb->CpuType; 283 if (Prcb->CpuID) 284 KeProcessorRevision = Prcb->CpuStep; 285 286 /* Set basic CPU Features that user mode can read */ 287 SharedUserData->ProcessorFeatures[PF_COMPARE_EXCHANGE_DOUBLE] = TRUE; 288 SharedUserData->ProcessorFeatures[PF_RDTSC_INSTRUCTION_AVAILABLE] = TRUE; 289 SharedUserData->ProcessorFeatures[PF_PPC_MOVEMEM_64BIT_OK] = TRUE; 290 SharedUserData->ProcessorFeatures[PF_PAE_ENABLED] = TRUE; // ??? 291 SharedUserData->ProcessorFeatures[PF_NX_ENABLED] = TRUE; 292 SharedUserData->ProcessorFeatures[PF_FASTFAIL_AVAILABLE] = TRUE; 293 SharedUserData->ProcessorFeatures[PF_XSAVE_ENABLED] = TRUE; 294 SharedUserData->ProcessorFeatures[PF_MMX_INSTRUCTIONS_AVAILABLE] = 295 (Prcb->FeatureBits & KF_MMX) ? TRUE: FALSE; 296 SharedUserData->ProcessorFeatures[PF_XMMI_INSTRUCTIONS_AVAILABLE] = 297 ((Prcb->FeatureBits & KF_FXSR) && (Prcb->FeatureBits & KF_XMMI)) ? TRUE: FALSE; 298 SharedUserData->ProcessorFeatures[PF_XMMI64_INSTRUCTIONS_AVAILABLE] = 299 ((Prcb->FeatureBits & KF_FXSR) && (Prcb->FeatureBits & KF_XMMI64)) ? TRUE: FALSE; 300 SharedUserData->ProcessorFeatures[PF_3DNOW_INSTRUCTIONS_AVAILABLE] = 301 (Prcb->FeatureBits & KF_3DNOW) ? TRUE: FALSE; 302 SharedUserData->ProcessorFeatures[PF_SSE3_INSTRUCTIONS_AVAILABLE] = 303 (Prcb->FeatureBits & KF_SSE3) ? TRUE: FALSE; 304 SharedUserData->ProcessorFeatures[PF_COMPARE_EXCHANGE128] = 305 (Prcb->FeatureBits & KF_CMPXCHG16B) ? TRUE: FALSE; 306 307 /* Set the default NX policy (opt-in) */ 308 SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTIN; 309 310 /* Check if NPX is always on */ 311 if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSON")) 312 { 313 /* Set it always on */ 314 SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSON; 315 Prcb->FeatureBits |= KF_NX_ENABLED; 316 } 317 else if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTOUT")) 318 { 319 /* Set it in opt-out mode */ 320 SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTOUT; 321 Prcb->FeatureBits |= KF_NX_ENABLED; 322 } 323 else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTIN")) || 324 (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE"))) 325 { 326 /* Set the feature bits */ 327 Prcb->FeatureBits |= KF_NX_ENABLED; 328 } 329 else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSOFF")) || 330 (strstr(KeLoaderBlock->LoadOptions, "EXECUTE"))) 331 { 332 /* Set disabled mode */ 333 SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSOFF; 334 Prcb->FeatureBits |= KF_NX_DISABLED; 335 } 336 } 337 338 static LDR_DATA_TABLE_ENTRY LdrCoreEntries[3]; 339 340 void 341 KiInitModuleList(IN PLOADER_PARAMETER_BLOCK LoaderBlock) 342 { 343 PLDR_DATA_TABLE_ENTRY LdrEntry; 344 PLIST_ENTRY Entry; 345 ULONG i; 346 347 /* Initialize the list head */ 348 InitializeListHead(&PsLoadedModuleList); 349 350 /* Loop the first 3 entries */ 351 for (Entry = LoaderBlock->LoadOrderListHead.Flink, i = 0; 352 Entry != &LoaderBlock->LoadOrderListHead && i < 3; 353 Entry = Entry->Flink, i++) 354 { 355 /* Get the data table entry */ 356 LdrEntry = CONTAINING_RECORD(Entry, 357 LDR_DATA_TABLE_ENTRY, 358 InLoadOrderLinks); 359 360 /* Copy the entry */ 361 LdrCoreEntries[i] = *LdrEntry; 362 363 /* Insert the copy into the list */ 364 InsertTailList(&PsLoadedModuleList, &LdrCoreEntries[i].InLoadOrderLinks); 365 } 366 } 367 368 CODE_SEG("INIT") 369 DECLSPEC_NORETURN 370 VOID 371 NTAPI 372 KiSystemStartup(IN PLOADER_PARAMETER_BLOCK LoaderBlock) 373 { 374 CCHAR Cpu; 375 PKTHREAD InitialThread; 376 ULONG64 InitialStack; 377 PKIPCR Pcr; 378 379 /* HACK */ 380 FrLdrDbgPrint = LoaderBlock->u.I386.CommonDataArea; 381 //FrLdrDbgPrint("Hello from KiSystemStartup!!!\n"); 382 383 /* Save the loader block */ 384 KeLoaderBlock = LoaderBlock; 385 386 /* Get the current CPU number */ 387 Cpu = KeNumberProcessors++; // FIXME 388 389 /* LoaderBlock initialization for Cpu 0 */ 390 if (Cpu == 0) 391 { 392 /* Set the initial stack, idle thread and process */ 393 LoaderBlock->KernelStack = (ULONG_PTR)P0BootStack; 394 LoaderBlock->Thread = (ULONG_PTR)&KiInitialThread; 395 LoaderBlock->Process = (ULONG_PTR)&KiInitialProcess.Pcb; 396 LoaderBlock->Prcb = (ULONG_PTR)&KiInitialPcr.Prcb; 397 } 398 399 /* Get Pcr from loader block */ 400 Pcr = CONTAINING_RECORD(LoaderBlock->Prcb, KIPCR, Prcb); 401 402 /* Set the PRCB for this Processor */ 403 KiProcessorBlock[Cpu] = &Pcr->Prcb; 404 405 /* Align stack to 16 bytes */ 406 LoaderBlock->KernelStack &= ~(16 - 1); 407 408 /* Save the initial thread and stack */ 409 InitialStack = LoaderBlock->KernelStack; // Checkme 410 InitialThread = (PKTHREAD)LoaderBlock->Thread; 411 412 /* Set us as the current process */ 413 InitialThread->ApcState.Process = (PVOID)LoaderBlock->Process; 414 415 /* Initialize the PCR */ 416 KiInitializePcr(Pcr, Cpu, InitialThread, (PVOID)KiDoubleFaultStack); 417 418 /* Initialize the CPU features */ 419 KiInitializeCpu(Pcr); 420 421 /* Initial setup for the boot CPU */ 422 if (Cpu == 0) 423 { 424 /* Initialize the module list (ntos, hal, kdcom) */ 425 KiInitModuleList(LoaderBlock); 426 427 /* Setup the TSS descriptors and entries */ 428 KiInitializeTss(Pcr->TssBase, InitialStack); 429 430 /* Setup the IDT */ 431 KeInitExceptions(); 432 433 /* Initialize debugging system */ 434 KdInitSystem(0, KeLoaderBlock); 435 436 /* Check for break-in */ 437 if (KdPollBreakIn()) DbgBreakPointWithStatus(DBG_STATUS_CONTROL_C); 438 } 439 440 DPRINT1("Pcr = %p, Gdt = %p, Idt = %p, Tss = %p\n", 441 Pcr, Pcr->GdtBase, Pcr->IdtBase, Pcr->TssBase); 442 443 /* Acquire lock */ 444 while (InterlockedBitTestAndSet64((PLONG64)&KiFreezeExecutionLock, 0)) 445 { 446 /* Loop until lock is free */ 447 while ((*(volatile KSPIN_LOCK*)&KiFreezeExecutionLock) & 1); 448 } 449 450 /* Initialize the Processor with HAL */ 451 HalInitializeProcessor(Cpu, KeLoaderBlock); 452 453 /* Set processor as active */ 454 KeActiveProcessors |= 1ULL << Cpu; 455 456 /* Release lock */ 457 InterlockedAnd64((PLONG64)&KiFreezeExecutionLock, 0); 458 459 /* Raise to HIGH_LEVEL */ 460 KfRaiseIrql(HIGH_LEVEL); 461 462 /* Machine specific kernel initialization */ 463 if (Cpu == 0) KiInitializeKernelMachineDependent(&Pcr->Prcb, LoaderBlock); 464 465 /* Switch to new kernel stack and start kernel bootstrapping */ 466 KiSwitchToBootStack(InitialStack & ~3); 467 } 468 469