1 /* 2 * PROJECT: ReactOS Kernel 3 * LICENSE: GPL - See COPYING in the top level directory 4 * FILE: ntoskrnl/ke/i386/kiinit.c 5 * PURPOSE: Kernel Initialization for x86 CPUs 6 * PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org) 7 */ 8 9 /* INCLUDES *****************************************************************/ 10 11 #include <ntoskrnl.h> 12 #define NDEBUG 13 #include <debug.h> 14 #include "internal/i386/trap_x.h" 15 16 /* GLOBALS *******************************************************************/ 17 18 /* Boot and double-fault/NMI/DPC stack */ 19 UCHAR DECLSPEC_ALIGN(PAGE_SIZE) P0BootStackData[KERNEL_STACK_SIZE] = {0}; 20 UCHAR DECLSPEC_ALIGN(PAGE_SIZE) KiDoubleFaultStackData[KERNEL_STACK_SIZE] = {0}; 21 ULONG_PTR P0BootStack = (ULONG_PTR)&P0BootStackData[KERNEL_STACK_SIZE]; 22 ULONG_PTR KiDoubleFaultStack = (ULONG_PTR)&KiDoubleFaultStackData[KERNEL_STACK_SIZE]; 23 24 /* Spinlocks used only on X86 */ 25 KSPIN_LOCK KiFreezeExecutionLock; 26 KSPIN_LOCK Ki486CompatibilityLock; 27 28 /* Perf */ 29 ULONG ProcessCount; 30 ULONGLONG BootCycles, BootCyclesEnd; 31 32 /* FUNCTIONS *****************************************************************/ 33 34 INIT_FUNCTION 35 VOID 36 NTAPI 37 KiInitMachineDependent(VOID) 38 { 39 ULONG CpuCount; 40 BOOLEAN FbCaching = FALSE; 41 NTSTATUS Status; 42 ULONG ReturnLength; 43 ULONG i, Affinity, Sample = 0; 44 PFX_SAVE_AREA FxSaveArea; 45 ULONG MXCsrMask = 0xFFBF; 46 CPU_INFO CpuInfo; 47 KI_SAMPLE_MAP Samples[10]; 48 PKI_SAMPLE_MAP CurrentSample = Samples; 49 LARGE_IDENTITY_MAP IdentityMap; 50 51 /* Check for large page support */ 52 if (KeFeatureBits & KF_LARGE_PAGE) 53 { 54 /* Do an IPI to enable it on all CPUs */ 55 if (Ki386CreateIdentityMap(&IdentityMap, Ki386EnableCurrentLargePage, 2)) 56 KeIpiGenericCall(Ki386EnableTargetLargePage, (ULONG_PTR)&IdentityMap); 57 58 /* Free the pages allocated for identity map */ 59 Ki386FreeIdentityMap(&IdentityMap); 60 } 61 62 /* Check for global page support */ 63 if (KeFeatureBits & KF_GLOBAL_PAGE) 64 { 65 /* Do an IPI to enable it on all CPUs */ 66 CpuCount = KeNumberProcessors; 67 KeIpiGenericCall(Ki386EnableGlobalPage, (ULONG_PTR)&CpuCount); 68 } 69 70 /* Check for PAT and/or MTRR support */ 71 if (KeFeatureBits & (KF_PAT | KF_MTRR)) 72 { 73 /* Query the HAL to make sure we can use it */ 74 Status = HalQuerySystemInformation(HalFrameBufferCachingInformation, 75 sizeof(BOOLEAN), 76 &FbCaching, 77 &ReturnLength); 78 if ((NT_SUCCESS(Status)) && (FbCaching)) 79 { 80 /* We can't, disable it */ 81 KeFeatureBits &= ~(KF_PAT | KF_MTRR); 82 } 83 } 84 85 /* Check for PAT support and enable it */ 86 if (KeFeatureBits & KF_PAT) KiInitializePAT(); 87 88 /* Assume no errata for now */ 89 SharedUserData->ProcessorFeatures[PF_FLOATING_POINT_PRECISION_ERRATA] = 0; 90 91 /* Check if we have an NPX */ 92 if (KeI386NpxPresent) 93 { 94 /* Loop every CPU */ 95 i = KeActiveProcessors; 96 for (Affinity = 1; i; Affinity <<= 1) 97 { 98 /* Check if this is part of the set */ 99 if (i & Affinity) 100 { 101 /* Run on this CPU */ 102 i &= ~Affinity; 103 KeSetSystemAffinityThread(Affinity); 104 105 /* Detect FPU errata */ 106 if (KiIsNpxErrataPresent()) 107 { 108 /* Disable NPX support */ 109 KeI386NpxPresent = FALSE; 110 SharedUserData-> 111 ProcessorFeatures[PF_FLOATING_POINT_PRECISION_ERRATA] = 112 TRUE; 113 break; 114 } 115 } 116 } 117 } 118 119 /* If there's no NPX, then we're emulating the FPU */ 120 SharedUserData->ProcessorFeatures[PF_FLOATING_POINT_EMULATED] = 121 !KeI386NpxPresent; 122 123 /* Check if there's no NPX, so that we can disable associated features */ 124 if (!KeI386NpxPresent) 125 { 126 /* Remove NPX-related bits */ 127 KeFeatureBits &= ~(KF_XMMI64 | KF_XMMI | KF_FXSR | KF_MMX); 128 129 /* Disable kernel flags */ 130 KeI386FxsrPresent = KeI386XMMIPresent = FALSE; 131 132 /* Disable processor features that might've been set until now */ 133 SharedUserData->ProcessorFeatures[PF_FLOATING_POINT_PRECISION_ERRATA] = 134 SharedUserData->ProcessorFeatures[PF_XMMI64_INSTRUCTIONS_AVAILABLE] = 135 SharedUserData->ProcessorFeatures[PF_XMMI_INSTRUCTIONS_AVAILABLE] = 136 SharedUserData->ProcessorFeatures[PF_3DNOW_INSTRUCTIONS_AVAILABLE] = 137 SharedUserData->ProcessorFeatures[PF_MMX_INSTRUCTIONS_AVAILABLE] = 0; 138 } 139 140 /* Check for CR4 support */ 141 if (KeFeatureBits & KF_CR4) 142 { 143 /* Do an IPI call to enable the Debug Exceptions */ 144 CpuCount = KeNumberProcessors; 145 KeIpiGenericCall(Ki386EnableDE, (ULONG_PTR)&CpuCount); 146 } 147 148 /* Check if FXSR was found */ 149 if (KeFeatureBits & KF_FXSR) 150 { 151 /* Do an IPI call to enable the FXSR */ 152 CpuCount = KeNumberProcessors; 153 KeIpiGenericCall(Ki386EnableFxsr, (ULONG_PTR)&CpuCount); 154 155 /* Check if XMM was found too */ 156 if (KeFeatureBits & KF_XMMI) 157 { 158 /* Do an IPI call to enable XMMI exceptions */ 159 CpuCount = KeNumberProcessors; 160 KeIpiGenericCall(Ki386EnableXMMIExceptions, (ULONG_PTR)&CpuCount); 161 162 /* FIXME: Implement and enable XMM Page Zeroing for Mm */ 163 164 /* Patch the RtlPrefetchMemoryNonTemporal routine to enable it */ 165 *(PCHAR)RtlPrefetchMemoryNonTemporal = 0x90; // NOP 166 } 167 } 168 169 /* Check for, and enable SYSENTER support */ 170 KiRestoreFastSyscallReturnState(); 171 172 /* Loop every CPU */ 173 i = KeActiveProcessors; 174 for (Affinity = 1; i; Affinity <<= 1) 175 { 176 /* Check if this is part of the set */ 177 if (i & Affinity) 178 { 179 /* Run on this CPU */ 180 i &= ~Affinity; 181 KeSetSystemAffinityThread(Affinity); 182 183 /* Reset MHz to 0 for this CPU */ 184 KeGetCurrentPrcb()->MHz = 0; 185 186 /* Check if we can use RDTSC */ 187 if (KeFeatureBits & KF_RDTSC) 188 { 189 /* Start sampling loop */ 190 for (;;) 191 { 192 /* Do a dummy CPUID to start the sample */ 193 KiCpuId(&CpuInfo, 0); 194 195 /* Fill out the starting data */ 196 CurrentSample->PerfStart = KeQueryPerformanceCounter(NULL); 197 CurrentSample->TSCStart = __rdtsc(); 198 CurrentSample->PerfFreq.QuadPart = -50000; 199 200 /* Sleep for this sample */ 201 KeDelayExecutionThread(KernelMode, 202 FALSE, 203 &CurrentSample->PerfFreq); 204 205 /* Do another dummy CPUID */ 206 KiCpuId(&CpuInfo, 0); 207 208 /* Fill out the ending data */ 209 CurrentSample->PerfEnd = 210 KeQueryPerformanceCounter(&CurrentSample->PerfFreq); 211 CurrentSample->TSCEnd = __rdtsc(); 212 213 /* Calculate the differences */ 214 CurrentSample->PerfDelta = CurrentSample->PerfEnd.QuadPart - 215 CurrentSample->PerfStart.QuadPart; 216 CurrentSample->TSCDelta = CurrentSample->TSCEnd - 217 CurrentSample->TSCStart; 218 219 /* Compute CPU Speed */ 220 CurrentSample->MHz = (ULONG)((CurrentSample->TSCDelta * 221 CurrentSample-> 222 PerfFreq.QuadPart + 500000) / 223 (CurrentSample->PerfDelta * 224 1000000)); 225 226 /* Check if this isn't the first sample */ 227 if (Sample) 228 { 229 /* Check if we got a good precision within 1MHz */ 230 if ((CurrentSample->MHz == CurrentSample[-1].MHz) || 231 (CurrentSample->MHz == CurrentSample[-1].MHz + 1) || 232 (CurrentSample->MHz == CurrentSample[-1].MHz - 1)) 233 { 234 /* We did, stop sampling */ 235 break; 236 } 237 } 238 239 /* Move on */ 240 CurrentSample++; 241 Sample++; 242 243 if (Sample == RTL_NUMBER_OF(Samples)) 244 { 245 /* No luck. Average the samples and be done */ 246 ULONG TotalMHz = 0; 247 while (Sample--) 248 { 249 TotalMHz += Samples[Sample].MHz; 250 } 251 CurrentSample[-1].MHz = TotalMHz / RTL_NUMBER_OF(Samples); 252 DPRINT1("Sampling CPU frequency failed. Using average of %lu MHz\n", CurrentSample[-1].MHz); 253 break; 254 } 255 } 256 257 /* Save the CPU Speed */ 258 KeGetCurrentPrcb()->MHz = CurrentSample[-1].MHz; 259 } 260 261 /* Check if we have MTRR */ 262 if (KeFeatureBits & KF_MTRR) 263 { 264 /* Then manually initialize MTRR for the CPU */ 265 KiInitializeMTRR(i ? FALSE : TRUE); 266 } 267 268 /* Check if we have AMD MTRR and initialize it for the CPU */ 269 if (KeFeatureBits & KF_AMDK6MTRR) KiAmdK6InitializeMTRR(); 270 271 /* Check if this is a buggy Pentium and apply the fixup if so */ 272 if (KiI386PentiumLockErrataPresent) KiI386PentiumLockErrataFixup(); 273 274 /* Check if the CPU supports FXSR */ 275 if (KeFeatureBits & KF_FXSR) 276 { 277 /* Get the current thread NPX state */ 278 FxSaveArea = KiGetThreadNpxArea(KeGetCurrentThread()); 279 280 /* Clear initial MXCsr mask */ 281 FxSaveArea->U.FxArea.MXCsrMask = 0; 282 283 /* Save the current NPX State */ 284 Ke386SaveFpuState(FxSaveArea); 285 286 /* Check if the current mask doesn't match the reserved bits */ 287 if (FxSaveArea->U.FxArea.MXCsrMask != 0) 288 { 289 /* Then use whatever it's holding */ 290 MXCsrMask = FxSaveArea->U.FxArea.MXCsrMask; 291 } 292 293 /* Check if nobody set the kernel-wide mask */ 294 if (!KiMXCsrMask) 295 { 296 /* Then use the one we calculated above */ 297 KiMXCsrMask = MXCsrMask; 298 } 299 else 300 { 301 /* Was it set to the same value we found now? */ 302 if (KiMXCsrMask != MXCsrMask) 303 { 304 /* No, something is definitely wrong */ 305 KeBugCheckEx(MULTIPROCESSOR_CONFIGURATION_NOT_SUPPORTED, 306 KF_FXSR, 307 KiMXCsrMask, 308 MXCsrMask, 309 0); 310 } 311 } 312 313 /* Now set the kernel mask */ 314 KiMXCsrMask &= MXCsrMask; 315 } 316 } 317 } 318 319 /* Return affinity back to where it was */ 320 KeRevertToUserAffinityThread(); 321 322 /* NT allows limiting the duration of an ISR with a registry key */ 323 if (KiTimeLimitIsrMicroseconds) 324 { 325 /* FIXME: TODO */ 326 DPRINT1("ISR Time Limit not yet supported\n"); 327 } 328 329 /* Set CR0 features based on detected CPU */ 330 KiSetCR0Bits(); 331 } 332 333 INIT_FUNCTION 334 VOID 335 NTAPI 336 KiInitializePcr(IN ULONG ProcessorNumber, 337 IN PKIPCR Pcr, 338 IN PKIDTENTRY Idt, 339 IN PKGDTENTRY Gdt, 340 IN PKTSS Tss, 341 IN PKTHREAD IdleThread, 342 IN PVOID DpcStack) 343 { 344 /* Setup the TIB */ 345 Pcr->NtTib.ExceptionList = EXCEPTION_CHAIN_END; 346 Pcr->NtTib.StackBase = 0; 347 Pcr->NtTib.StackLimit = 0; 348 Pcr->NtTib.Self = NULL; 349 350 /* Set the Current Thread */ 351 Pcr->PrcbData.CurrentThread = IdleThread; 352 353 /* Set pointers to ourselves */ 354 Pcr->SelfPcr = (PKPCR)Pcr; 355 Pcr->Prcb = &Pcr->PrcbData; 356 357 /* Set the PCR Version */ 358 Pcr->MajorVersion = PCR_MAJOR_VERSION; 359 Pcr->MinorVersion = PCR_MINOR_VERSION; 360 361 /* Set the PCRB Version */ 362 Pcr->PrcbData.MajorVersion = 1; 363 Pcr->PrcbData.MinorVersion = 1; 364 365 /* Set the Build Type */ 366 Pcr->PrcbData.BuildType = 0; 367 #ifndef CONFIG_SMP 368 Pcr->PrcbData.BuildType |= PRCB_BUILD_UNIPROCESSOR; 369 #endif 370 #if DBG 371 Pcr->PrcbData.BuildType |= PRCB_BUILD_DEBUG; 372 #endif 373 374 /* Set the Processor Number and current Processor Mask */ 375 Pcr->PrcbData.Number = (UCHAR)ProcessorNumber; 376 Pcr->PrcbData.SetMember = 1 << ProcessorNumber; 377 378 /* Set the PRCB for this Processor */ 379 KiProcessorBlock[ProcessorNumber] = Pcr->Prcb; 380 381 /* Start us out at PASSIVE_LEVEL */ 382 Pcr->Irql = PASSIVE_LEVEL; 383 384 /* Set the GDI, IDT, TSS and DPC Stack */ 385 Pcr->GDT = (PVOID)Gdt; 386 Pcr->IDT = Idt; 387 Pcr->TSS = Tss; 388 Pcr->TssCopy = Tss; 389 Pcr->PrcbData.DpcStack = DpcStack; 390 391 /* Setup the processor set */ 392 Pcr->PrcbData.MultiThreadProcessorSet = Pcr->PrcbData.SetMember; 393 } 394 395 INIT_FUNCTION 396 VOID 397 NTAPI 398 KiInitializeKernel(IN PKPROCESS InitProcess, 399 IN PKTHREAD InitThread, 400 IN PVOID IdleStack, 401 IN PKPRCB Prcb, 402 IN CCHAR Number, 403 IN PLOADER_PARAMETER_BLOCK LoaderBlock) 404 { 405 BOOLEAN NpxPresent; 406 ULONG FeatureBits; 407 ULONG PageDirectory[2]; 408 PVOID DpcStack; 409 ULONG Vendor[3]; 410 KIRQL DummyIrql; 411 412 /* Detect and set the CPU Type */ 413 KiSetProcessorType(); 414 415 /* Check if an FPU is present */ 416 NpxPresent = KiIsNpxPresent(); 417 418 /* Initialize the Power Management Support for this PRCB */ 419 PoInitializePrcb(Prcb); 420 421 /* Bugcheck if this is a 386 CPU */ 422 if (Prcb->CpuType == 3) KeBugCheckEx(UNSUPPORTED_PROCESSOR, 0x386, 0, 0, 0); 423 424 /* Get the processor features for the CPU */ 425 FeatureBits = KiGetFeatureBits(); 426 427 /* Set the default NX policy (opt-in) */ 428 SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTIN; 429 430 /* Check if NPX is always on */ 431 if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSON")) 432 { 433 /* Set it always on */ 434 SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSON; 435 FeatureBits |= KF_NX_ENABLED; 436 } 437 else if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTOUT")) 438 { 439 /* Set it in opt-out mode */ 440 SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTOUT; 441 FeatureBits |= KF_NX_ENABLED; 442 } 443 else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTIN")) || 444 (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE"))) 445 { 446 /* Set the feature bits */ 447 FeatureBits |= KF_NX_ENABLED; 448 } 449 else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSOFF")) || 450 (strstr(KeLoaderBlock->LoadOptions, "EXECUTE"))) 451 { 452 /* Set disabled mode */ 453 SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSOFF; 454 FeatureBits |= KF_NX_DISABLED; 455 } 456 457 /* Save feature bits */ 458 Prcb->FeatureBits = FeatureBits; 459 460 /* Save CPU state */ 461 KiSaveProcessorControlState(&Prcb->ProcessorState); 462 463 /* Get cache line information for this CPU */ 464 KiGetCacheInformation(); 465 466 /* Initialize spinlocks and DPC data */ 467 KiInitSpinLocks(Prcb, Number); 468 469 /* Check if this is the Boot CPU */ 470 if (!Number) 471 { 472 /* Set Node Data */ 473 KeNodeBlock[0] = &KiNode0; 474 Prcb->ParentNode = KeNodeBlock[0]; 475 KeNodeBlock[0]->ProcessorMask = Prcb->SetMember; 476 477 /* Set boot-level flags */ 478 KeI386NpxPresent = NpxPresent; 479 KeI386CpuType = Prcb->CpuType; 480 KeI386CpuStep = Prcb->CpuStep; 481 KeProcessorArchitecture = PROCESSOR_ARCHITECTURE_INTEL; 482 KeProcessorLevel = (USHORT)Prcb->CpuType; 483 if (Prcb->CpuID) KeProcessorRevision = Prcb->CpuStep; 484 KeFeatureBits = FeatureBits; 485 KeI386FxsrPresent = (KeFeatureBits & KF_FXSR) ? TRUE : FALSE; 486 KeI386XMMIPresent = (KeFeatureBits & KF_XMMI) ? TRUE : FALSE; 487 488 /* Detect 8-byte compare exchange support */ 489 if (!(KeFeatureBits & KF_CMPXCHG8B)) 490 { 491 /* Copy the vendor string */ 492 RtlCopyMemory(Vendor, Prcb->VendorString, sizeof(Vendor)); 493 494 /* Bugcheck the system. Windows *requires* this */ 495 KeBugCheckEx(UNSUPPORTED_PROCESSOR, 496 (1 << 24 ) | (Prcb->CpuType << 16) | Prcb->CpuStep, 497 Vendor[0], 498 Vendor[1], 499 Vendor[2]); 500 } 501 502 /* Set the current MP Master KPRCB to the Boot PRCB */ 503 Prcb->MultiThreadSetMaster = Prcb; 504 505 /* Lower to APC_LEVEL */ 506 KeLowerIrql(APC_LEVEL); 507 508 /* Initialize some spinlocks */ 509 KeInitializeSpinLock(&KiFreezeExecutionLock); 510 KeInitializeSpinLock(&Ki486CompatibilityLock); 511 512 /* Initialize portable parts of the OS */ 513 KiInitSystem(); 514 515 /* Initialize the Idle Process and the Process Listhead */ 516 InitializeListHead(&KiProcessListHead); 517 PageDirectory[0] = 0; 518 PageDirectory[1] = 0; 519 KeInitializeProcess(InitProcess, 520 0, 521 0xFFFFFFFF, 522 PageDirectory, 523 FALSE); 524 InitProcess->QuantumReset = MAXCHAR; 525 } 526 else 527 { 528 /* FIXME */ 529 DPRINT1("SMP Boot support not yet present\n"); 530 } 531 532 /* Setup the Idle Thread */ 533 KeInitializeThread(InitProcess, 534 InitThread, 535 NULL, 536 NULL, 537 NULL, 538 NULL, 539 NULL, 540 IdleStack); 541 InitThread->NextProcessor = Number; 542 InitThread->Priority = HIGH_PRIORITY; 543 InitThread->State = Running; 544 InitThread->Affinity = 1 << Number; 545 InitThread->WaitIrql = DISPATCH_LEVEL; 546 InitProcess->ActiveProcessors = 1 << Number; 547 548 /* HACK for MmUpdatePageDir */ 549 ((PETHREAD)InitThread)->ThreadsProcess = (PEPROCESS)InitProcess; 550 551 /* Set basic CPU Features that user mode can read */ 552 SharedUserData->ProcessorFeatures[PF_MMX_INSTRUCTIONS_AVAILABLE] = 553 (KeFeatureBits & KF_MMX) ? TRUE: FALSE; 554 SharedUserData->ProcessorFeatures[PF_COMPARE_EXCHANGE_DOUBLE] = 555 (KeFeatureBits & KF_CMPXCHG8B) ? TRUE: FALSE; 556 SharedUserData->ProcessorFeatures[PF_XMMI_INSTRUCTIONS_AVAILABLE] = 557 ((KeFeatureBits & KF_FXSR) && (KeFeatureBits & KF_XMMI)) ? TRUE: FALSE; 558 SharedUserData->ProcessorFeatures[PF_XMMI64_INSTRUCTIONS_AVAILABLE] = 559 ((KeFeatureBits & KF_FXSR) && (KeFeatureBits & KF_XMMI64)) ? TRUE: FALSE; 560 SharedUserData->ProcessorFeatures[PF_3DNOW_INSTRUCTIONS_AVAILABLE] = 561 (KeFeatureBits & KF_3DNOW) ? TRUE: FALSE; 562 SharedUserData->ProcessorFeatures[PF_RDTSC_INSTRUCTION_AVAILABLE] = 563 (KeFeatureBits & KF_RDTSC) ? TRUE: FALSE; 564 565 /* Set up the thread-related fields in the PRCB */ 566 Prcb->CurrentThread = InitThread; 567 Prcb->NextThread = NULL; 568 Prcb->IdleThread = InitThread; 569 570 /* Initialize the Kernel Executive */ 571 ExpInitializeExecutive(Number, LoaderBlock); 572 573 /* Only do this on the boot CPU */ 574 if (!Number) 575 { 576 /* Calculate the time reciprocal */ 577 KiTimeIncrementReciprocal = 578 KiComputeReciprocal(KeMaximumIncrement, 579 &KiTimeIncrementShiftCount); 580 581 /* Update DPC Values in case they got updated by the executive */ 582 Prcb->MaximumDpcQueueDepth = KiMaximumDpcQueueDepth; 583 Prcb->MinimumDpcRate = KiMinimumDpcRate; 584 Prcb->AdjustDpcThreshold = KiAdjustDpcThreshold; 585 586 /* Allocate the DPC Stack */ 587 DpcStack = MmCreateKernelStack(FALSE, 0); 588 if (!DpcStack) KeBugCheckEx(NO_PAGES_AVAILABLE, 1, 0, 0, 0); 589 Prcb->DpcStack = DpcStack; 590 591 /* Allocate the IOPM save area. */ 592 Ki386IopmSaveArea = ExAllocatePoolWithTag(PagedPool, 593 PAGE_SIZE * 2, 594 ' eK'); 595 if (!Ki386IopmSaveArea) 596 { 597 /* Bugcheck. We need this for V86/VDM support. */ 598 KeBugCheckEx(NO_PAGES_AVAILABLE, 2, PAGE_SIZE * 2, 0, 0); 599 } 600 } 601 602 /* Raise to Dispatch */ 603 KeRaiseIrql(DISPATCH_LEVEL, 604 &DummyIrql); 605 606 /* Set the Idle Priority to 0. This will jump into Phase 1 */ 607 KeSetPriorityThread(InitThread, 0); 608 609 /* If there's no thread scheduled, put this CPU in the Idle summary */ 610 KiAcquirePrcbLock(Prcb); 611 if (!Prcb->NextThread) KiIdleSummary |= 1 << Number; 612 KiReleasePrcbLock(Prcb); 613 614 /* Raise back to HIGH_LEVEL and clear the PRCB for the loader block */ 615 KeRaiseIrql(HIGH_LEVEL, 616 &DummyIrql); 617 LoaderBlock->Prcb = 0; 618 } 619 620 INIT_FUNCTION 621 VOID 622 FASTCALL 623 KiGetMachineBootPointers(IN PKGDTENTRY *Gdt, 624 IN PKIDTENTRY *Idt, 625 IN PKIPCR *Pcr, 626 IN PKTSS *Tss) 627 { 628 KDESCRIPTOR GdtDescriptor, IdtDescriptor; 629 KGDTENTRY TssSelector, PcrSelector; 630 USHORT Tr, Fs; 631 632 /* Get GDT and IDT descriptors */ 633 Ke386GetGlobalDescriptorTable(&GdtDescriptor.Limit); 634 __sidt(&IdtDescriptor.Limit); 635 636 /* Save IDT and GDT */ 637 *Gdt = (PKGDTENTRY)GdtDescriptor.Base; 638 *Idt = (PKIDTENTRY)IdtDescriptor.Base; 639 640 /* Get TSS and FS Selectors */ 641 Tr = Ke386GetTr(); 642 Fs = Ke386GetFs(); 643 644 /* Get PCR Selector, mask it and get its GDT Entry */ 645 PcrSelector = *(PKGDTENTRY)((ULONG_PTR)*Gdt + (Fs & ~RPL_MASK)); 646 647 /* Get the KPCR itself */ 648 *Pcr = (PKIPCR)(ULONG_PTR)(PcrSelector.BaseLow | 649 PcrSelector.HighWord.Bytes.BaseMid << 16 | 650 PcrSelector.HighWord.Bytes.BaseHi << 24); 651 652 /* Get TSS Selector, mask it and get its GDT Entry */ 653 TssSelector = *(PKGDTENTRY)((ULONG_PTR)*Gdt + (Tr & ~RPL_MASK)); 654 655 /* Get the KTSS itself */ 656 *Tss = (PKTSS)(ULONG_PTR)(TssSelector.BaseLow | 657 TssSelector.HighWord.Bytes.BaseMid << 16 | 658 TssSelector.HighWord.Bytes.BaseHi << 24); 659 } 660 661 INIT_FUNCTION 662 VOID 663 NTAPI 664 KiSystemStartupBootStack(VOID) 665 { 666 PKTHREAD Thread; 667 668 /* Initialize the kernel for the current CPU */ 669 KiInitializeKernel(&KiInitialProcess.Pcb, 670 (PKTHREAD)KeLoaderBlock->Thread, 671 (PVOID)(KeLoaderBlock->KernelStack & ~3), 672 (PKPRCB)__readfsdword(KPCR_PRCB), 673 KeNumberProcessors - 1, 674 KeLoaderBlock); 675 676 /* Set the priority of this thread to 0 */ 677 Thread = KeGetCurrentThread(); 678 Thread->Priority = 0; 679 680 /* Force interrupts enabled and lower IRQL back to DISPATCH_LEVEL */ 681 _enable(); 682 KeLowerIrql(DISPATCH_LEVEL); 683 684 /* Set the right wait IRQL */ 685 Thread->WaitIrql = DISPATCH_LEVEL; 686 687 /* Jump into the idle loop */ 688 KiIdleLoop(); 689 } 690 691 static 692 VOID 693 KiMarkPageAsReadOnly( 694 PVOID Address) 695 { 696 PHARDWARE_PTE PointerPte; 697 698 /* Make sure the address is page aligned */ 699 ASSERT(ALIGN_DOWN_POINTER_BY(Address, PAGE_SIZE) == Address); 700 701 /* Get the PTE address */ 702 PointerPte = ((PHARDWARE_PTE)PTE_BASE) + ((ULONG_PTR)Address / PAGE_SIZE); 703 ASSERT(PointerPte->Valid); 704 ASSERT(PointerPte->Write); 705 706 /* Set as read-only */ 707 PointerPte->Write = 0; 708 709 /* Flush the TLB entry */ 710 __invlpg(Address); 711 } 712 713 INIT_FUNCTION 714 VOID 715 NTAPI 716 KiSystemStartup(IN PLOADER_PARAMETER_BLOCK LoaderBlock) 717 { 718 ULONG Cpu; 719 PKTHREAD InitialThread; 720 ULONG InitialStack; 721 PKGDTENTRY Gdt; 722 PKIDTENTRY Idt; 723 KIDTENTRY NmiEntry, DoubleFaultEntry; 724 PKTSS Tss; 725 PKIPCR Pcr; 726 KIRQL DummyIrql; 727 728 /* Boot cycles timestamp */ 729 BootCycles = __rdtsc(); 730 731 /* Save the loader block and get the current CPU */ 732 KeLoaderBlock = LoaderBlock; 733 Cpu = KeNumberProcessors; 734 if (!Cpu) 735 { 736 /* If this is the boot CPU, set FS and the CPU Number*/ 737 Ke386SetFs(KGDT_R0_PCR); 738 __writefsdword(KPCR_PROCESSOR_NUMBER, Cpu); 739 740 /* Set the initial stack and idle thread as well */ 741 LoaderBlock->KernelStack = (ULONG_PTR)P0BootStack; 742 LoaderBlock->Thread = (ULONG_PTR)&KiInitialThread; 743 } 744 745 /* Save the initial thread and stack */ 746 InitialStack = LoaderBlock->KernelStack; 747 InitialThread = (PKTHREAD)LoaderBlock->Thread; 748 749 /* Clean the APC List Head */ 750 InitializeListHead(&InitialThread->ApcState.ApcListHead[KernelMode]); 751 752 /* Initialize the machine type */ 753 KiInitializeMachineType(); 754 755 /* Skip initial setup if this isn't the Boot CPU */ 756 if (Cpu) goto AppCpuInit; 757 758 /* Get GDT, IDT, PCR and TSS pointers */ 759 KiGetMachineBootPointers(&Gdt, &Idt, &Pcr, &Tss); 760 761 /* Setup the TSS descriptors and entries */ 762 Ki386InitializeTss(Tss, Idt, Gdt); 763 764 /* Initialize the PCR */ 765 RtlZeroMemory(Pcr, PAGE_SIZE); 766 KiInitializePcr(Cpu, 767 Pcr, 768 Idt, 769 Gdt, 770 Tss, 771 InitialThread, 772 (PVOID)KiDoubleFaultStack); 773 774 /* Set us as the current process */ 775 InitialThread->ApcState.Process = &KiInitialProcess.Pcb; 776 777 /* Clear DR6/7 to cleanup bootloader debugging */ 778 __writefsdword(KPCR_TEB, 0); 779 __writefsdword(KPCR_DR6, 0); 780 __writefsdword(KPCR_DR7, 0); 781 782 /* Setup the IDT */ 783 KeInitExceptions(); 784 785 /* Load Ring 3 selectors for DS/ES */ 786 Ke386SetDs(KGDT_R3_DATA | RPL_MASK); 787 Ke386SetEs(KGDT_R3_DATA | RPL_MASK); 788 789 /* Save NMI and double fault traps */ 790 RtlCopyMemory(&NmiEntry, &Idt[2], sizeof(KIDTENTRY)); 791 RtlCopyMemory(&DoubleFaultEntry, &Idt[8], sizeof(KIDTENTRY)); 792 793 /* Copy kernel's trap handlers */ 794 RtlCopyMemory(Idt, 795 (PVOID)KiIdtDescriptor.Base, 796 KiIdtDescriptor.Limit + 1); 797 798 /* Restore NMI and double fault */ 799 RtlCopyMemory(&Idt[2], &NmiEntry, sizeof(KIDTENTRY)); 800 RtlCopyMemory(&Idt[8], &DoubleFaultEntry, sizeof(KIDTENTRY)); 801 802 AppCpuInit: 803 /* Loop until we can release the freeze lock */ 804 do 805 { 806 /* Loop until execution can continue */ 807 while (*(volatile PKSPIN_LOCK*)&KiFreezeExecutionLock == (PVOID)1); 808 } while(InterlockedBitTestAndSet((PLONG)&KiFreezeExecutionLock, 0)); 809 810 /* Setup CPU-related fields */ 811 __writefsdword(KPCR_NUMBER, Cpu); 812 __writefsdword(KPCR_SET_MEMBER, 1 << Cpu); 813 __writefsdword(KPCR_SET_MEMBER_COPY, 1 << Cpu); 814 __writefsdword(KPCR_PRCB_SET_MEMBER, 1 << Cpu); 815 816 /* Initialize the Processor with HAL */ 817 HalInitializeProcessor(Cpu, KeLoaderBlock); 818 819 /* Set active processors */ 820 KeActiveProcessors |= __readfsdword(KPCR_SET_MEMBER); 821 KeNumberProcessors++; 822 823 /* Check if this is the boot CPU */ 824 if (!Cpu) 825 { 826 /* Initialize debugging system */ 827 KdInitSystem(0, KeLoaderBlock); 828 829 /* Check for break-in */ 830 if (KdPollBreakIn()) DbgBreakPointWithStatus(DBG_STATUS_CONTROL_C); 831 832 /* Make the lowest page of the boot and double fault stack read-only */ 833 KiMarkPageAsReadOnly(P0BootStackData); 834 KiMarkPageAsReadOnly(KiDoubleFaultStackData); 835 } 836 837 /* Raise to HIGH_LEVEL */ 838 KeRaiseIrql(HIGH_LEVEL, 839 &DummyIrql); 840 841 /* Switch to new kernel stack and start kernel bootstrapping */ 842 KiSwitchToBootStack(InitialStack & ~3); 843 } 844