1 /*
2 * PROJECT: ReactOS Kernel
3 * LICENSE: GPL - See COPYING in the top level directory
4 * FILE: ntoskrnl/ke/i386/kiinit.c
5 * PURPOSE: Kernel Initialization for x86 CPUs
6 * PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org)
7 */
8
9 /* INCLUDES *****************************************************************/
10
11 #include <ntoskrnl.h>
12 #define NDEBUG
13 #include <debug.h>
14 #include "internal/i386/trap_x.h"
15
16 /* GLOBALS *******************************************************************/
17
18 /* Boot and double-fault/NMI/DPC stack */
19 UCHAR DECLSPEC_ALIGN(PAGE_SIZE) P0BootStackData[KERNEL_STACK_SIZE] = {0};
20 UCHAR DECLSPEC_ALIGN(PAGE_SIZE) KiDoubleFaultStackData[KERNEL_STACK_SIZE] = {0};
21 ULONG_PTR P0BootStack = (ULONG_PTR)&P0BootStackData[KERNEL_STACK_SIZE];
22 ULONG_PTR KiDoubleFaultStack = (ULONG_PTR)&KiDoubleFaultStackData[KERNEL_STACK_SIZE];
23
24 /* Spinlocks used only on X86 */
25 KSPIN_LOCK KiFreezeExecutionLock;
26 KSPIN_LOCK Ki486CompatibilityLock;
27
28 /* Perf */
29 ULONG ProcessCount;
30 ULONGLONG BootCycles, BootCyclesEnd;
31
32 /* FUNCTIONS *****************************************************************/
33
34 CODE_SEG("INIT")
35 VOID
36 NTAPI
KiInitMachineDependent(VOID)37 KiInitMachineDependent(VOID)
38 {
39 ULONG CpuCount;
40 BOOLEAN FbCaching = FALSE;
41 NTSTATUS Status;
42 ULONG ReturnLength;
43 ULONG i, Affinity, Sample = 0;
44 PFX_SAVE_AREA FxSaveArea;
45 ULONG MXCsrMask = 0xFFBF;
46 CPU_INFO CpuInfo;
47 KI_SAMPLE_MAP Samples[10];
48 PKI_SAMPLE_MAP CurrentSample = Samples;
49 LARGE_IDENTITY_MAP IdentityMap;
50
51 /* Check for large page support */
52 if (KeFeatureBits & KF_LARGE_PAGE)
53 {
54 /* Do an IPI to enable it on all CPUs */
55 if (Ki386CreateIdentityMap(&IdentityMap, Ki386EnableCurrentLargePage, 2))
56 KeIpiGenericCall(Ki386EnableTargetLargePage, (ULONG_PTR)&IdentityMap);
57
58 /* Free the pages allocated for identity map */
59 Ki386FreeIdentityMap(&IdentityMap);
60 }
61
62 /* Check for global page support */
63 if (KeFeatureBits & KF_GLOBAL_PAGE)
64 {
65 /* Do an IPI to enable it on all CPUs */
66 CpuCount = KeNumberProcessors;
67 KeIpiGenericCall(Ki386EnableGlobalPage, (ULONG_PTR)&CpuCount);
68 }
69
70 /* Check for PAT and/or MTRR support */
71 if (KeFeatureBits & (KF_PAT | KF_MTRR))
72 {
73 /* Query the HAL to make sure we can use it */
74 Status = HalQuerySystemInformation(HalFrameBufferCachingInformation,
75 sizeof(BOOLEAN),
76 &FbCaching,
77 &ReturnLength);
78 if ((NT_SUCCESS(Status)) && (FbCaching))
79 {
80 /* We can't, disable it */
81 KeFeatureBits &= ~(KF_PAT | KF_MTRR);
82 }
83 }
84
85 /* Check for PAT support and enable it */
86 if (KeFeatureBits & KF_PAT) KiInitializePAT();
87
88 /* Check for CR4 support */
89 if (KeFeatureBits & KF_CR4)
90 {
91 /* Do an IPI call to enable the Debug Exceptions */
92 CpuCount = KeNumberProcessors;
93 KeIpiGenericCall(Ki386EnableDE, (ULONG_PTR)&CpuCount);
94 }
95
96 /* Check if FXSR was found */
97 if (KeFeatureBits & KF_FXSR)
98 {
99 /* Do an IPI call to enable the FXSR */
100 CpuCount = KeNumberProcessors;
101 KeIpiGenericCall(Ki386EnableFxsr, (ULONG_PTR)&CpuCount);
102
103 /* Check if XMM was found too */
104 if (KeFeatureBits & KF_XMMI)
105 {
106 /* Do an IPI call to enable XMMI exceptions */
107 CpuCount = KeNumberProcessors;
108 KeIpiGenericCall(Ki386EnableXMMIExceptions, (ULONG_PTR)&CpuCount);
109
110 /* FIXME: Implement and enable XMM Page Zeroing for Mm */
111
112 /* Patch the RtlPrefetchMemoryNonTemporal routine to enable it */
113 *(PCHAR)RtlPrefetchMemoryNonTemporal = 0x90; // NOP
114 }
115 }
116
117 /* Check for, and enable SYSENTER support */
118 KiRestoreFastSyscallReturnState();
119
120 /* Loop every CPU */
121 i = KeActiveProcessors;
122 for (Affinity = 1; i; Affinity <<= 1)
123 {
124 /* Check if this is part of the set */
125 if (i & Affinity)
126 {
127 /* Run on this CPU */
128 i &= ~Affinity;
129 KeSetSystemAffinityThread(Affinity);
130
131 /* Reset MHz to 0 for this CPU */
132 KeGetCurrentPrcb()->MHz = 0;
133
134 /* Check if we can use RDTSC */
135 if (KeFeatureBits & KF_RDTSC)
136 {
137 /* Start sampling loop */
138 for (;;)
139 {
140 /* Do a dummy CPUID to start the sample */
141 KiCpuId(&CpuInfo, 0);
142
143 /* Fill out the starting data */
144 CurrentSample->PerfStart = KeQueryPerformanceCounter(NULL);
145 CurrentSample->TSCStart = __rdtsc();
146 CurrentSample->PerfFreq.QuadPart = -50000;
147
148 /* Sleep for this sample */
149 KeDelayExecutionThread(KernelMode,
150 FALSE,
151 &CurrentSample->PerfFreq);
152
153 /* Do another dummy CPUID */
154 KiCpuId(&CpuInfo, 0);
155
156 /* Fill out the ending data */
157 CurrentSample->PerfEnd =
158 KeQueryPerformanceCounter(&CurrentSample->PerfFreq);
159 CurrentSample->TSCEnd = __rdtsc();
160
161 /* Calculate the differences */
162 CurrentSample->PerfDelta = CurrentSample->PerfEnd.QuadPart -
163 CurrentSample->PerfStart.QuadPart;
164 CurrentSample->TSCDelta = CurrentSample->TSCEnd -
165 CurrentSample->TSCStart;
166
167 /* Compute CPU Speed */
168 CurrentSample->MHz = (ULONG)((CurrentSample->TSCDelta *
169 CurrentSample->
170 PerfFreq.QuadPart + 500000) /
171 (CurrentSample->PerfDelta *
172 1000000));
173
174 /* Check if this isn't the first sample */
175 if (Sample)
176 {
177 /* Check if we got a good precision within 1MHz */
178 if ((CurrentSample->MHz == CurrentSample[-1].MHz) ||
179 (CurrentSample->MHz == CurrentSample[-1].MHz + 1) ||
180 (CurrentSample->MHz == CurrentSample[-1].MHz - 1))
181 {
182 /* We did, stop sampling */
183 break;
184 }
185 }
186
187 /* Move on */
188 CurrentSample++;
189 Sample++;
190
191 if (Sample == RTL_NUMBER_OF(Samples))
192 {
193 /* No luck. Average the samples and be done */
194 ULONG TotalMHz = 0;
195 while (Sample--)
196 {
197 TotalMHz += Samples[Sample].MHz;
198 }
199 CurrentSample[-1].MHz = TotalMHz / RTL_NUMBER_OF(Samples);
200 DPRINT1("Sampling CPU frequency failed. Using average of %lu MHz\n", CurrentSample[-1].MHz);
201 break;
202 }
203 }
204
205 /* Save the CPU Speed */
206 KeGetCurrentPrcb()->MHz = CurrentSample[-1].MHz;
207 }
208
209 /* Check if we have MTRR */
210 if (KeFeatureBits & KF_MTRR)
211 {
212 /* Then manually initialize MTRR for the CPU */
213 KiInitializeMTRR(i ? FALSE : TRUE);
214 }
215
216 /* Check if we have AMD MTRR and initialize it for the CPU */
217 if (KeFeatureBits & KF_AMDK6MTRR) KiAmdK6InitializeMTRR();
218
219 /* Check if this is a buggy Pentium and apply the fixup if so */
220 if (KiI386PentiumLockErrataPresent) KiI386PentiumLockErrataFixup();
221
222 /* Check if the CPU supports FXSR */
223 if (KeFeatureBits & KF_FXSR)
224 {
225 /* Get the current thread NPX state */
226 FxSaveArea = KiGetThreadNpxArea(KeGetCurrentThread());
227
228 /* Clear initial MXCsr mask */
229 FxSaveArea->U.FxArea.MXCsrMask = 0;
230
231 /* Save the current NPX State */
232 Ke386SaveFpuState(FxSaveArea);
233
234 /* Check if the current mask doesn't match the reserved bits */
235 if (FxSaveArea->U.FxArea.MXCsrMask != 0)
236 {
237 /* Then use whatever it's holding */
238 MXCsrMask = FxSaveArea->U.FxArea.MXCsrMask;
239 }
240
241 /* Check if nobody set the kernel-wide mask */
242 if (!KiMXCsrMask)
243 {
244 /* Then use the one we calculated above */
245 KiMXCsrMask = MXCsrMask;
246 }
247 else
248 {
249 /* Was it set to the same value we found now? */
250 if (KiMXCsrMask != MXCsrMask)
251 {
252 /* No, something is definitely wrong */
253 KeBugCheckEx(MULTIPROCESSOR_CONFIGURATION_NOT_SUPPORTED,
254 KF_FXSR,
255 KiMXCsrMask,
256 MXCsrMask,
257 0);
258 }
259 }
260
261 /* Now set the kernel mask */
262 KiMXCsrMask &= MXCsrMask;
263 }
264 }
265 }
266
267 /* Return affinity back to where it was */
268 KeRevertToUserAffinityThread();
269
270 /* NT allows limiting the duration of an ISR with a registry key */
271 if (KiTimeLimitIsrMicroseconds)
272 {
273 /* FIXME: TODO */
274 DPRINT1("ISR Time Limit not yet supported\n");
275 }
276
277 /* Set CR0 features based on detected CPU */
278 KiSetCR0Bits();
279 }
280
281 CODE_SEG("INIT")
282 VOID
283 NTAPI
KiInitializePcr(IN ULONG ProcessorNumber,IN PKIPCR Pcr,IN PKIDTENTRY Idt,IN PKGDTENTRY Gdt,IN PKTSS Tss,IN PKTHREAD IdleThread,IN PVOID DpcStack)284 KiInitializePcr(IN ULONG ProcessorNumber,
285 IN PKIPCR Pcr,
286 IN PKIDTENTRY Idt,
287 IN PKGDTENTRY Gdt,
288 IN PKTSS Tss,
289 IN PKTHREAD IdleThread,
290 IN PVOID DpcStack)
291 {
292 /* Setup the TIB */
293 Pcr->NtTib.ExceptionList = EXCEPTION_CHAIN_END;
294 Pcr->NtTib.StackBase = 0;
295 Pcr->NtTib.StackLimit = 0;
296 Pcr->NtTib.Self = NULL;
297
298 /* Set the Current Thread */
299 Pcr->PrcbData.CurrentThread = IdleThread;
300
301 /* Set pointers to ourselves */
302 Pcr->SelfPcr = (PKPCR)Pcr;
303 Pcr->Prcb = &Pcr->PrcbData;
304
305 /* Set the PCR Version */
306 Pcr->MajorVersion = PCR_MAJOR_VERSION;
307 Pcr->MinorVersion = PCR_MINOR_VERSION;
308
309 /* Set the PCRB Version */
310 Pcr->PrcbData.MajorVersion = PRCB_MAJOR_VERSION;
311 Pcr->PrcbData.MinorVersion = PRCB_MINOR_VERSION;
312
313 /* Set the Build Type */
314 Pcr->PrcbData.BuildType = 0;
315 #ifndef CONFIG_SMP
316 Pcr->PrcbData.BuildType |= PRCB_BUILD_UNIPROCESSOR;
317 #endif
318 #if DBG
319 Pcr->PrcbData.BuildType |= PRCB_BUILD_DEBUG;
320 #endif
321
322 /* Set the Processor Number and current Processor Mask */
323 Pcr->PrcbData.Number = (UCHAR)ProcessorNumber;
324 Pcr->PrcbData.SetMember = 1 << ProcessorNumber;
325
326 /* Set the PRCB for this Processor */
327 KiProcessorBlock[ProcessorNumber] = Pcr->Prcb;
328
329 /* Start us out at PASSIVE_LEVEL */
330 Pcr->Irql = PASSIVE_LEVEL;
331
332 /* Set the GDI, IDT, TSS and DPC Stack */
333 Pcr->GDT = (PVOID)Gdt;
334 Pcr->IDT = Idt;
335 Pcr->TSS = Tss;
336 Pcr->TssCopy = Tss;
337 Pcr->PrcbData.DpcStack = DpcStack;
338
339 /* Setup the processor set */
340 Pcr->PrcbData.MultiThreadProcessorSet = Pcr->PrcbData.SetMember;
341 }
342
343 static
344 CODE_SEG("INIT")
345 VOID
KiVerifyCpuFeatures(PKPRCB Prcb)346 KiVerifyCpuFeatures(PKPRCB Prcb)
347 {
348 CPU_INFO CpuInfo;
349
350 // 1. Check CPUID support
351 ULONG EFlags = __readeflags();
352
353 /* XOR out the ID bit and update EFlags */
354 ULONG NewEFlags = EFlags ^ EFLAGS_ID;
355 __writeeflags(NewEFlags);
356
357 /* Get them back and see if they were modified */
358 NewEFlags = __readeflags();
359
360 if (NewEFlags == EFlags)
361 {
362 /* The modification did not work, so CPUID is not supported. */
363 KeBugCheckEx(UNSUPPORTED_PROCESSOR, 0x1, 0, 0, 0);
364 }
365 else
366 {
367 /* CPUID is supported. Set the ID Bit again. */
368 EFlags |= EFLAGS_ID;
369 __writeeflags(EFlags);
370 }
371
372 /* Peform CPUID 0 to see if CPUID 1 is supported */
373 KiCpuId(&CpuInfo, 0);
374 if (CpuInfo.Eax == 0)
375 {
376 // 0x1 - missing CPUID instruction
377 KeBugCheckEx(UNSUPPORTED_PROCESSOR, 0x1, 0, 0, 0);
378 }
379
380 // 2. Detect and set the CPU Type
381 KiSetProcessorType();
382
383 if (Prcb->CpuType == 3)
384 KeBugCheckEx(UNSUPPORTED_PROCESSOR, 0x386, 0, 0, 0);
385
386 // 3. Finally, obtain CPU features.
387 ULONG FeatureBits = KiGetFeatureBits();
388
389 // 4. Verify it supports everything we need.
390 if (!(FeatureBits & KF_RDTSC))
391 {
392 // 0x2 - missing CPUID features
393 // second paramenter - edx flag which is missing
394 KeBugCheckEx(UNSUPPORTED_PROCESSOR, 0x2, 0x00000010, 0, 0);
395 }
396
397 if (!(FeatureBits & KF_CMPXCHG8B))
398 {
399 KeBugCheckEx(UNSUPPORTED_PROCESSOR, 0x2, 0x00000100, 0, 0);
400 }
401
402 // Check x87 FPU is present. FIXME: put this into FeatureBits?
403 KiCpuId(&CpuInfo, 1);
404
405 if (!(CpuInfo.Edx & 0x00000001))
406 {
407 KeBugCheckEx(UNSUPPORTED_PROCESSOR, 0x2, 0x00000001, 0, 0);
408 }
409
410 // Set up FPU-related CR0 flags.
411 ULONG Cr0 = __readcr0();
412 // Disable emulation and monitoring.
413 Cr0 &= ~(CR0_EM | CR0_MP);
414 // Enable FPU exceptions.
415 Cr0 |= CR0_NE;
416
417 __writecr0(Cr0);
418
419 // Check for Pentium FPU bug.
420 if (KiIsNpxErrataPresent())
421 {
422 KeBugCheckEx(UNSUPPORTED_PROCESSOR, 0x2, 0x00000001, 0, 0);
423 }
424
425 // 5. Save feature bits.
426 Prcb->FeatureBits = FeatureBits;
427 }
428
429 CODE_SEG("INIT")
430 VOID
431 NTAPI
KiInitializeKernel(IN PKPROCESS InitProcess,IN PKTHREAD InitThread,IN PVOID IdleStack,IN PKPRCB Prcb,IN CCHAR Number,IN PLOADER_PARAMETER_BLOCK LoaderBlock)432 KiInitializeKernel(IN PKPROCESS InitProcess,
433 IN PKTHREAD InitThread,
434 IN PVOID IdleStack,
435 IN PKPRCB Prcb,
436 IN CCHAR Number,
437 IN PLOADER_PARAMETER_BLOCK LoaderBlock)
438 {
439 ULONG PageDirectory[2];
440 PVOID DpcStack;
441 KIRQL DummyIrql;
442
443 /* Initialize the Power Management Support for this PRCB */
444 PoInitializePrcb(Prcb);
445
446 /* Set boot-level flags */
447 if (Number == 0)
448 KeFeatureBits = Prcb->FeatureBits;
449
450 /* Set the default NX policy (opt-in) */
451 SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTIN;
452
453 /* Check if NPX is always on */
454 if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSON"))
455 {
456 /* Set it always on */
457 SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSON;
458 KeFeatureBits |= KF_NX_ENABLED;
459 }
460 else if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTOUT"))
461 {
462 /* Set it in opt-out mode */
463 SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTOUT;
464 KeFeatureBits |= KF_NX_ENABLED;
465 }
466 else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTIN")) ||
467 (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE")))
468 {
469 /* Set the feature bits */
470 KeFeatureBits |= KF_NX_ENABLED;
471 }
472 else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSOFF")) ||
473 (strstr(KeLoaderBlock->LoadOptions, "EXECUTE")))
474 {
475 /* Set disabled mode */
476 SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSOFF;
477 KeFeatureBits |= KF_NX_DISABLED;
478 }
479
480 /* Save CPU state */
481 KiSaveProcessorControlState(&Prcb->ProcessorState);
482
483 #if DBG
484 /* Print applied kernel features/policies and boot CPU features */
485 if (Number == 0)
486 KiReportCpuFeatures();
487 #endif
488
489 /* Get cache line information for this CPU */
490 KiGetCacheInformation();
491
492 /* Initialize spinlocks and DPC data */
493 KiInitSpinLocks(Prcb, Number);
494
495 /* Set Node Data */
496 Prcb->ParentNode = KeNodeBlock[0];
497 Prcb->ParentNode->ProcessorMask |= Prcb->SetMember;
498
499 /* Check if this is the Boot CPU */
500 if (!Number)
501 {
502 /* Set boot-level flags */
503 KeI386CpuType = Prcb->CpuType;
504 KeI386CpuStep = Prcb->CpuStep;
505 KeProcessorArchitecture = PROCESSOR_ARCHITECTURE_INTEL;
506 KeProcessorLevel = (USHORT)Prcb->CpuType;
507 if (Prcb->CpuID) KeProcessorRevision = Prcb->CpuStep;
508 KeI386FxsrPresent = (KeFeatureBits & KF_FXSR) ? TRUE : FALSE;
509 KeI386XMMIPresent = (KeFeatureBits & KF_XMMI) ? TRUE : FALSE;
510
511 /* Set the current MP Master KPRCB to the Boot PRCB */
512 Prcb->MultiThreadSetMaster = Prcb;
513
514 /* Lower to APC_LEVEL */
515 KeLowerIrql(APC_LEVEL);
516
517 /* Initialize some spinlocks */
518 KeInitializeSpinLock(&KiFreezeExecutionLock);
519 KeInitializeSpinLock(&Ki486CompatibilityLock);
520
521 /* Initialize portable parts of the OS */
522 KiInitSystem();
523
524 /* Initialize the Idle Process and the Process Listhead */
525 InitializeListHead(&KiProcessListHead);
526 PageDirectory[0] = 0;
527 PageDirectory[1] = 0;
528 KeInitializeProcess(InitProcess,
529 0,
530 MAXULONG_PTR,
531 PageDirectory,
532 FALSE);
533 InitProcess->QuantumReset = MAXCHAR;
534 }
535 else
536 {
537 /* FIXME */
538 DPRINT1("Starting CPU#%u - you are brave\n", Number);
539 }
540
541 /* Setup the Idle Thread */
542 KeInitializeThread(InitProcess,
543 InitThread,
544 NULL,
545 NULL,
546 NULL,
547 NULL,
548 NULL,
549 IdleStack);
550 InitThread->NextProcessor = Number;
551 InitThread->Priority = HIGH_PRIORITY;
552 InitThread->State = Running;
553 InitThread->Affinity = 1 << Number;
554 InitThread->WaitIrql = DISPATCH_LEVEL;
555 InitProcess->ActiveProcessors |= 1 << Number;
556
557 /* HACK for MmUpdatePageDir */
558 ((PETHREAD)InitThread)->ThreadsProcess = (PEPROCESS)InitProcess;
559
560 /* Set basic CPU Features that user mode can read */
561 SharedUserData->ProcessorFeatures[PF_FLOATING_POINT_PRECISION_ERRATA] = FALSE;
562 SharedUserData->ProcessorFeatures[PF_MMX_INSTRUCTIONS_AVAILABLE] =
563 (KeFeatureBits & KF_MMX) ? TRUE: FALSE;
564 SharedUserData->ProcessorFeatures[PF_COMPARE_EXCHANGE_DOUBLE] =
565 (KeFeatureBits & KF_CMPXCHG8B) ? TRUE: FALSE;
566 SharedUserData->ProcessorFeatures[PF_XMMI_INSTRUCTIONS_AVAILABLE] =
567 ((KeFeatureBits & KF_FXSR) && (KeFeatureBits & KF_XMMI)) ? TRUE: FALSE;
568 SharedUserData->ProcessorFeatures[PF_XMMI64_INSTRUCTIONS_AVAILABLE] =
569 ((KeFeatureBits & KF_FXSR) && (KeFeatureBits & KF_XMMI64)) ? TRUE: FALSE;
570 SharedUserData->ProcessorFeatures[PF_3DNOW_INSTRUCTIONS_AVAILABLE] =
571 (KeFeatureBits & KF_3DNOW) ? TRUE: FALSE;
572 SharedUserData->ProcessorFeatures[PF_RDTSC_INSTRUCTION_AVAILABLE] =
573 (KeFeatureBits & KF_RDTSC) ? TRUE: FALSE;
574
575 /* Set up the thread-related fields in the PRCB */
576 Prcb->CurrentThread = InitThread;
577 Prcb->NextThread = NULL;
578 Prcb->IdleThread = InitThread;
579
580 /* Initialize the Kernel Executive */
581 ExpInitializeExecutive(Number, LoaderBlock);
582
583 /* Only do this on the boot CPU */
584 if (!Number)
585 {
586 /* Calculate the time reciprocal */
587 KiTimeIncrementReciprocal =
588 KiComputeReciprocal(KeMaximumIncrement,
589 &KiTimeIncrementShiftCount);
590
591 /* Update DPC Values in case they got updated by the executive */
592 Prcb->MaximumDpcQueueDepth = KiMaximumDpcQueueDepth;
593 Prcb->MinimumDpcRate = KiMinimumDpcRate;
594 Prcb->AdjustDpcThreshold = KiAdjustDpcThreshold;
595
596 /* Allocate the DPC Stack */
597 DpcStack = MmCreateKernelStack(FALSE, 0);
598 if (!DpcStack) KeBugCheckEx(NO_PAGES_AVAILABLE, 1, 0, 0, 0);
599 Prcb->DpcStack = DpcStack;
600
601 /* Allocate the IOPM save area */
602 Ki386IopmSaveArea = ExAllocatePoolWithTag(PagedPool,
603 IOPM_SIZE,
604 TAG_KERNEL);
605 if (!Ki386IopmSaveArea)
606 {
607 /* Bugcheck. We need this for V86/VDM support. */
608 KeBugCheckEx(NO_PAGES_AVAILABLE, 2, IOPM_SIZE, 0, 0);
609 }
610 }
611
612 /* Raise to Dispatch */
613 KeRaiseIrql(DISPATCH_LEVEL, &DummyIrql);
614
615 /* Set the Idle Priority to 0. This will jump into Phase 1 */
616 KeSetPriorityThread(InitThread, 0);
617
618 /* If there's no thread scheduled, put this CPU in the Idle summary */
619 KiAcquirePrcbLock(Prcb);
620 if (!Prcb->NextThread) KiIdleSummary |= 1 << Number;
621 KiReleasePrcbLock(Prcb);
622
623 /* Raise back to HIGH_LEVEL and clear the PRCB for the loader block */
624 KeRaiseIrql(HIGH_LEVEL, &DummyIrql);
625 LoaderBlock->Prcb = 0;
626 }
627
628 CODE_SEG("INIT")
629 VOID
630 FASTCALL
KiGetMachineBootPointers(IN PKGDTENTRY * Gdt,IN PKIDTENTRY * Idt,IN PKIPCR * Pcr,IN PKTSS * Tss)631 KiGetMachineBootPointers(IN PKGDTENTRY *Gdt,
632 IN PKIDTENTRY *Idt,
633 IN PKIPCR *Pcr,
634 IN PKTSS *Tss)
635 {
636 KDESCRIPTOR GdtDescriptor, IdtDescriptor;
637 KGDTENTRY TssSelector, PcrSelector;
638 USHORT Tr, Fs;
639
640 /* Get GDT and IDT descriptors */
641 Ke386GetGlobalDescriptorTable(&GdtDescriptor.Limit);
642 __sidt(&IdtDescriptor.Limit);
643
644 /* Save IDT and GDT */
645 *Gdt = (PKGDTENTRY)GdtDescriptor.Base;
646 *Idt = (PKIDTENTRY)IdtDescriptor.Base;
647
648 /* Get TSS and FS Selectors */
649 Tr = Ke386GetTr();
650 Fs = Ke386GetFs();
651
652 /* Get PCR Selector, mask it and get its GDT Entry */
653 PcrSelector = *(PKGDTENTRY)((ULONG_PTR)*Gdt + (Fs & ~RPL_MASK));
654
655 /* Get the KPCR itself */
656 *Pcr = (PKIPCR)(ULONG_PTR)(PcrSelector.BaseLow |
657 PcrSelector.HighWord.Bytes.BaseMid << 16 |
658 PcrSelector.HighWord.Bytes.BaseHi << 24);
659
660 /* Get TSS Selector, mask it and get its GDT Entry */
661 TssSelector = *(PKGDTENTRY)((ULONG_PTR)*Gdt + (Tr & ~RPL_MASK));
662
663 /* Get the KTSS itself */
664 *Tss = (PKTSS)(ULONG_PTR)(TssSelector.BaseLow |
665 TssSelector.HighWord.Bytes.BaseMid << 16 |
666 TssSelector.HighWord.Bytes.BaseHi << 24);
667 }
668
669 CODE_SEG("INIT")
670 DECLSPEC_NORETURN
671 VOID
672 NTAPI
KiSystemStartupBootStack(VOID)673 KiSystemStartupBootStack(VOID)
674 {
675 PKTHREAD Thread;
676
677 /* Initialize the kernel for the current CPU */
678 KiInitializeKernel(&KiInitialProcess.Pcb,
679 (PKTHREAD)KeLoaderBlock->Thread,
680 (PVOID)(KeLoaderBlock->KernelStack & ~3),
681 (PKPRCB)__readfsdword(KPCR_PRCB),
682 KeNumberProcessors - 1,
683 KeLoaderBlock);
684
685 /* Set the priority of this thread to 0 */
686 Thread = KeGetCurrentThread();
687 Thread->Priority = 0;
688
689 /* Force interrupts enabled and lower IRQL back to DISPATCH_LEVEL */
690 _enable();
691 KeLowerIrql(DISPATCH_LEVEL);
692
693 /* Set the right wait IRQL */
694 Thread->WaitIrql = DISPATCH_LEVEL;
695
696 /* Jump into the idle loop */
697 KiIdleLoop();
698 }
699
700 static
701 VOID
KiMarkPageAsReadOnly(PVOID Address)702 KiMarkPageAsReadOnly(
703 PVOID Address)
704 {
705 PHARDWARE_PTE PointerPte;
706
707 /* Make sure the address is page aligned */
708 ASSERT(ALIGN_DOWN_POINTER_BY(Address, PAGE_SIZE) == Address);
709
710 /* Get the PTE address */
711 PointerPte = ((PHARDWARE_PTE)PTE_BASE) + ((ULONG_PTR)Address / PAGE_SIZE);
712 ASSERT(PointerPte->Valid);
713 ASSERT(PointerPte->Write);
714
715 /* Set as read-only */
716 PointerPte->Write = 0;
717
718 /* Flush the TLB entry */
719 __invlpg(Address);
720 }
721
722 CODE_SEG("INIT")
723 DECLSPEC_NORETURN
724 VOID
725 NTAPI
KiSystemStartup(IN PLOADER_PARAMETER_BLOCK LoaderBlock)726 KiSystemStartup(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
727 {
728 ULONG Cpu;
729 PKTHREAD InitialThread;
730 ULONG InitialStack;
731 PKGDTENTRY Gdt;
732 PKIDTENTRY Idt;
733 KIDTENTRY NmiEntry, DoubleFaultEntry;
734 PKTSS Tss;
735 PKIPCR Pcr;
736 KIRQL DummyIrql;
737
738 /* Boot cycles timestamp */
739 BootCycles = __rdtsc();
740
741 /* Save the loader block and get the current CPU */
742 KeLoaderBlock = LoaderBlock;
743 Cpu = KeNumberProcessors;
744 if (!Cpu)
745 {
746 /* If this is the boot CPU, set FS and the CPU Number*/
747 Ke386SetFs(KGDT_R0_PCR);
748 __writefsdword(KPCR_PROCESSOR_NUMBER, Cpu);
749
750 /* Set the initial stack and idle thread as well */
751 LoaderBlock->KernelStack = (ULONG_PTR)P0BootStack;
752 LoaderBlock->Thread = (ULONG_PTR)&KiInitialThread;
753 }
754
755 /* Save the initial thread and stack */
756 InitialStack = LoaderBlock->KernelStack;
757 InitialThread = (PKTHREAD)LoaderBlock->Thread;
758
759 /* Clean the APC List Head */
760 InitializeListHead(&InitialThread->ApcState.ApcListHead[KernelMode]);
761
762 /* Initialize the machine type */
763 KiInitializeMachineType();
764
765 /* Skip initial setup if this isn't the Boot CPU */
766 if (Cpu) goto AppCpuInit;
767
768 /* Get GDT, IDT, PCR and TSS pointers */
769 KiGetMachineBootPointers(&Gdt, &Idt, &Pcr, &Tss);
770
771 /* Setup the TSS descriptors and entries */
772 Ki386InitializeTss(Tss, Idt, Gdt);
773
774 /* Initialize the PCR */
775 RtlZeroMemory(Pcr, PAGE_SIZE);
776 KiInitializePcr(Cpu,
777 Pcr,
778 Idt,
779 Gdt,
780 Tss,
781 InitialThread,
782 (PVOID)KiDoubleFaultStack);
783
784 /* Set us as the current process */
785 InitialThread->ApcState.Process = &KiInitialProcess.Pcb;
786
787 /* Clear DR6/7 to cleanup bootloader debugging */
788 __writefsdword(KPCR_TEB, 0);
789 __writefsdword(KPCR_DR6, 0);
790 __writefsdword(KPCR_DR7, 0);
791
792 /* Setup the IDT */
793 KeInitExceptions();
794
795 /* Load Ring 3 selectors for DS/ES */
796 Ke386SetDs(KGDT_R3_DATA | RPL_MASK);
797 Ke386SetEs(KGDT_R3_DATA | RPL_MASK);
798
799 /* Save NMI and double fault traps */
800 RtlCopyMemory(&NmiEntry, &Idt[2], sizeof(KIDTENTRY));
801 RtlCopyMemory(&DoubleFaultEntry, &Idt[8], sizeof(KIDTENTRY));
802
803 /* Copy kernel's trap handlers */
804 RtlCopyMemory(Idt,
805 (PVOID)KiIdtDescriptor.Base,
806 KiIdtDescriptor.Limit + 1);
807
808 /* Restore NMI and double fault */
809 RtlCopyMemory(&Idt[2], &NmiEntry, sizeof(KIDTENTRY));
810 RtlCopyMemory(&Idt[8], &DoubleFaultEntry, sizeof(KIDTENTRY));
811
812 AppCpuInit:
813 //TODO: We don't setup IPIs yet so freeze other processors here.
814 if (Cpu)
815 {
816 KeMemoryBarrier();
817 LoaderBlock->Prcb = 0;
818
819 for (;;)
820 {
821 YieldProcessor();
822 }
823 }
824
825 /* Loop until we can release the freeze lock */
826 do
827 {
828 /* Loop until execution can continue */
829 while (*(volatile PKSPIN_LOCK*)&KiFreezeExecutionLock == (PVOID)1);
830 } while(InterlockedBitTestAndSet((PLONG)&KiFreezeExecutionLock, 0));
831
832 /* Setup CPU-related fields */
833 __writefsdword(KPCR_NUMBER, Cpu);
834 __writefsdword(KPCR_SET_MEMBER, 1 << Cpu);
835 __writefsdword(KPCR_SET_MEMBER_COPY, 1 << Cpu);
836 __writefsdword(KPCR_PRCB_SET_MEMBER, 1 << Cpu);
837
838 KiVerifyCpuFeatures(Pcr->Prcb);
839
840 /* Initialize the Processor with HAL */
841 HalInitializeProcessor(Cpu, KeLoaderBlock);
842
843 /* Set active processors */
844 KeActiveProcessors |= __readfsdword(KPCR_SET_MEMBER);
845 KeNumberProcessors++;
846
847 /* Check if this is the boot CPU */
848 if (!Cpu)
849 {
850 /* Initialize debugging system */
851 KdInitSystem(0, KeLoaderBlock);
852
853 /* Check for break-in */
854 if (KdPollBreakIn()) DbgBreakPointWithStatus(DBG_STATUS_CONTROL_C);
855
856 /* Make the lowest page of the boot and double fault stack read-only */
857 KiMarkPageAsReadOnly(P0BootStackData);
858 KiMarkPageAsReadOnly(KiDoubleFaultStackData);
859 }
860
861 /* Raise to HIGH_LEVEL */
862 KeRaiseIrql(HIGH_LEVEL, &DummyIrql);
863
864 /* Switch to new kernel stack and start kernel bootstrapping */
865 KiSwitchToBootStack(InitialStack & ~3);
866 }
867