xref: /reactos/ntoskrnl/ke/amd64/kiinit.c (revision cce399e7) 
1 /*
2  * PROJECT:         ReactOS Kernel
3  * LICENSE:         GPL - See COPYING in the top level directory
4  * FILE:            ntoskrnl/ke/amd64/kiinit.c
5  * PURPOSE:         Kernel Initialization for x86 CPUs
6  * PROGRAMMERS:     Alex Ionescu (alex.ionescu@reactos.org)
7  *                  Timo Kreuzer (timo.kreuzer@reactos.org)
8  */
9 
10 /* INCLUDES *****************************************************************/
11 
12 #include <ntoskrnl.h>
13 #define NDEBUG
14 #include <debug.h>
15 
16 #define REQUIRED_FEATURE_BITS (KF_RDTSC|KF_CR4|KF_CMPXCHG8B|KF_XMMI|KF_XMMI64| \
17                                KF_LARGE_PAGE|KF_FAST_SYSCALL|KF_GLOBAL_PAGE| \
18                                KF_CMOV|KF_PAT|KF_MMX|KF_FXSR|KF_NX_BIT|KF_MTRR)
19 
20 /* GLOBALS *******************************************************************/
21 
22 /* Function pointer for early debug prints */
23 ULONG (*FrLdrDbgPrint)(const char *Format, ...);
24 
25 /* Spinlocks used only on X86 */
26 KSPIN_LOCK KiFreezeExecutionLock;
27 
28 
29 KIPCR KiInitialPcr;
30 
31 /* Boot and double-fault/NMI/DPC stack */
32 UCHAR DECLSPEC_ALIGN(16) P0BootStackData[KERNEL_STACK_SIZE] = {0};
33 UCHAR DECLSPEC_ALIGN(16) KiDoubleFaultStackData[KERNEL_STACK_SIZE] = {0};
34 ULONG_PTR P0BootStack = (ULONG_PTR)&P0BootStackData[KERNEL_STACK_SIZE];
35 ULONG_PTR KiDoubleFaultStack = (ULONG_PTR)&KiDoubleFaultStackData[KERNEL_STACK_SIZE];
36 
37 ULONGLONG BootCycles, BootCyclesEnd;
38 
39 void KiInitializeSegments();
40 void KiSystemCallEntry64();
41 void KiSystemCallEntry32();
42 
43 /* FUNCTIONS *****************************************************************/
44 
45 CODE_SEG("INIT")
46 VOID
47 NTAPI
48 KiInitMachineDependent(VOID)
49 {
50     /* Check for large page support */
51     if (KeFeatureBits & KF_LARGE_PAGE)
52     {
53         /* FIXME: Support this */
54         DPRINT("Large Page support detected but not yet taken advantage of!\n");
55     }
56 
57     /* Check for global page support */
58     if (KeFeatureBits & KF_GLOBAL_PAGE)
59     {
60         /* FIXME: Support this */
61         DPRINT("Global Page support detected but not yet taken advantage of!\n");
62     }
63 
64     /* Check if we have MTRR */
65     if (KeFeatureBits & KF_MTRR)
66     {
67         /* FIXME: Support this */
68         DPRINT("MTRR support detected but not yet taken advantage of!\n");
69     }
70 
71     /* Check for PAT and/or MTRR support */
72     if (KeFeatureBits & KF_PAT)
73     {
74         /* FIXME: Support this */
75         DPRINT("PAT support detected but not yet taken advantage of!\n");
76     }
77 
78 //        /* Allocate the IOPM save area */
79 //        Ki386IopmSaveArea = ExAllocatePoolWithTag(PagedPool,
80 //                                                  IOPM_SIZE,
81 //                                                  '  eK');
82 //        if (!Ki386IopmSaveArea)
83 //        {
84 //            /* Bugcheck. We need this for V86/VDM support. */
85 //            KeBugCheckEx(NO_PAGES_AVAILABLE, 2, IOPM_SIZE, 0, 0);
86 //        }
87 
88 }
89 
90 VOID
91 NTAPI
92 KiInitializePcr(IN PKIPCR Pcr,
93                 IN ULONG ProcessorNumber,
94                 IN PKTHREAD IdleThread,
95                 IN PVOID DpcStack)
96 {
97     KDESCRIPTOR GdtDescriptor = {{0},0,0}, IdtDescriptor = {{0},0,0};
98     PKGDTENTRY64 TssEntry;
99     USHORT Tr = 0;
100 
101     /* Zero out the PCR */
102     RtlZeroMemory(Pcr, sizeof(KIPCR));
103 
104     /* Set pointers to ourselves */
105     Pcr->Self = (PKPCR)Pcr;
106     Pcr->CurrentPrcb = &Pcr->Prcb;
107 
108     /* Set the PCR Version */
109     Pcr->MajorVersion = PCR_MAJOR_VERSION;
110     Pcr->MinorVersion = PCR_MINOR_VERSION;
111 
112     /* Set the PRCB Version */
113     Pcr->Prcb.MajorVersion = PRCB_MAJOR_VERSION;
114     Pcr->Prcb.MinorVersion = PRCB_MINOR_VERSION;
115 
116     /* Set the Build Type */
117     Pcr->Prcb.BuildType = 0;
118 #ifndef CONFIG_SMP
119     Pcr->Prcb.BuildType |= PRCB_BUILD_UNIPROCESSOR;
120 #endif
121 #if DBG
122     Pcr->Prcb.BuildType |= PRCB_BUILD_DEBUG;
123 #endif
124 
125     /* Set the Processor Number and current Processor Mask */
126     Pcr->Prcb.Number = (UCHAR)ProcessorNumber;
127     Pcr->Prcb.SetMember = 1ULL << ProcessorNumber;
128 
129     /* Get GDT and IDT descriptors */
130     __sgdt(&GdtDescriptor.Limit);
131     __sidt(&IdtDescriptor.Limit);
132     Pcr->GdtBase = (PVOID)GdtDescriptor.Base;
133     Pcr->IdtBase = (PKIDTENTRY)IdtDescriptor.Base;
134 
135     /* Get TSS Selector */
136     __str(&Tr);
137     ASSERT(Tr == KGDT64_SYS_TSS);
138 
139     /* Get TSS Entry */
140     TssEntry = KiGetGdtEntry(Pcr->GdtBase, Tr);
141 
142     /* Get the KTSS itself */
143     Pcr->TssBase = KiGetGdtDescriptorBase(TssEntry);
144 
145     Pcr->Prcb.RspBase = Pcr->TssBase->Rsp0; // FIXME
146 
147     /* Set DPC Stack */
148     Pcr->Prcb.DpcStack = DpcStack;
149 
150     /* Setup the processor set */
151     Pcr->Prcb.MultiThreadProcessorSet = Pcr->Prcb.SetMember;
152 
153     /* Clear DR6/7 to cleanup bootloader debugging */
154     Pcr->Prcb.ProcessorState.SpecialRegisters.KernelDr6 = 0;
155     Pcr->Prcb.ProcessorState.SpecialRegisters.KernelDr7 = 0;
156 
157     /* Initialize MXCSR (all exceptions masked) */
158     Pcr->Prcb.MxCsr = INITIAL_MXCSR;
159 
160     /* Set the Current Thread */
161     Pcr->Prcb.CurrentThread = IdleThread;
162 
163     /* Start us out at PASSIVE_LEVEL */
164     Pcr->Irql = PASSIVE_LEVEL;
165     KeSetCurrentIrql(PASSIVE_LEVEL);
166 }
167 
168 VOID
169 NTAPI
170 KiInitializeCpu(PKIPCR Pcr)
171 {
172     ULONG64 Pat;
173     ULONG64 FeatureBits;
174 
175     /* Initialize gs */
176     KiInitializeSegments();
177 
178     /* Set GS base */
179     __writemsr(MSR_GS_BASE, (ULONG64)Pcr);
180     __writemsr(MSR_GS_SWAP, (ULONG64)Pcr);
181 
182     /* Detect and set the CPU Type */
183     KiSetProcessorType();
184 
185     /* Get the processor features for this CPU */
186     FeatureBits = KiGetFeatureBits();
187 
188     /* Check if we support all needed features */
189     if ((FeatureBits & REQUIRED_FEATURE_BITS) != REQUIRED_FEATURE_BITS)
190     {
191         /* If not, bugcheck system */
192         FrLdrDbgPrint("CPU doesn't have needed features! Has: 0x%x, required: 0x%x\n",
193                 FeatureBits, REQUIRED_FEATURE_BITS);
194         KeBugCheck(0);
195     }
196 
197     /* Set DEP to always on */
198     SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSON;
199     FeatureBits |= KF_NX_ENABLED;
200 
201     /* Save feature bits */
202     Pcr->Prcb.FeatureBits = (ULONG)FeatureBits;
203     Pcr->Prcb.FeatureBitsHigh = FeatureBits >> 32;
204 
205     /* Enable fx save restore support */
206     __writecr4(__readcr4() | CR4_FXSR);
207 
208     /* Enable XMMI exceptions */
209     __writecr4(__readcr4() | CR4_XMMEXCPT);
210 
211     /* Enable Write-Protection */
212     __writecr0(__readcr0() | CR0_WP);
213 
214     /* Disable fpu monitoring */
215     __writecr0(__readcr0() & ~CR0_MP);
216 
217     /* Disable x87 fpu exceptions */
218     __writecr0(__readcr0() & ~CR0_NE);
219 
220     /* LDT is unused */
221     __lldt(0);
222 
223     /* Set the systemcall entry points */
224     __writemsr(MSR_LSTAR, (ULONG64)KiSystemCallEntry64);
225     __writemsr(MSR_CSTAR, (ULONG64)KiSystemCallEntry32);
226 
227     __writemsr(MSR_STAR, ((ULONG64)KGDT64_R0_CODE << 32) |
228                          ((ULONG64)(KGDT64_R3_CMCODE|RPL_MASK) << 48));
229 
230     /* Set the flags to be cleared when doing a syscall */
231     __writemsr(MSR_SYSCALL_MASK, EFLAGS_IF_MASK | EFLAGS_TF | EFLAGS_DF);
232 
233     /* Enable syscall instruction and no-execute support */
234     __writemsr(MSR_EFER, __readmsr(MSR_EFER) | MSR_SCE | MSR_NXE);
235 
236     /* Initialize the PAT */
237     Pat = (PAT_WB << 0)  | (PAT_WC << 8) | (PAT_UCM << 16) | (PAT_UC << 24) |
238           (PAT_WB << 32) | (PAT_WC << 40) | (PAT_UCM << 48) | (PAT_UC << 56);
239     __writemsr(MSR_PAT, Pat);
240 
241     /* Initialize MXCSR */
242     _mm_setcsr(INITIAL_MXCSR);
243 }
244 
245 VOID
246 FASTCALL
247 KiInitializeTss(IN PKTSS64 Tss,
248                 IN UINT64 Stack)
249 {
250     PKGDTENTRY64 TssEntry;
251 
252     /* Get pointer to the GDT entry */
253     TssEntry = KiGetGdtEntry(KeGetPcr()->GdtBase, KGDT64_SYS_TSS);
254 
255     /* Initialize the GDT entry */
256     KiInitGdtEntry(TssEntry, (ULONG64)Tss, sizeof(KTSS64), AMD64_TSS, 0);
257 
258     /* Zero out the TSS */
259     RtlZeroMemory(Tss, sizeof(KTSS64));
260 
261     /* FIXME: I/O Map? */
262     Tss->IoMapBase = 0x68;
263 
264     /* Setup ring 0 stack pointer */
265     Tss->Rsp0 = Stack;
266 
267     /* Setup a stack for Double Fault Traps */
268     Tss->Ist[1] = (ULONG64)KiDoubleFaultStack;
269 
270     /* Setup a stack for CheckAbort Traps */
271     Tss->Ist[2] = (ULONG64)KiDoubleFaultStack;
272 
273     /* Setup a stack for NMI Traps */
274     Tss->Ist[3] = (ULONG64)KiDoubleFaultStack;
275 
276     /* Load the task register */
277     __ltr(KGDT64_SYS_TSS);
278 }
279 
280 CODE_SEG("INIT")
281 VOID
282 NTAPI
283 KiInitializeKernelMachineDependent(
284     IN PKPRCB Prcb,
285     IN PLOADER_PARAMETER_BLOCK LoaderBlock)
286 {
287     ULONG64 FeatureBits;
288 
289     /* Set boot-level flags */
290     KeI386CpuType = Prcb->CpuType;
291     KeI386CpuStep = Prcb->CpuStep;
292     KeProcessorArchitecture = PROCESSOR_ARCHITECTURE_AMD64;
293     KeProcessorLevel = (USHORT)Prcb->CpuType;
294     if (Prcb->CpuID)
295         KeProcessorRevision = Prcb->CpuStep;
296 
297     FeatureBits = Prcb->FeatureBits | (ULONG64)Prcb->FeatureBitsHigh << 32;
298 
299     /* Set basic CPU Features that user mode can read */
300     SharedUserData->ProcessorFeatures[PF_FLOATING_POINT_PRECISION_ERRATA] = FALSE;
301     SharedUserData->ProcessorFeatures[PF_FLOATING_POINT_EMULATED] = FALSE;
302     SharedUserData->ProcessorFeatures[PF_COMPARE_EXCHANGE_DOUBLE] = TRUE;
303     SharedUserData->ProcessorFeatures[PF_MMX_INSTRUCTIONS_AVAILABLE] =
304         (FeatureBits & KF_MMX) ? TRUE : FALSE;
305     SharedUserData->ProcessorFeatures[PF_XMMI_INSTRUCTIONS_AVAILABLE] =
306         ((FeatureBits & KF_FXSR) && (FeatureBits & KF_XMMI)) ? TRUE : FALSE;
307     SharedUserData->ProcessorFeatures[PF_3DNOW_INSTRUCTIONS_AVAILABLE] =
308         (FeatureBits & KF_3DNOW) ? TRUE : FALSE;
309     SharedUserData->ProcessorFeatures[PF_RDTSC_INSTRUCTION_AVAILABLE] = TRUE;
310     SharedUserData->ProcessorFeatures[PF_PAE_ENABLED] = TRUE; // ???
311     SharedUserData->ProcessorFeatures[PF_XMMI64_INSTRUCTIONS_AVAILABLE] =
312         ((FeatureBits & KF_FXSR) && (FeatureBits & KF_XMMI64)) ? TRUE : FALSE;
313     SharedUserData->ProcessorFeatures[PF_SSE_DAZ_MODE_AVAILABLE] = FALSE; // ???
314     SharedUserData->ProcessorFeatures[PF_NX_ENABLED] = TRUE;
315     SharedUserData->ProcessorFeatures[PF_SSE3_INSTRUCTIONS_AVAILABLE] =
316         (FeatureBits & KF_SSE3) ? TRUE : FALSE;
317     SharedUserData->ProcessorFeatures[PF_COMPARE_EXCHANGE128] =
318         (FeatureBits & KF_CMPXCHG16B) ? TRUE : FALSE;
319     SharedUserData->ProcessorFeatures[PF_COMPARE64_EXCHANGE128] = FALSE; // ???
320     SharedUserData->ProcessorFeatures[PF_CHANNELS_ENABLED] = FALSE; // ???
321     SharedUserData->ProcessorFeatures[PF_XSAVE_ENABLED] = FALSE; // FIXME
322     SharedUserData->ProcessorFeatures[PF_SECOND_LEVEL_ADDRESS_TRANSLATION] =
323         (FeatureBits & KF_SLAT) ? TRUE : FALSE;
324     SharedUserData->ProcessorFeatures[PF_VIRT_FIRMWARE_ENABLED] =
325         (FeatureBits & KF_VIRT_FIRMWARE_ENABLED) ? TRUE : FALSE;
326     SharedUserData->ProcessorFeatures[PF_RDWRFSGSBASE_AVAILABLE] =
327         (FeatureBits & KF_RDWRFSGSBASE) ? TRUE : FALSE;
328     SharedUserData->ProcessorFeatures[PF_FASTFAIL_AVAILABLE] = TRUE;
329     SharedUserData->ProcessorFeatures[PF_RDRAND_INSTRUCTION_AVAILABLE] =
330         (FeatureBits & KF_RDRAND) ? TRUE : FALSE;
331     SharedUserData->ProcessorFeatures[PF_RDTSCP_INSTRUCTION_AVAILABLE] =
332         (FeatureBits & KF_RDTSCP) ? TRUE : FALSE;
333     SharedUserData->ProcessorFeatures[PF_RDPID_INSTRUCTION_AVAILABLE] = FALSE; // ???
334     SharedUserData->ProcessorFeatures[PF_SSSE3_INSTRUCTIONS_AVAILABLE] =
335         (FeatureBits & KF_SSSE3) ? TRUE : FALSE;
336     SharedUserData->ProcessorFeatures[PF_SSE4_1_INSTRUCTIONS_AVAILABLE] =
337         (FeatureBits & KF_SSE4_1) ? TRUE : FALSE;
338     SharedUserData->ProcessorFeatures[PF_SSE4_2_INSTRUCTIONS_AVAILABLE] =
339         (FeatureBits & KF_SSE4_2) ? TRUE : FALSE;
340     SharedUserData->ProcessorFeatures[PF_AVX_INSTRUCTIONS_AVAILABLE] = FALSE; // FIXME
341     SharedUserData->ProcessorFeatures[PF_AVX2_INSTRUCTIONS_AVAILABLE] = FALSE; // FIXME
342     SharedUserData->ProcessorFeatures[PF_AVX512F_INSTRUCTIONS_AVAILABLE] = FALSE; // FIXME
343 
344     /* Set the default NX policy (opt-in) */
345     SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTIN;
346 
347     /* Check if NPX is always on */
348     if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSON"))
349     {
350         /* Set it always on */
351         SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSON;
352         Prcb->FeatureBits |= KF_NX_ENABLED;
353     }
354     else if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTOUT"))
355     {
356         /* Set it in opt-out mode */
357         SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTOUT;
358         Prcb->FeatureBits |= KF_NX_ENABLED;
359     }
360     else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTIN")) ||
361              (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE")))
362     {
363         /* Set the feature bits */
364         Prcb->FeatureBits |= KF_NX_ENABLED;
365     }
366     else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSOFF")) ||
367              (strstr(KeLoaderBlock->LoadOptions, "EXECUTE")))
368     {
369         /* Set disabled mode */
370         SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSOFF;
371         Prcb->FeatureBits |= KF_NX_DISABLED;
372     }
373 
374 #if DBG
375     /* Print applied kernel features/policies and boot CPU features */
376     KiReportCpuFeatures(Prcb);
377 #endif
378 }
379 
380 static LDR_DATA_TABLE_ENTRY LdrCoreEntries[3];
381 
382 void
383 KiInitModuleList(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
384 {
385     PLDR_DATA_TABLE_ENTRY LdrEntry;
386     PLIST_ENTRY Entry;
387     ULONG i;
388 
389     /* Initialize the list head */
390     InitializeListHead(&PsLoadedModuleList);
391 
392     /* Loop the first 3 entries */
393     for (Entry = LoaderBlock->LoadOrderListHead.Flink, i = 0;
394          Entry != &LoaderBlock->LoadOrderListHead && i < 3;
395          Entry = Entry->Flink, i++)
396     {
397         /* Get the data table entry */
398         LdrEntry = CONTAINING_RECORD(Entry,
399                                      LDR_DATA_TABLE_ENTRY,
400                                      InLoadOrderLinks);
401 
402         /* Copy the entry */
403         LdrCoreEntries[i] = *LdrEntry;
404 
405         /* Insert the copy into the list */
406         InsertTailList(&PsLoadedModuleList, &LdrCoreEntries[i].InLoadOrderLinks);
407     }
408 }
409 
410 CODE_SEG("INIT")
411 DECLSPEC_NORETURN
412 VOID
413 NTAPI
414 KiSystemStartup(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
415 {
416     CCHAR Cpu;
417     PKTHREAD InitialThread;
418     ULONG64 InitialStack;
419     PKIPCR Pcr;
420 
421     /* Boot cycles timestamp */
422     BootCycles = __rdtsc();
423 
424     /* HACK */
425     FrLdrDbgPrint = LoaderBlock->u.I386.CommonDataArea;
426     //FrLdrDbgPrint("Hello from KiSystemStartup!!!\n");
427 
428     /* Save the loader block */
429     KeLoaderBlock = LoaderBlock;
430 
431     /* Get the current CPU number */
432     Cpu = KeNumberProcessors++; // FIXME
433 
434     /* LoaderBlock initialization for Cpu 0 */
435     if (Cpu == 0)
436     {
437         /* Set the initial stack, idle thread and process */
438         LoaderBlock->KernelStack = (ULONG_PTR)P0BootStack;
439         LoaderBlock->Thread = (ULONG_PTR)&KiInitialThread;
440         LoaderBlock->Process = (ULONG_PTR)&KiInitialProcess.Pcb;
441         LoaderBlock->Prcb = (ULONG_PTR)&KiInitialPcr.Prcb;
442     }
443 
444     /* Get Pcr from loader block */
445     Pcr = CONTAINING_RECORD(LoaderBlock->Prcb, KIPCR, Prcb);
446 
447     /* Set the PRCB for this Processor */
448     KiProcessorBlock[Cpu] = &Pcr->Prcb;
449 
450     /* Align stack to 16 bytes */
451     LoaderBlock->KernelStack &= ~(16 - 1);
452 
453     /* Save the initial thread and stack */
454     InitialStack = LoaderBlock->KernelStack; // Checkme
455     InitialThread = (PKTHREAD)LoaderBlock->Thread;
456 
457     /* Set us as the current process */
458     InitialThread->ApcState.Process = (PVOID)LoaderBlock->Process;
459 
460     /* Initialize the PCR */
461     KiInitializePcr(Pcr, Cpu, InitialThread, (PVOID)KiDoubleFaultStack);
462 
463     /* Initialize the CPU features */
464     KiInitializeCpu(Pcr);
465 
466     /* Initial setup for the boot CPU */
467     if (Cpu == 0)
468     {
469         /* Initialize the module list (ntos, hal, kdcom) */
470         KiInitModuleList(LoaderBlock);
471 
472         /* Setup the TSS descriptors and entries */
473         KiInitializeTss(Pcr->TssBase, InitialStack);
474 
475         /* Setup the IDT */
476         KeInitExceptions();
477 
478          /* Initialize debugging system */
479         KdInitSystem(0, KeLoaderBlock);
480 
481         /* Check for break-in */
482         if (KdPollBreakIn()) DbgBreakPointWithStatus(DBG_STATUS_CONTROL_C);
483     }
484 
485     DPRINT1("Pcr = %p, Gdt = %p, Idt = %p, Tss = %p\n",
486            Pcr, Pcr->GdtBase, Pcr->IdtBase, Pcr->TssBase);
487 
488     /* Acquire lock */
489     while (InterlockedBitTestAndSet64((PLONG64)&KiFreezeExecutionLock, 0))
490     {
491         /* Loop until lock is free */
492         while ((*(volatile KSPIN_LOCK*)&KiFreezeExecutionLock) & 1);
493     }
494 
495     /* Initialize the Processor with HAL */
496     HalInitializeProcessor(Cpu, KeLoaderBlock);
497 
498     /* Set processor as active */
499     KeActiveProcessors |= 1ULL << Cpu;
500 
501     /* Release lock */
502     InterlockedAnd64((PLONG64)&KiFreezeExecutionLock, 0);
503 
504     /* Raise to HIGH_LEVEL */
505     KfRaiseIrql(HIGH_LEVEL);
506 
507     /* Machine specific kernel initialization */
508     if (Cpu == 0) KiInitializeKernelMachineDependent(&Pcr->Prcb, LoaderBlock);
509 
510     /* Switch to new kernel stack and start kernel bootstrapping */
511     KiSwitchToBootStack(InitialStack & ~3);
512 }
513 
514