xref: /reactos/ntoskrnl/ke/amd64/kiinit.c (revision c2c66aff) 
1 /*
2  * PROJECT:         ReactOS Kernel
3  * LICENSE:         GPL - See COPYING in the top level directory
4  * FILE:            ntoskrnl/ke/amd64/kiinit.c
5  * PURPOSE:         Kernel Initialization for x86 CPUs
6  * PROGRAMMERS:     Alex Ionescu (alex.ionescu@reactos.org)
7  *                  Timo Kreuzer (timo.kreuzer@reactos.org)
8  */
9 
10 /* INCLUDES *****************************************************************/
11 
12 #include <ntoskrnl.h>
13 #define NDEBUG
14 #include <debug.h>
15 
16 #define REQUIRED_FEATURE_BITS (KF_RDTSC|KF_CR4|KF_CMPXCHG8B|KF_XMMI|KF_XMMI64| \
17                                KF_LARGE_PAGE|KF_FAST_SYSCALL|KF_GLOBAL_PAGE| \
18                                KF_CMOV|KF_PAT|KF_MMX|KF_FXSR|KF_NX_BIT|KF_MTRR)
19 
20 /* GLOBALS *******************************************************************/
21 
22 extern BOOLEAN RtlpUse16ByteSLists;
23 
24 /* Function pointer for early debug prints */
25 ULONG (*FrLdrDbgPrint)(const char *Format, ...);
26 
27 /* Spinlocks used only on X86 */
28 KSPIN_LOCK KiFreezeExecutionLock;
29 
30 
31 KIPCR KiInitialPcr;
32 
33 /* Boot and double-fault/NMI/DPC stack */
34 UCHAR DECLSPEC_ALIGN(16) P0BootStackData[KERNEL_STACK_SIZE] = {0};
35 UCHAR DECLSPEC_ALIGN(16) KiDoubleFaultStackData[KERNEL_STACK_SIZE] = {0};
36 ULONG_PTR P0BootStack = (ULONG_PTR)&P0BootStackData[KERNEL_STACK_SIZE];
37 ULONG_PTR KiDoubleFaultStack = (ULONG_PTR)&KiDoubleFaultStackData[KERNEL_STACK_SIZE];
38 
39 void KiInitializeSegments();
40 void KiSystemCallEntry64();
41 void KiSystemCallEntry32();
42 
43 /* FUNCTIONS *****************************************************************/
44 
45 VOID
46 NTAPI
47 KiInitMachineDependent(VOID)
48 {
49     /* Check for large page support */
50     if (KeFeatureBits & KF_LARGE_PAGE)
51     {
52         /* FIXME: Support this */
53         DPRINT("Large Page support detected but not yet taken advantage of!\n");
54     }
55 
56     /* Check for global page support */
57     if (KeFeatureBits & KF_GLOBAL_PAGE)
58     {
59         /* FIXME: Support this */
60         DPRINT("Global Page support detected but not yet taken advantage of!\n");
61     }
62 
63     /* Check if we have MTRR */
64     if (KeFeatureBits & KF_MTRR)
65     {
66         /* FIXME: Support this */
67         DPRINT("MTRR support detected but not yet taken advantage of!\n");
68     }
69 
70     /* Check for PAT and/or MTRR support */
71     if (KeFeatureBits & KF_PAT)
72     {
73         /* FIXME: Support this */
74         DPRINT("PAT support detected but not yet taken advantage of!\n");
75     }
76 
77         /* Allocate the IOPM save area. */
78 //        Ki386IopmSaveArea = ExAllocatePoolWithTag(PagedPool,
79 //                                                  PAGE_SIZE * 2,
80 //                                                  TAG('K', 'e', ' ', ' '));
81 //        if (!Ki386IopmSaveArea)
82 //        {
83 //            /* Bugcheck. We need this for V86/VDM support. */
84 //            KeBugCheckEx(NO_PAGES_AVAILABLE, 2, PAGE_SIZE * 2, 0, 0);
85 //        }
86 
87     /* Initialize 8/16 bit SList support */
88     RtlpUse16ByteSLists = (KeFeatureBits & KF_CMPXCHG16B) ? TRUE: FALSE;
89 }
90 
91 VOID
92 NTAPI
93 KiInitializePcr(IN PKIPCR Pcr,
94                 IN ULONG ProcessorNumber,
95                 IN PKTHREAD IdleThread,
96                 IN PVOID DpcStack)
97 {
98     KDESCRIPTOR GdtDescriptor = {{0},0,0}, IdtDescriptor = {{0},0,0};
99     PKGDTENTRY64 TssEntry;
100     USHORT Tr = 0;
101 
102     /* Zero out the PCR */
103     RtlZeroMemory(Pcr, sizeof(KIPCR));
104 
105     /* Set pointers to ourselves */
106     Pcr->Self = (PKPCR)Pcr;
107     Pcr->CurrentPrcb = &Pcr->Prcb;
108 
109     /* Set the PCR Version */
110     Pcr->MajorVersion = PCR_MAJOR_VERSION;
111     Pcr->MinorVersion = PCR_MINOR_VERSION;
112 
113     /* Set the PRCB Version */
114     Pcr->Prcb.MajorVersion = 1;
115     Pcr->Prcb.MinorVersion = 1;
116 
117     /* Set the Build Type */
118     Pcr->Prcb.BuildType = 0;
119 #ifndef CONFIG_SMP
120     Pcr->Prcb.BuildType |= PRCB_BUILD_UNIPROCESSOR;
121 #endif
122 #if DBG
123     Pcr->Prcb.BuildType |= PRCB_BUILD_DEBUG;
124 #endif
125 
126     /* Set the Processor Number and current Processor Mask */
127     Pcr->Prcb.Number = (UCHAR)ProcessorNumber;
128     Pcr->Prcb.SetMember = 1ULL << ProcessorNumber;
129 
130     /* Get GDT and IDT descriptors */
131     __sgdt(&GdtDescriptor.Limit);
132     __sidt(&IdtDescriptor.Limit);
133     Pcr->GdtBase = (PVOID)GdtDescriptor.Base;
134     Pcr->IdtBase = (PKIDTENTRY)IdtDescriptor.Base;
135 
136     /* Get TSS Selector */
137     __str(&Tr);
138     ASSERT(Tr == KGDT64_SYS_TSS);
139 
140     /* Get TSS Entry */
141     TssEntry = KiGetGdtEntry(Pcr->GdtBase, Tr);
142 
143     /* Get the KTSS itself */
144     Pcr->TssBase = KiGetGdtDescriptorBase(TssEntry);
145 
146     Pcr->Prcb.RspBase = Pcr->TssBase->Rsp0; // FIXME
147 
148     /* Set DPC Stack */
149     Pcr->Prcb.DpcStack = DpcStack;
150 
151     /* Setup the processor set */
152     Pcr->Prcb.MultiThreadProcessorSet = Pcr->Prcb.SetMember;
153 
154     /* Clear DR6/7 to cleanup bootloader debugging */
155     Pcr->Prcb.ProcessorState.SpecialRegisters.KernelDr6 = 0;
156     Pcr->Prcb.ProcessorState.SpecialRegisters.KernelDr7 = 0;
157 
158     /* Set the Current Thread */
159     Pcr->Prcb.CurrentThread = IdleThread;
160 
161     /* Start us out at PASSIVE_LEVEL */
162     Pcr->Irql = PASSIVE_LEVEL;
163     KeSetCurrentIrql(PASSIVE_LEVEL);
164 }
165 
166 VOID
167 NTAPI
168 KiInitializeCpu(PKIPCR Pcr)
169 {
170     ULONG64 Pat;
171     ULONG FeatureBits;
172 
173     /* Initialize gs */
174     KiInitializeSegments();
175 
176     /* Set GS base */
177     __writemsr(MSR_GS_BASE, (ULONG64)Pcr);
178     __writemsr(MSR_GS_SWAP, (ULONG64)Pcr);
179 
180     /* Detect and set the CPU Type */
181     KiSetProcessorType();
182 
183     /* Get the processor features for this CPU */
184     FeatureBits = KiGetFeatureBits();
185 
186     /* Check if we support all needed features */
187     if ((FeatureBits & REQUIRED_FEATURE_BITS) != REQUIRED_FEATURE_BITS)
188     {
189         /* If not, bugcheck system */
190         FrLdrDbgPrint("CPU doesn't have needed features! Has: 0x%x, required: 0x%x\n",
191                 FeatureBits, REQUIRED_FEATURE_BITS);
192         KeBugCheck(0);
193     }
194 
195     /* Set DEP to always on */
196     SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSON;
197     FeatureBits |= KF_NX_ENABLED;
198 
199     /* Save feature bits */
200     Pcr->Prcb.FeatureBits = FeatureBits;
201 
202     /* Enable fx save restore support */
203     __writecr4(__readcr4() | CR4_FXSR);
204 
205     /* Enable XMMI exceptions */
206     __writecr4(__readcr4() | CR4_XMMEXCPT);
207 
208     /* Enable Write-Protection */
209     __writecr0(__readcr0() | CR0_WP);
210 
211     /* Disable fpu monitoring */
212     __writecr0(__readcr0() & ~CR0_MP);
213 
214     /* Disable x87 fpu exceptions */
215     __writecr0(__readcr0() & ~CR0_NE);
216 
217     /* LDT is unused */
218     __lldt(0);
219 
220     /* Set the systemcall entry points */
221     __writemsr(MSR_LSTAR, (ULONG64)KiSystemCallEntry64);
222     __writemsr(MSR_CSTAR, (ULONG64)KiSystemCallEntry32);
223 
224     __writemsr(MSR_STAR, ((ULONG64)KGDT64_R0_CODE << 32) |
225                          ((ULONG64)(KGDT64_R3_CMCODE|RPL_MASK) << 48));
226 
227     /* Set the flags to be cleared when doing a syscall */
228     __writemsr(MSR_SYSCALL_MASK, EFLAGS_IF_MASK | EFLAGS_TF | EFLAGS_DF);
229 
230     /* Enable syscall instruction and no-execute support */
231     __writemsr(MSR_EFER, __readmsr(MSR_EFER) | MSR_SCE | MSR_NXE);
232 
233     /* Initialize the PAT */
234     Pat = (PAT_WB << 0)  | (PAT_WC << 8) | (PAT_UCM << 16) | (PAT_UC << 24) |
235           (PAT_WB << 32) | (PAT_WC << 40) | (PAT_UCM << 48) | (PAT_UC << 56);
236     __writemsr(MSR_PAT, Pat);
237 }
238 
239 VOID
240 FASTCALL
241 KiInitializeTss(IN PKTSS64 Tss,
242                 IN UINT64 Stack)
243 {
244     PKGDTENTRY64 TssEntry;
245 
246     /* Get pointer to the GDT entry */
247     TssEntry = KiGetGdtEntry(KeGetPcr()->GdtBase, KGDT64_SYS_TSS);
248 
249     /* Initialize the GDT entry */
250     KiInitGdtEntry(TssEntry, (ULONG64)Tss, sizeof(KTSS64), AMD64_TSS, 0);
251 
252     /* Zero out the TSS */
253     RtlZeroMemory(Tss, sizeof(KTSS64));
254 
255     /* FIXME: I/O Map? */
256     Tss->IoMapBase = 0x68;
257 
258     /* Setup ring 0 stack pointer */
259     Tss->Rsp0 = Stack;
260 
261     /* Setup a stack for Double Fault Traps */
262     Tss->Ist[1] = (ULONG64)KiDoubleFaultStack;
263 
264     /* Setup a stack for CheckAbort Traps */
265     Tss->Ist[2] = (ULONG64)KiDoubleFaultStack;
266 
267     /* Setup a stack for NMI Traps */
268     Tss->Ist[3] = (ULONG64)KiDoubleFaultStack;
269 
270     /* Load the task register */
271     __ltr(KGDT64_SYS_TSS);
272 }
273 
274 VOID
275 NTAPI
276 INIT_FUNCTION
277 KiInitializeKernelMachineDependent(
278     IN PKPRCB Prcb,
279     IN PLOADER_PARAMETER_BLOCK LoaderBlock)
280 {
281     /* Set boot-level flags */
282     KeI386CpuType = Prcb->CpuType;
283     KeI386CpuStep = Prcb->CpuStep;
284     KeProcessorArchitecture = PROCESSOR_ARCHITECTURE_AMD64;
285     KeProcessorLevel = (USHORT)Prcb->CpuType;
286     if (Prcb->CpuID)
287         KeProcessorRevision = Prcb->CpuStep;
288 
289     /* Set basic CPU Features that user mode can read */
290     SharedUserData->ProcessorFeatures[PF_COMPARE_EXCHANGE_DOUBLE] = TRUE;
291     SharedUserData->ProcessorFeatures[PF_RDTSC_INSTRUCTION_AVAILABLE] = TRUE;
292     SharedUserData->ProcessorFeatures[PF_PPC_MOVEMEM_64BIT_OK] = TRUE;
293     SharedUserData->ProcessorFeatures[PF_PAE_ENABLED] = TRUE; // ???
294     SharedUserData->ProcessorFeatures[PF_NX_ENABLED] = TRUE;
295     SharedUserData->ProcessorFeatures[PF_FASTFAIL_AVAILABLE] = TRUE;
296     SharedUserData->ProcessorFeatures[PF_XSAVE_ENABLED] = TRUE;
297     SharedUserData->ProcessorFeatures[PF_MMX_INSTRUCTIONS_AVAILABLE] =
298         (Prcb->FeatureBits & KF_MMX) ? TRUE: FALSE;
299     SharedUserData->ProcessorFeatures[PF_XMMI_INSTRUCTIONS_AVAILABLE] =
300         ((Prcb->FeatureBits & KF_FXSR) && (Prcb->FeatureBits & KF_XMMI)) ? TRUE: FALSE;
301     SharedUserData->ProcessorFeatures[PF_XMMI64_INSTRUCTIONS_AVAILABLE] =
302         ((Prcb->FeatureBits & KF_FXSR) && (Prcb->FeatureBits & KF_XMMI64)) ? TRUE: FALSE;
303     SharedUserData->ProcessorFeatures[PF_3DNOW_INSTRUCTIONS_AVAILABLE] =
304         (Prcb->FeatureBits & KF_3DNOW) ? TRUE: FALSE;
305     SharedUserData->ProcessorFeatures[PF_SSE3_INSTRUCTIONS_AVAILABLE] =
306         (Prcb->FeatureBits & KF_SSE3) ? TRUE: FALSE;
307     SharedUserData->ProcessorFeatures[PF_COMPARE_EXCHANGE128] =
308         (Prcb->FeatureBits & KF_CMPXCHG16B) ? TRUE: FALSE;
309 
310     /* Set the default NX policy (opt-in) */
311     SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTIN;
312 
313     /* Check if NPX is always on */
314     if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSON"))
315     {
316         /* Set it always on */
317         SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSON;
318         Prcb->FeatureBits |= KF_NX_ENABLED;
319     }
320     else if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTOUT"))
321     {
322         /* Set it in opt-out mode */
323         SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTOUT;
324         Prcb->FeatureBits |= KF_NX_ENABLED;
325     }
326     else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTIN")) ||
327              (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE")))
328     {
329         /* Set the feature bits */
330         Prcb->FeatureBits |= KF_NX_ENABLED;
331     }
332     else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSOFF")) ||
333              (strstr(KeLoaderBlock->LoadOptions, "EXECUTE")))
334     {
335         /* Set disabled mode */
336         SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSOFF;
337         Prcb->FeatureBits |= KF_NX_DISABLED;
338     }
339 }
340 
341 static LDR_DATA_TABLE_ENTRY LdrCoreEntries[3];
342 
343 void
344 KiInitModuleList(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
345 {
346     PLDR_DATA_TABLE_ENTRY LdrEntry;
347     PLIST_ENTRY Entry;
348     ULONG i;
349 
350     /* Initialize the list head */
351     InitializeListHead(&PsLoadedModuleList);
352 
353     /* Loop the first 3 entries */
354     for (Entry = LoaderBlock->LoadOrderListHead.Flink, i = 0;
355          Entry != &LoaderBlock->LoadOrderListHead && i < 3;
356          Entry = Entry->Flink, i++)
357     {
358         /* Get the data table entry */
359         LdrEntry = CONTAINING_RECORD(Entry,
360                                      LDR_DATA_TABLE_ENTRY,
361                                      InLoadOrderLinks);
362 
363         /* Copy the entry */
364         LdrCoreEntries[i] = *LdrEntry;
365 
366         /* Insert the copy into the list */
367         InsertTailList(&PsLoadedModuleList, &LdrCoreEntries[i].InLoadOrderLinks);
368     }
369 }
370 
371 VOID
372 NTAPI
373 KiSystemStartup(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
374 {
375     CCHAR Cpu;
376     PKTHREAD InitialThread;
377     ULONG64 InitialStack;
378     PKIPCR Pcr;
379 
380     /* HACK */
381     FrLdrDbgPrint = LoaderBlock->u.I386.CommonDataArea;
382     //FrLdrDbgPrint("Hello from KiSystemStartup!!!\n");
383 
384     /* Save the loader block */
385     KeLoaderBlock = LoaderBlock;
386 
387     /* Get the current CPU number */
388     Cpu = KeNumberProcessors++; // FIXME
389 
390     /* LoaderBlock initialization for Cpu 0 */
391     if (Cpu == 0)
392     {
393         /* Set the initial stack, idle thread and process */
394         LoaderBlock->KernelStack = (ULONG_PTR)P0BootStack;
395         LoaderBlock->Thread = (ULONG_PTR)&KiInitialThread;
396         LoaderBlock->Process = (ULONG_PTR)&KiInitialProcess.Pcb;
397         LoaderBlock->Prcb = (ULONG_PTR)&KiInitialPcr.Prcb;
398     }
399 
400     /* Get Pcr from loader block */
401     Pcr = CONTAINING_RECORD(LoaderBlock->Prcb, KIPCR, Prcb);
402 
403     /* Set the PRCB for this Processor */
404     KiProcessorBlock[Cpu] = &Pcr->Prcb;
405 
406     /* Align stack to 16 bytes */
407     LoaderBlock->KernelStack &= ~(16 - 1);
408 
409     /* Save the initial thread and stack */
410     InitialStack = LoaderBlock->KernelStack; // Checkme
411     InitialThread = (PKTHREAD)LoaderBlock->Thread;
412 
413     /* Set us as the current process */
414     InitialThread->ApcState.Process = (PVOID)LoaderBlock->Process;
415 
416     /* Initialize the PCR */
417     KiInitializePcr(Pcr, Cpu, InitialThread, (PVOID)KiDoubleFaultStack);
418 
419     /* Initialize the CPU features */
420     KiInitializeCpu(Pcr);
421 
422     /* Initial setup for the boot CPU */
423     if (Cpu == 0)
424     {
425         /* Initialize the module list (ntos, hal, kdcom) */
426         KiInitModuleList(LoaderBlock);
427 
428         /* Setup the TSS descriptors and entries */
429         KiInitializeTss(Pcr->TssBase, InitialStack);
430 
431         /* Setup the IDT */
432         KeInitExceptions();
433 
434          /* Initialize debugging system */
435         KdInitSystem(0, KeLoaderBlock);
436 
437         /* Check for break-in */
438         if (KdPollBreakIn()) DbgBreakPointWithStatus(DBG_STATUS_CONTROL_C);
439     }
440 
441     DPRINT1("Pcr = %p, Gdt = %p, Idt = %p, Tss = %p\n",
442            Pcr, Pcr->GdtBase, Pcr->IdtBase, Pcr->TssBase);
443 
444     /* Acquire lock */
445     while (InterlockedBitTestAndSet64((PLONG64)&KiFreezeExecutionLock, 0))
446     {
447         /* Loop until lock is free */
448         while ((*(volatile KSPIN_LOCK*)&KiFreezeExecutionLock) & 1);
449     }
450 
451     /* Initialize the Processor with HAL */
452     HalInitializeProcessor(Cpu, KeLoaderBlock);
453 
454     /* Set processor as active */
455     KeActiveProcessors |= 1ULL << Cpu;
456 
457     /* Release lock */
458     InterlockedAnd64((PLONG64)&KiFreezeExecutionLock, 0);
459 
460     /* Raise to HIGH_LEVEL */
461     KfRaiseIrql(HIGH_LEVEL);
462 
463     /* Machine specific kernel initialization */
464     if (Cpu == 0) KiInitializeKernelMachineDependent(&Pcr->Prcb, LoaderBlock);
465 
466     /* Switch to new kernel stack and start kernel bootstrapping */
467     KiSwitchToBootStack(InitialStack & ~3);
468 }
469 
470