xref: /reactos/ntoskrnl/ke/amd64/kiinit.c (revision 8da0f868) 
1 /*
2  * PROJECT:         ReactOS Kernel
3  * LICENSE:         GPL - See COPYING in the top level directory
4  * FILE:            ntoskrnl/ke/amd64/kiinit.c
5  * PURPOSE:         Kernel Initialization for x86 CPUs
6  * PROGRAMMERS:     Alex Ionescu (alex.ionescu@reactos.org)
7  *                  Timo Kreuzer (timo.kreuzer@reactos.org)
8  */
9 
10 /* INCLUDES *****************************************************************/
11 
12 #include <ntoskrnl.h>
13 #define NDEBUG
14 #include <debug.h>
15 
16 #define REQUIRED_FEATURE_BITS (KF_RDTSC|KF_CR4|KF_CMPXCHG8B|KF_XMMI|KF_XMMI64| \
17                                KF_LARGE_PAGE|KF_FAST_SYSCALL|KF_GLOBAL_PAGE| \
18                                KF_CMOV|KF_PAT|KF_MMX|KF_FXSR|KF_NX_BIT|KF_MTRR)
19 
20 /* GLOBALS *******************************************************************/
21 
22 /* Function pointer for early debug prints */
23 ULONG (*FrLdrDbgPrint)(const char *Format, ...);
24 
25 /* Spinlocks used only on X86 */
26 KSPIN_LOCK KiFreezeExecutionLock;
27 
28 
29 KIPCR KiInitialPcr;
30 
31 /* Boot and double-fault/NMI/DPC stack */
32 UCHAR DECLSPEC_ALIGN(16) P0BootStackData[KERNEL_STACK_SIZE] = {0};
33 UCHAR DECLSPEC_ALIGN(16) KiDoubleFaultStackData[KERNEL_STACK_SIZE] = {0};
34 ULONG_PTR P0BootStack = (ULONG_PTR)&P0BootStackData[KERNEL_STACK_SIZE];
35 ULONG_PTR KiDoubleFaultStack = (ULONG_PTR)&KiDoubleFaultStackData[KERNEL_STACK_SIZE];
36 
37 void KiInitializeSegments();
38 void KiSystemCallEntry64();
39 void KiSystemCallEntry32();
40 
41 /* FUNCTIONS *****************************************************************/
42 
43 CODE_SEG("INIT")
44 VOID
45 NTAPI
46 KiInitMachineDependent(VOID)
47 {
48     /* Check for large page support */
49     if (KeFeatureBits & KF_LARGE_PAGE)
50     {
51         /* FIXME: Support this */
52         DPRINT("Large Page support detected but not yet taken advantage of!\n");
53     }
54 
55     /* Check for global page support */
56     if (KeFeatureBits & KF_GLOBAL_PAGE)
57     {
58         /* FIXME: Support this */
59         DPRINT("Global Page support detected but not yet taken advantage of!\n");
60     }
61 
62     /* Check if we have MTRR */
63     if (KeFeatureBits & KF_MTRR)
64     {
65         /* FIXME: Support this */
66         DPRINT("MTRR support detected but not yet taken advantage of!\n");
67     }
68 
69     /* Check for PAT and/or MTRR support */
70     if (KeFeatureBits & KF_PAT)
71     {
72         /* FIXME: Support this */
73         DPRINT("PAT support detected but not yet taken advantage of!\n");
74     }
75 
76 //        /* Allocate the IOPM save area */
77 //        Ki386IopmSaveArea = ExAllocatePoolWithTag(PagedPool,
78 //                                                  IOPM_SIZE,
79 //                                                  '  eK');
80 //        if (!Ki386IopmSaveArea)
81 //        {
82 //            /* Bugcheck. We need this for V86/VDM support. */
83 //            KeBugCheckEx(NO_PAGES_AVAILABLE, 2, IOPM_SIZE, 0, 0);
84 //        }
85 
86 }
87 
88 VOID
89 NTAPI
90 KiInitializePcr(IN PKIPCR Pcr,
91                 IN ULONG ProcessorNumber,
92                 IN PKTHREAD IdleThread,
93                 IN PVOID DpcStack)
94 {
95     KDESCRIPTOR GdtDescriptor = {{0},0,0}, IdtDescriptor = {{0},0,0};
96     PKGDTENTRY64 TssEntry;
97     USHORT Tr = 0;
98 
99     /* Zero out the PCR */
100     RtlZeroMemory(Pcr, sizeof(KIPCR));
101 
102     /* Set pointers to ourselves */
103     Pcr->Self = (PKPCR)Pcr;
104     Pcr->CurrentPrcb = &Pcr->Prcb;
105 
106     /* Set the PCR Version */
107     Pcr->MajorVersion = PCR_MAJOR_VERSION;
108     Pcr->MinorVersion = PCR_MINOR_VERSION;
109 
110     /* Set the PRCB Version */
111     Pcr->Prcb.MajorVersion = 1;
112     Pcr->Prcb.MinorVersion = 1;
113 
114     /* Set the Build Type */
115     Pcr->Prcb.BuildType = 0;
116 #ifndef CONFIG_SMP
117     Pcr->Prcb.BuildType |= PRCB_BUILD_UNIPROCESSOR;
118 #endif
119 #if DBG
120     Pcr->Prcb.BuildType |= PRCB_BUILD_DEBUG;
121 #endif
122 
123     /* Set the Processor Number and current Processor Mask */
124     Pcr->Prcb.Number = (UCHAR)ProcessorNumber;
125     Pcr->Prcb.SetMember = 1ULL << ProcessorNumber;
126 
127     /* Get GDT and IDT descriptors */
128     __sgdt(&GdtDescriptor.Limit);
129     __sidt(&IdtDescriptor.Limit);
130     Pcr->GdtBase = (PVOID)GdtDescriptor.Base;
131     Pcr->IdtBase = (PKIDTENTRY)IdtDescriptor.Base;
132 
133     /* Get TSS Selector */
134     __str(&Tr);
135     ASSERT(Tr == KGDT64_SYS_TSS);
136 
137     /* Get TSS Entry */
138     TssEntry = KiGetGdtEntry(Pcr->GdtBase, Tr);
139 
140     /* Get the KTSS itself */
141     Pcr->TssBase = KiGetGdtDescriptorBase(TssEntry);
142 
143     Pcr->Prcb.RspBase = Pcr->TssBase->Rsp0; // FIXME
144 
145     /* Set DPC Stack */
146     Pcr->Prcb.DpcStack = DpcStack;
147 
148     /* Setup the processor set */
149     Pcr->Prcb.MultiThreadProcessorSet = Pcr->Prcb.SetMember;
150 
151     /* Clear DR6/7 to cleanup bootloader debugging */
152     Pcr->Prcb.ProcessorState.SpecialRegisters.KernelDr6 = 0;
153     Pcr->Prcb.ProcessorState.SpecialRegisters.KernelDr7 = 0;
154 
155     /* Set the Current Thread */
156     Pcr->Prcb.CurrentThread = IdleThread;
157 
158     /* Start us out at PASSIVE_LEVEL */
159     Pcr->Irql = PASSIVE_LEVEL;
160     KeSetCurrentIrql(PASSIVE_LEVEL);
161 }
162 
163 CODE_SEG("INIT")
164 VOID
165 NTAPI
166 KiInitializeCpu(PKIPCR Pcr)
167 {
168     ULONG64 Pat;
169     ULONG FeatureBits;
170 
171     /* Initialize gs */
172     KiInitializeSegments();
173 
174     /* Set GS base */
175     __writemsr(MSR_GS_BASE, (ULONG64)Pcr);
176     __writemsr(MSR_GS_SWAP, (ULONG64)Pcr);
177 
178     /* Detect and set the CPU Type */
179     KiSetProcessorType();
180 
181     /* Get the processor features for this CPU */
182     FeatureBits = KiGetFeatureBits();
183 
184     /* Check if we support all needed features */
185     if ((FeatureBits & REQUIRED_FEATURE_BITS) != REQUIRED_FEATURE_BITS)
186     {
187         /* If not, bugcheck system */
188         FrLdrDbgPrint("CPU doesn't have needed features! Has: 0x%x, required: 0x%x\n",
189                 FeatureBits, REQUIRED_FEATURE_BITS);
190         KeBugCheck(0);
191     }
192 
193     /* Set DEP to always on */
194     SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSON;
195     FeatureBits |= KF_NX_ENABLED;
196 
197     /* Save feature bits */
198     Pcr->Prcb.FeatureBits = FeatureBits;
199 
200     /* Enable fx save restore support */
201     __writecr4(__readcr4() | CR4_FXSR);
202 
203     /* Enable XMMI exceptions */
204     __writecr4(__readcr4() | CR4_XMMEXCPT);
205 
206     /* Enable Write-Protection */
207     __writecr0(__readcr0() | CR0_WP);
208 
209     /* Disable fpu monitoring */
210     __writecr0(__readcr0() & ~CR0_MP);
211 
212     /* Disable x87 fpu exceptions */
213     __writecr0(__readcr0() & ~CR0_NE);
214 
215     /* LDT is unused */
216     __lldt(0);
217 
218     /* Set the systemcall entry points */
219     __writemsr(MSR_LSTAR, (ULONG64)KiSystemCallEntry64);
220     __writemsr(MSR_CSTAR, (ULONG64)KiSystemCallEntry32);
221 
222     __writemsr(MSR_STAR, ((ULONG64)KGDT64_R0_CODE << 32) |
223                          ((ULONG64)(KGDT64_R3_CMCODE|RPL_MASK) << 48));
224 
225     /* Set the flags to be cleared when doing a syscall */
226     __writemsr(MSR_SYSCALL_MASK, EFLAGS_IF_MASK | EFLAGS_TF | EFLAGS_DF);
227 
228     /* Enable syscall instruction and no-execute support */
229     __writemsr(MSR_EFER, __readmsr(MSR_EFER) | MSR_SCE | MSR_NXE);
230 
231     /* Initialize the PAT */
232     Pat = (PAT_WB << 0)  | (PAT_WC << 8) | (PAT_UCM << 16) | (PAT_UC << 24) |
233           (PAT_WB << 32) | (PAT_WC << 40) | (PAT_UCM << 48) | (PAT_UC << 56);
234     __writemsr(MSR_PAT, Pat);
235 }
236 
237 VOID
238 FASTCALL
239 KiInitializeTss(IN PKTSS64 Tss,
240                 IN UINT64 Stack)
241 {
242     PKGDTENTRY64 TssEntry;
243 
244     /* Get pointer to the GDT entry */
245     TssEntry = KiGetGdtEntry(KeGetPcr()->GdtBase, KGDT64_SYS_TSS);
246 
247     /* Initialize the GDT entry */
248     KiInitGdtEntry(TssEntry, (ULONG64)Tss, sizeof(KTSS64), AMD64_TSS, 0);
249 
250     /* Zero out the TSS */
251     RtlZeroMemory(Tss, sizeof(KTSS64));
252 
253     /* FIXME: I/O Map? */
254     Tss->IoMapBase = 0x68;
255 
256     /* Setup ring 0 stack pointer */
257     Tss->Rsp0 = Stack;
258 
259     /* Setup a stack for Double Fault Traps */
260     Tss->Ist[1] = (ULONG64)KiDoubleFaultStack;
261 
262     /* Setup a stack for CheckAbort Traps */
263     Tss->Ist[2] = (ULONG64)KiDoubleFaultStack;
264 
265     /* Setup a stack for NMI Traps */
266     Tss->Ist[3] = (ULONG64)KiDoubleFaultStack;
267 
268     /* Load the task register */
269     __ltr(KGDT64_SYS_TSS);
270 }
271 
272 CODE_SEG("INIT")
273 VOID
274 NTAPI
275 KiInitializeKernelMachineDependent(
276     IN PKPRCB Prcb,
277     IN PLOADER_PARAMETER_BLOCK LoaderBlock)
278 {
279     /* Set boot-level flags */
280     KeI386CpuType = Prcb->CpuType;
281     KeI386CpuStep = Prcb->CpuStep;
282     KeProcessorArchitecture = PROCESSOR_ARCHITECTURE_AMD64;
283     KeProcessorLevel = (USHORT)Prcb->CpuType;
284     if (Prcb->CpuID)
285         KeProcessorRevision = Prcb->CpuStep;
286 
287     /* Set basic CPU Features that user mode can read */
288     SharedUserData->ProcessorFeatures[PF_COMPARE_EXCHANGE_DOUBLE] = TRUE;
289     SharedUserData->ProcessorFeatures[PF_RDTSC_INSTRUCTION_AVAILABLE] = TRUE;
290     SharedUserData->ProcessorFeatures[PF_PPC_MOVEMEM_64BIT_OK] = TRUE;
291     SharedUserData->ProcessorFeatures[PF_PAE_ENABLED] = TRUE; // ???
292     SharedUserData->ProcessorFeatures[PF_NX_ENABLED] = TRUE;
293     SharedUserData->ProcessorFeatures[PF_FASTFAIL_AVAILABLE] = TRUE;
294     SharedUserData->ProcessorFeatures[PF_XSAVE_ENABLED] = TRUE;
295     SharedUserData->ProcessorFeatures[PF_MMX_INSTRUCTIONS_AVAILABLE] =
296         (Prcb->FeatureBits & KF_MMX) ? TRUE: FALSE;
297     SharedUserData->ProcessorFeatures[PF_XMMI_INSTRUCTIONS_AVAILABLE] =
298         ((Prcb->FeatureBits & KF_FXSR) && (Prcb->FeatureBits & KF_XMMI)) ? TRUE: FALSE;
299     SharedUserData->ProcessorFeatures[PF_XMMI64_INSTRUCTIONS_AVAILABLE] =
300         ((Prcb->FeatureBits & KF_FXSR) && (Prcb->FeatureBits & KF_XMMI64)) ? TRUE: FALSE;
301     SharedUserData->ProcessorFeatures[PF_3DNOW_INSTRUCTIONS_AVAILABLE] =
302         (Prcb->FeatureBits & KF_3DNOW) ? TRUE: FALSE;
303     SharedUserData->ProcessorFeatures[PF_SSE3_INSTRUCTIONS_AVAILABLE] =
304         (Prcb->FeatureBits & KF_SSE3) ? TRUE: FALSE;
305     SharedUserData->ProcessorFeatures[PF_COMPARE_EXCHANGE128] =
306         (Prcb->FeatureBits & KF_CMPXCHG16B) ? TRUE: FALSE;
307 
308     /* Set the default NX policy (opt-in) */
309     SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTIN;
310 
311     /* Check if NPX is always on */
312     if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSON"))
313     {
314         /* Set it always on */
315         SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSON;
316         Prcb->FeatureBits |= KF_NX_ENABLED;
317     }
318     else if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTOUT"))
319     {
320         /* Set it in opt-out mode */
321         SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTOUT;
322         Prcb->FeatureBits |= KF_NX_ENABLED;
323     }
324     else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTIN")) ||
325              (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE")))
326     {
327         /* Set the feature bits */
328         Prcb->FeatureBits |= KF_NX_ENABLED;
329     }
330     else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSOFF")) ||
331              (strstr(KeLoaderBlock->LoadOptions, "EXECUTE")))
332     {
333         /* Set disabled mode */
334         SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSOFF;
335         Prcb->FeatureBits |= KF_NX_DISABLED;
336     }
337 }
338 
339 static LDR_DATA_TABLE_ENTRY LdrCoreEntries[3];
340 
341 void
342 KiInitModuleList(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
343 {
344     PLDR_DATA_TABLE_ENTRY LdrEntry;
345     PLIST_ENTRY Entry;
346     ULONG i;
347 
348     /* Initialize the list head */
349     InitializeListHead(&PsLoadedModuleList);
350 
351     /* Loop the first 3 entries */
352     for (Entry = LoaderBlock->LoadOrderListHead.Flink, i = 0;
353          Entry != &LoaderBlock->LoadOrderListHead && i < 3;
354          Entry = Entry->Flink, i++)
355     {
356         /* Get the data table entry */
357         LdrEntry = CONTAINING_RECORD(Entry,
358                                      LDR_DATA_TABLE_ENTRY,
359                                      InLoadOrderLinks);
360 
361         /* Copy the entry */
362         LdrCoreEntries[i] = *LdrEntry;
363 
364         /* Insert the copy into the list */
365         InsertTailList(&PsLoadedModuleList, &LdrCoreEntries[i].InLoadOrderLinks);
366     }
367 }
368 
369 CODE_SEG("INIT")
370 VOID
371 NTAPI
372 KiSystemStartup(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
373 {
374     CCHAR Cpu;
375     PKTHREAD InitialThread;
376     ULONG64 InitialStack;
377     PKIPCR Pcr;
378 
379     /* HACK */
380     FrLdrDbgPrint = LoaderBlock->u.I386.CommonDataArea;
381     //FrLdrDbgPrint("Hello from KiSystemStartup!!!\n");
382 
383     /* Save the loader block */
384     KeLoaderBlock = LoaderBlock;
385 
386     /* Get the current CPU number */
387     Cpu = KeNumberProcessors++; // FIXME
388 
389     /* LoaderBlock initialization for Cpu 0 */
390     if (Cpu == 0)
391     {
392         /* Set the initial stack, idle thread and process */
393         LoaderBlock->KernelStack = (ULONG_PTR)P0BootStack;
394         LoaderBlock->Thread = (ULONG_PTR)&KiInitialThread;
395         LoaderBlock->Process = (ULONG_PTR)&KiInitialProcess.Pcb;
396         LoaderBlock->Prcb = (ULONG_PTR)&KiInitialPcr.Prcb;
397     }
398 
399     /* Get Pcr from loader block */
400     Pcr = CONTAINING_RECORD(LoaderBlock->Prcb, KIPCR, Prcb);
401 
402     /* Set the PRCB for this Processor */
403     KiProcessorBlock[Cpu] = &Pcr->Prcb;
404 
405     /* Align stack to 16 bytes */
406     LoaderBlock->KernelStack &= ~(16 - 1);
407 
408     /* Save the initial thread and stack */
409     InitialStack = LoaderBlock->KernelStack; // Checkme
410     InitialThread = (PKTHREAD)LoaderBlock->Thread;
411 
412     /* Set us as the current process */
413     InitialThread->ApcState.Process = (PVOID)LoaderBlock->Process;
414 
415     /* Initialize the PCR */
416     KiInitializePcr(Pcr, Cpu, InitialThread, (PVOID)KiDoubleFaultStack);
417 
418     /* Initialize the CPU features */
419     KiInitializeCpu(Pcr);
420 
421     /* Initial setup for the boot CPU */
422     if (Cpu == 0)
423     {
424         /* Initialize the module list (ntos, hal, kdcom) */
425         KiInitModuleList(LoaderBlock);
426 
427         /* Setup the TSS descriptors and entries */
428         KiInitializeTss(Pcr->TssBase, InitialStack);
429 
430         /* Setup the IDT */
431         KeInitExceptions();
432 
433          /* Initialize debugging system */
434         KdInitSystem(0, KeLoaderBlock);
435 
436         /* Check for break-in */
437         if (KdPollBreakIn()) DbgBreakPointWithStatus(DBG_STATUS_CONTROL_C);
438     }
439 
440     DPRINT1("Pcr = %p, Gdt = %p, Idt = %p, Tss = %p\n",
441            Pcr, Pcr->GdtBase, Pcr->IdtBase, Pcr->TssBase);
442 
443     /* Acquire lock */
444     while (InterlockedBitTestAndSet64((PLONG64)&KiFreezeExecutionLock, 0))
445     {
446         /* Loop until lock is free */
447         while ((*(volatile KSPIN_LOCK*)&KiFreezeExecutionLock) & 1);
448     }
449 
450     /* Initialize the Processor with HAL */
451     HalInitializeProcessor(Cpu, KeLoaderBlock);
452 
453     /* Set processor as active */
454     KeActiveProcessors |= 1ULL << Cpu;
455 
456     /* Release lock */
457     InterlockedAnd64((PLONG64)&KiFreezeExecutionLock, 0);
458 
459     /* Raise to HIGH_LEVEL */
460     KfRaiseIrql(HIGH_LEVEL);
461 
462     /* Machine specific kernel initialization */
463     if (Cpu == 0) KiInitializeKernelMachineDependent(&Pcr->Prcb, LoaderBlock);
464 
465     /* Switch to new kernel stack and start kernel bootstrapping */
466     KiSwitchToBootStack(InitialStack & ~3);
467 }
468 
469