1 /*
2 * COPYRIGHT: GPL, See COPYING in the top level directory
3 * PROJECT: ReactOS kernel
4 * FILE: ntoskrnl/mm/amd64/init.c
5 * PURPOSE: Memory Manager Initialization for amd64
6 *
7 * PROGRAMMERS: Timo kreuzer (timo.kreuzer@reactos.org)
8 * ReactOS Portable Systems Group
9 */
10
11 /* INCLUDES ***************************************************************/
12
13 #include <ntoskrnl.h>
14 //#define NDEBUG
15 #include <debug.h>
16
17 #include <mm/ARM3/miarm.h>
18 #include <fltkernel.h>
19
20 extern PMMPTE MmDebugPte;
21
22 /* Helper macros */
23 #define IS_PAGE_ALIGNED(addr) IS_ALIGNED(addr, PAGE_SIZE)
24
25 /* GLOBALS *****************************************************************/
26
27 /* Template PTE and PDE for a kernel page */
28 MMPTE ValidKernelPde = {{PTE_VALID|PTE_EXECUTE_READWRITE|PTE_DIRTY|PTE_ACCESSED}};
29 MMPTE ValidKernelPte = {{PTE_VALID|PTE_EXECUTE_READWRITE|PTE_DIRTY|PTE_ACCESSED}};
30
31 /* The same, but for local pages */
32 MMPTE ValidKernelPdeLocal = {{PTE_VALID|PTE_EXECUTE_READWRITE|PTE_DIRTY|PTE_ACCESSED}};
33 MMPTE ValidKernelPteLocal = {{PTE_VALID|PTE_EXECUTE_READWRITE|PTE_DIRTY|PTE_ACCESSED}};
34
35 /* Template PDE for a demand-zero page */
36 MMPDE DemandZeroPde = {{MM_EXECUTE_READWRITE << MM_PTE_SOFTWARE_PROTECTION_BITS}};
37 MMPTE DemandZeroPte = {{MM_READWRITE << MM_PTE_SOFTWARE_PROTECTION_BITS}};
38
39 /* Template PTE for prototype page */
40 MMPTE PrototypePte = {{(MM_EXECUTE_READWRITE << MM_PTE_SOFTWARE_PROTECTION_BITS) |
41 PTE_PROTOTYPE | (MI_PTE_LOOKUP_NEEDED << 32)}};
42
43 /* Template PTE for decommited page */
44 MMPTE MmDecommittedPte = {{MM_DECOMMIT << MM_PTE_SOFTWARE_PROTECTION_BITS}};
45
46 /* Address ranges */
47 PVOID MiSessionViewEnd;
48 PVOID MiSystemPteSpaceStart;
49 PVOID MiSystemPteSpaceEnd;
50
51 ULONG64 MxPfnSizeInBytes;
52 BOOLEAN MiIncludeType[LoaderMaximum];
53 PFN_NUMBER MxFreePageBase;
54 ULONG64 MxFreePageCount = 0;
55
56 BOOLEAN MiPfnsInitialized = FALSE;
57
58 /* FUNCTIONS *****************************************************************/
59
60 CODE_SEG("INIT")
61 VOID
62 NTAPI
MiInitializeSessionSpaceLayout(VOID)63 MiInitializeSessionSpaceLayout(VOID)
64 {
65 /* This is the entire size */
66 MmSessionSize = MI_SESSION_SIZE;
67
68 /* Start with session space end */
69 MiSessionSpaceEnd = (PVOID)MI_SESSION_SPACE_END;
70
71 /* The highest range is the session image range */
72 MmSessionImageSize = MI_SESSION_IMAGE_SIZE;
73 MiSessionImageEnd = MiSessionSpaceEnd;
74 MiSessionImageStart = (PUCHAR)MiSessionImageEnd - MmSessionImageSize;
75 ASSERT(IS_PAGE_ALIGNED(MiSessionImageStart));
76
77 /* Session working set is below the session image range */
78 MiSessionSpaceWs = (PUCHAR)MiSessionImageStart - MI_SESSION_WORKING_SET_SIZE;
79
80 /* Session view is below the session working set */
81 MmSessionViewSize = MI_SESSION_VIEW_SIZE;
82 MiSessionViewEnd = MiSessionSpaceWs;
83 MiSessionViewStart = (PUCHAR)MiSessionViewEnd - MmSessionViewSize;
84 ASSERT(IS_PAGE_ALIGNED(MiSessionViewStart));
85
86 /* Session pool is below session view */
87 MmSessionPoolSize = MI_SESSION_POOL_SIZE;
88 MiSessionPoolEnd = MiSessionViewStart;
89 MiSessionPoolStart = (PUCHAR)MiSessionPoolEnd - MmSessionPoolSize;
90 ASSERT(IS_PAGE_ALIGNED(MiSessionPoolStart));
91
92 /* And it all begins here */
93 MmSessionBase = MiSessionPoolStart;
94
95 /* System view space ends at session space, so now that we know where
96 * this is, we can compute the base address of system view space itself. */
97 MmSystemViewSize = MI_SYSTEM_VIEW_SIZE;
98 MiSystemViewStart = (PUCHAR)MmSessionBase - MmSystemViewSize;
99 ASSERT(IS_PAGE_ALIGNED(MiSystemViewStart));
100
101 /* Sanity checks */
102 ASSERT(Add2Ptr(MmSessionBase, MmSessionSize) == MiSessionSpaceEnd);
103 ASSERT(MiSessionViewEnd <= MiSessionImageStart);
104 ASSERT(MmSessionBase <= MiSessionPoolStart);
105
106 /* Compute the PTE addresses for all the addresses we carved out */
107 MiSessionImagePteStart = MiAddressToPte(MiSessionImageStart);
108 MiSessionImagePteEnd = MiAddressToPte(MiSessionImageEnd);
109 MiSessionBasePte = MiAddressToPte(MmSessionBase);
110 MiSessionLastPte = MiAddressToPte(MiSessionSpaceEnd);
111
112 /* Initialize the pointer to the session space structure */
113 MmSessionSpace = (PMM_SESSION_SPACE)Add2Ptr(MiSessionImageStart, 0x10000);
114 }
115
116 VOID
117 NTAPI
MiMapPPEs(PVOID StartAddress,PVOID EndAddress)118 MiMapPPEs(
119 PVOID StartAddress,
120 PVOID EndAddress)
121 {
122 PMMPDE PointerPpe;
123 MMPDE TmplPde = ValidKernelPde;
124
125 /* Loop the PPEs */
126 for (PointerPpe = MiAddressToPpe(StartAddress);
127 PointerPpe <= MiAddressToPpe(EndAddress);
128 PointerPpe++)
129 {
130 /* Check if its already mapped */
131 if (!PointerPpe->u.Hard.Valid)
132 {
133 /* No, map it! */
134 TmplPde.u.Hard.PageFrameNumber = MxGetNextPage(1);
135 MI_WRITE_VALID_PTE(PointerPpe, TmplPde);
136
137 /* Zero out the page table */
138 RtlZeroMemory(MiPteToAddress(PointerPpe), PAGE_SIZE);
139 }
140 }
141 }
142
143 VOID
144 NTAPI
MiMapPDEs(PVOID StartAddress,PVOID EndAddress)145 MiMapPDEs(
146 PVOID StartAddress,
147 PVOID EndAddress)
148 {
149 PMMPDE PointerPde;
150 MMPDE TmplPde = ValidKernelPde;
151
152 /* Loop the PDEs */
153 for (PointerPde = MiAddressToPde(StartAddress);
154 PointerPde <= MiAddressToPde(EndAddress);
155 PointerPde++)
156 {
157 /* Check if its already mapped */
158 if (!PointerPde->u.Hard.Valid)
159 {
160 /* No, map it! */
161 TmplPde.u.Hard.PageFrameNumber = MxGetNextPage(1);
162 MI_WRITE_VALID_PTE(PointerPde, TmplPde);
163
164 /* Zero out the page table */
165 RtlZeroMemory(MiPteToAddress(PointerPde), PAGE_SIZE);
166 }
167 }
168 }
169
170 VOID
171 NTAPI
MiMapPTEs(PVOID StartAddress,PVOID EndAddress)172 MiMapPTEs(
173 PVOID StartAddress,
174 PVOID EndAddress)
175 {
176 PMMPTE PointerPte;
177 MMPTE TmplPte = ValidKernelPte;
178
179 /* Loop the PTEs */
180 for (PointerPte = MiAddressToPte(StartAddress);
181 PointerPte <= MiAddressToPte(EndAddress);
182 PointerPte++)
183 {
184 /* Check if its already mapped */
185 if (!PointerPte->u.Hard.Valid)
186 {
187 /* No, map it! */
188 TmplPte.u.Hard.PageFrameNumber = MxGetNextPage(1);
189 MI_WRITE_VALID_PTE(PointerPte, TmplPte);
190
191 /* Zero out the page (FIXME: not always neccessary) */
192 RtlZeroMemory(MiPteToAddress(PointerPte), PAGE_SIZE);
193 }
194 }
195 }
196
197 CODE_SEG("INIT")
198 VOID
199 NTAPI
MiInitializePageTable(VOID)200 MiInitializePageTable(VOID)
201 {
202 ULONG64 PxePhysicalAddress;
203 MMPTE TmplPte, *PointerPxe;
204 PFN_NUMBER PxePfn;
205
206 /* Get current directory base */
207 PxePfn = ((PMMPTE)PXE_SELFMAP)->u.Hard.PageFrameNumber;
208 PxePhysicalAddress = PxePfn << PAGE_SHIFT;
209 ASSERT(PxePhysicalAddress == __readcr3());
210
211 /* Set directory base for the system process */
212 PsGetCurrentProcess()->Pcb.DirectoryTableBase[0] = PxePhysicalAddress;
213
214 /* Enable global pages */
215 __writecr4(__readcr4() | CR4_PGE);
216 ASSERT(__readcr4() & CR4_PGE);
217
218 /* Loop the user mode PXEs */
219 for (PointerPxe = MiAddressToPxe(0);
220 PointerPxe <= MiAddressToPxe(MmHighestUserAddress);
221 PointerPxe++)
222 {
223 /* Zero the PXE, clear all mappings */
224 PointerPxe->u.Long = 0;
225 }
226
227 /* Flush the TLB */
228 KeFlushCurrentTb();
229
230 /* Set up a template PTE */
231 TmplPte.u.Long = 0;
232 TmplPte.u.Flush.Valid = 1;
233 TmplPte.u.Flush.Write = 1;
234 HyperTemplatePte = TmplPte;
235
236 /* Create PDPTs (72 KB) for shared system address space,
237 * skip page tables TODO: use global pages. */
238
239 /* Loop the PXEs */
240 for (PointerPxe = MiAddressToPxe((PVOID)HYPER_SPACE);
241 PointerPxe <= MiAddressToPxe(MI_HIGHEST_SYSTEM_ADDRESS);
242 PointerPxe++)
243 {
244 /* Is the PXE already valid? */
245 if (!PointerPxe->u.Hard.Valid)
246 {
247 /* It's not Initialize it */
248 TmplPte.u.Flush.PageFrameNumber = MxGetNextPage(1);
249 *PointerPxe = TmplPte;
250
251 /* Zero the page. The PXE is the PTE for the PDPT. */
252 RtlZeroMemory(MiPteToAddress(PointerPxe), PAGE_SIZE);
253 }
254 }
255 PxePfn = PFN_FROM_PXE(MiAddressToPxe((PVOID)HYPER_SPACE));
256 PsGetCurrentProcess()->Pcb.DirectoryTableBase[1] = PxePfn << PAGE_SHIFT;
257
258 /* Map PPEs for paged pool */
259 MiMapPPEs(MmPagedPoolStart, MmPagedPoolEnd);
260
261 /* Setup 1 PPE for hyper space */
262 MiMapPPEs((PVOID)HYPER_SPACE, (PVOID)HYPER_SPACE_END);
263
264 /* Setup PPEs for system space view */
265 MiMapPPEs(MiSystemViewStart, (PCHAR)MiSystemViewStart + MmSystemViewSize);
266
267 /* Setup the mapping PDEs */
268 MiMapPDEs((PVOID)MI_MAPPING_RANGE_START, (PVOID)MI_MAPPING_RANGE_END);
269
270 /* Setup the mapping PTEs */
271 MmFirstReservedMappingPte = MiAddressToPte((PVOID)MI_MAPPING_RANGE_START);
272 MmLastReservedMappingPte = MiAddressToPte((PVOID)MI_MAPPING_RANGE_END);
273 MmFirstReservedMappingPte->u.Hard.PageFrameNumber = MI_HYPERSPACE_PTES;
274
275 /* Setup debug mapping PTE */
276 MiMapPPEs((PVOID)MI_DEBUG_MAPPING, (PVOID)MI_DEBUG_MAPPING);
277 MiMapPDEs((PVOID)MI_DEBUG_MAPPING, (PVOID)MI_DEBUG_MAPPING);
278 MmDebugPte = MiAddressToPte((PVOID)MI_DEBUG_MAPPING);
279
280 /* Setup PDE and PTEs for VAD bitmap and working set list */
281 MiMapPDEs((PVOID)MI_VAD_BITMAP, (PVOID)(MI_WORKING_SET_LIST + PAGE_SIZE - 1));
282 MiMapPTEs((PVOID)MI_VAD_BITMAP, (PVOID)(MI_WORKING_SET_LIST + PAGE_SIZE - 1));
283 }
284
285 CODE_SEG("INIT")
286 VOID
287 NTAPI
MiBuildNonPagedPool(VOID)288 MiBuildNonPagedPool(VOID)
289 {
290 /* Check if this is a machine with less than 256MB of RAM, and no overide */
291 if ((MmNumberOfPhysicalPages <= MI_MIN_PAGES_FOR_NONPAGED_POOL_TUNING) &&
292 !(MmSizeOfNonPagedPoolInBytes))
293 {
294 /* Force the non paged pool to be 2MB so we can reduce RAM usage */
295 MmSizeOfNonPagedPoolInBytes = 2 * 1024 * 1024;
296 }
297
298 /* Check if the user gave a ridicuously large nonpaged pool RAM size */
299 if ((MmSizeOfNonPagedPoolInBytes >> PAGE_SHIFT) >
300 (MmNumberOfPhysicalPages * 7 / 8))
301 {
302 /* More than 7/8ths of RAM was dedicated to nonpaged pool, ignore! */
303 MmSizeOfNonPagedPoolInBytes = 0;
304 }
305
306 /* Check if no registry setting was set, or if the setting was too low */
307 if (MmSizeOfNonPagedPoolInBytes < MmMinimumNonPagedPoolSize)
308 {
309 /* Start with the minimum (256 KB) and add 32 KB for each MB above 4 */
310 MmSizeOfNonPagedPoolInBytes = MmMinimumNonPagedPoolSize;
311 MmSizeOfNonPagedPoolInBytes += (MmNumberOfPhysicalPages - 1024) /
312 256 * MmMinAdditionNonPagedPoolPerMb;
313 }
314
315 /* Check if the registy setting or our dynamic calculation was too high */
316 if (MmSizeOfNonPagedPoolInBytes > MI_MAX_INIT_NONPAGED_POOL_SIZE)
317 {
318 /* Set it to the maximum */
319 MmSizeOfNonPagedPoolInBytes = MI_MAX_INIT_NONPAGED_POOL_SIZE;
320 }
321
322 /* Check if a percentage cap was set through the registry */
323 if (MmMaximumNonPagedPoolPercent)
324 {
325 /* Don't feel like supporting this right now */
326 UNIMPLEMENTED;
327 }
328
329 /* Page-align the nonpaged pool size */
330 MmSizeOfNonPagedPoolInBytes &= ~(PAGE_SIZE - 1);
331
332 /* Now, check if there was a registry size for the maximum size */
333 if (!MmMaximumNonPagedPoolInBytes)
334 {
335 /* Start with the default (1MB) and add 400 KB for each MB above 4 */
336 MmMaximumNonPagedPoolInBytes = MmDefaultMaximumNonPagedPool;
337 MmMaximumNonPagedPoolInBytes += (MmNumberOfPhysicalPages - 1024) /
338 256 * MmMaxAdditionNonPagedPoolPerMb;
339 }
340
341 /* Don't let the maximum go too high */
342 if (MmMaximumNonPagedPoolInBytes > MI_MAX_NONPAGED_POOL_SIZE)
343 {
344 /* Set it to the upper limit */
345 MmMaximumNonPagedPoolInBytes = MI_MAX_NONPAGED_POOL_SIZE;
346 }
347
348 /* Convert nonpaged pool size from bytes to pages */
349 MmMaximumNonPagedPoolInPages = MmMaximumNonPagedPoolInBytes >> PAGE_SHIFT;
350
351 /* Non paged pool starts after the PFN database */
352 MmNonPagedPoolStart = MmPfnDatabase + MxPfnAllocation * PAGE_SIZE;
353
354 /* Calculate the nonpaged pool expansion start region */
355 MmNonPagedPoolExpansionStart = (PCHAR)MmNonPagedPoolStart +
356 MmSizeOfNonPagedPoolInBytes;
357 ASSERT(IS_PAGE_ALIGNED(MmNonPagedPoolExpansionStart));
358
359 /* And this is where the none paged pool ends */
360 MmNonPagedPoolEnd = (PCHAR)MmNonPagedPoolStart + MmMaximumNonPagedPoolInBytes;
361 ASSERT(MmNonPagedPoolEnd < (PVOID)MM_HAL_VA_START);
362
363 /* Map PPEs and PDEs for non paged pool (including expansion) */
364 MiMapPPEs(MmNonPagedPoolStart, MmNonPagedPoolEnd);
365 MiMapPDEs(MmNonPagedPoolStart, MmNonPagedPoolEnd);
366
367 /* Map the nonpaged pool PTEs (without expansion) */
368 MiMapPTEs(MmNonPagedPoolStart, (PCHAR)MmNonPagedPoolExpansionStart - 1);
369
370 /* Initialize the ARM3 nonpaged pool */
371 MiInitializeNonPagedPool();
372 MiInitializeNonPagedPoolThresholds();
373
374 }
375
376 CODE_SEG("INIT")
377 VOID
378 NTAPI
MiBuildSystemPteSpace(VOID)379 MiBuildSystemPteSpace(VOID)
380 {
381 PMMPTE PointerPte;
382 SIZE_T NonPagedSystemSize;
383
384 /* Use the default number of system PTEs */
385 MmNumberOfSystemPtes = MI_NUMBER_SYSTEM_PTES;
386 NonPagedSystemSize = (MmNumberOfSystemPtes + 1) * PAGE_SIZE;
387
388 /* Put system PTEs at the start of the system VA space */
389 MiSystemPteSpaceStart = MmNonPagedSystemStart;
390 MiSystemPteSpaceEnd = (PUCHAR)MiSystemPteSpaceStart + NonPagedSystemSize;
391
392 /* Map the PPEs and PDEs for the system PTEs */
393 MiMapPPEs(MiSystemPteSpaceStart, MiSystemPteSpaceEnd);
394 MiMapPDEs(MiSystemPteSpaceStart, MiSystemPteSpaceEnd);
395
396 /* Initialize the system PTE space */
397 PointerPte = MiAddressToPte(MiSystemPteSpaceStart);
398 MiInitializeSystemPtes(PointerPte, MmNumberOfSystemPtes, SystemPteSpace);
399
400 /* Reserve system PTEs for zeroing PTEs and clear them */
401 MiFirstReservedZeroingPte = MiReserveSystemPtes(MI_ZERO_PTES + 1,
402 SystemPteSpace);
403 RtlZeroMemory(MiFirstReservedZeroingPte, (MI_ZERO_PTES + 1) * sizeof(MMPTE));
404
405 /* Set the counter to maximum */
406 MiFirstReservedZeroingPte->u.Hard.PageFrameNumber = MI_ZERO_PTES;
407 }
408
409 static
410 VOID
MiSetupPfnForPageTable(PFN_NUMBER PageFrameIndex,PMMPTE PointerPte)411 MiSetupPfnForPageTable(
412 PFN_NUMBER PageFrameIndex,
413 PMMPTE PointerPte)
414 {
415 PMMPFN Pfn;
416 PMMPDE PointerPde;
417
418 /* Get the pfn entry for this page */
419 Pfn = MiGetPfnEntry(PageFrameIndex);
420
421 /* Check if it's valid memory */
422 if ((PageFrameIndex <= MmHighestPhysicalPage) &&
423 (MmIsAddressValid(Pfn)) &&
424 (Pfn->u3.e1.PageLocation == ActiveAndValid))
425 {
426 /* Setup the PFN entry */
427 Pfn->u1.WsIndex = 0;
428 Pfn->u2.ShareCount++;
429 Pfn->PteAddress = PointerPte;
430 Pfn->OriginalPte = *PointerPte;
431 Pfn->u3.e1.PageLocation = ActiveAndValid;
432 Pfn->u3.e1.CacheAttribute = MiNonCached;
433 Pfn->u3.e2.ReferenceCount = 1;
434 Pfn->u4.PteFrame = PFN_FROM_PTE(MiAddressToPte(PointerPte));
435 }
436
437 /* Increase the shared count of the PFN entry for the PDE */
438 PointerPde = MiAddressToPde(MiPteToAddress(PointerPte));
439 Pfn = MiGetPfnEntry(PFN_FROM_PTE(PointerPde));
440 Pfn->u2.ShareCount++;
441 }
442
443 CODE_SEG("INIT")
444 static
445 VOID
MiBuildPfnDatabaseFromPageTables(VOID)446 MiBuildPfnDatabaseFromPageTables(VOID)
447 {
448 PVOID Address = NULL;
449 PFN_NUMBER PageFrameIndex;
450 PMMPDE PointerPde;
451 PMMPTE PointerPte;
452 ULONG k, l;
453 PMMPFN Pfn;
454 #if (_MI_PAGING_LEVELS >= 3)
455 PMMPDE PointerPpe;
456 ULONG j;
457 #endif
458 #if (_MI_PAGING_LEVELS == 4)
459 PMMPDE PointerPxe;
460 ULONG i;
461 #endif
462
463 /* Manual setup of the top level page directory */
464 #if (_MI_PAGING_LEVELS == 4)
465 PageFrameIndex = PFN_FROM_PTE(MiAddressToPte(PXE_BASE));
466 #elif (_MI_PAGING_LEVELS == 3)
467 PageFrameIndex = PFN_FROM_PTE(MiAddressToPte(PPE_BASE));
468 #else
469 PageFrameIndex = PFN_FROM_PTE(MiAddressToPte(PDE_BASE));
470 #endif
471 Pfn = MiGetPfnEntry(PageFrameIndex);
472 ASSERT(Pfn->u3.e1.PageLocation == ActiveAndValid);
473 Pfn->u1.WsIndex = 0;
474 Pfn->u2.ShareCount = 1;
475 Pfn->PteAddress = NULL;
476 Pfn->u3.e1.CacheAttribute = MiNonCached;
477 Pfn->u3.e2.ReferenceCount = 1;
478 Pfn->u4.PteFrame = 0;
479
480 #if (_MI_PAGING_LEVELS == 4)
481 /* Loop all PXEs in the PML4 */
482 PointerPxe = MiAddressToPxe(Address);
483 for (i = 0; i < PXE_PER_PAGE; i++, PointerPxe++)
484 {
485 /* Skip invalid PXEs */
486 if (!PointerPxe->u.Hard.Valid) continue;
487
488 /* Handle the PFN */
489 PageFrameIndex = PFN_FROM_PXE(PointerPxe);
490 MiSetupPfnForPageTable(PageFrameIndex, PointerPxe);
491
492 /* Get starting VA for this PXE */
493 Address = MiPxeToAddress(PointerPxe);
494 #endif
495 #if (_MI_PAGING_LEVELS >= 3)
496 /* Loop all PPEs in this PDP */
497 PointerPpe = MiAddressToPpe(Address);
498 for (j = 0; j < PPE_PER_PAGE; j++, PointerPpe++)
499 {
500 /* Skip invalid PPEs */
501 if (!PointerPpe->u.Hard.Valid) continue;
502
503 /* Handle the PFN */
504 PageFrameIndex = PFN_FROM_PPE(PointerPpe);
505 MiSetupPfnForPageTable(PageFrameIndex, PointerPpe);
506
507 /* Get starting VA for this PPE */
508 Address = MiPpeToAddress(PointerPpe);
509 #endif
510 /* Loop all PDEs in this PD */
511 PointerPde = MiAddressToPde(Address);
512 for (k = 0; k < PDE_PER_PAGE; k++, PointerPde++)
513 {
514 /* Skip invalid PDEs */
515 if (!PointerPde->u.Hard.Valid) continue;
516
517 /* Handle the PFN */
518 PageFrameIndex = PFN_FROM_PDE(PointerPde);
519 MiSetupPfnForPageTable(PageFrameIndex, PointerPde);
520
521 /* Get starting VA for this PDE */
522 Address = MiPdeToAddress(PointerPde);
523
524 /* Loop all PTEs in this PT */
525 PointerPte = MiAddressToPte(Address);
526 for (l = 0; l < PTE_PER_PAGE; l++, PointerPte++)
527 {
528 /* Skip invalid PTEs */
529 if (!PointerPte->u.Hard.Valid) continue;
530
531 /* Handle the PFN */
532 PageFrameIndex = PFN_FROM_PTE(PointerPte);
533 MiSetupPfnForPageTable(PageFrameIndex, PointerPte);
534 }
535 }
536 #if (_MI_PAGING_LEVELS >= 3)
537 }
538 #endif
539 #if (_MI_PAGING_LEVELS == 4)
540 }
541 #endif
542 }
543
544 CODE_SEG("INIT")
545 static
546 VOID
MiAddDescriptorToDatabase(PFN_NUMBER BasePage,PFN_NUMBER PageCount,TYPE_OF_MEMORY MemoryType)547 MiAddDescriptorToDatabase(
548 PFN_NUMBER BasePage,
549 PFN_NUMBER PageCount,
550 TYPE_OF_MEMORY MemoryType)
551 {
552 PMMPFN Pfn;
553
554 ASSERT(!MiIsMemoryTypeInvisible(MemoryType));
555
556 /* Check if the memory is free */
557 if (MiIsMemoryTypeFree(MemoryType))
558 {
559 /* Get the last pfn of this descriptor. Note we loop backwards */
560 Pfn = &MmPfnDatabase[BasePage + PageCount - 1];
561
562 /* Loop all pages */
563 while (PageCount--)
564 {
565 /* Add it to the free list */
566 Pfn->u3.e1.CacheAttribute = MiNonCached; // FIXME: Windows ASSERTs MiChached, but why not MiNotMapped?
567 MiInsertPageInFreeList(BasePage + PageCount);
568
569 /* Go to the previous page */
570 Pfn--;
571 }
572 }
573 else if (MemoryType == LoaderXIPRom)
574 {
575 Pfn = &MmPfnDatabase[BasePage];
576 while (PageCount--)
577 {
578 /* Make it a pseudo-I/O ROM mapping */
579 Pfn->PteAddress = 0;
580 Pfn->u1.Flink = 0;
581 Pfn->u2.ShareCount = 0;
582 Pfn->u3.e1.PageLocation = 0;
583 Pfn->u3.e1.CacheAttribute = MiNonCached;
584 Pfn->u3.e1.Rom = 1;
585 Pfn->u3.e1.PrototypePte = 1;
586 Pfn->u3.e2.ReferenceCount = 0;
587 Pfn->u4.InPageError = 0;
588 Pfn->u4.PteFrame = 0;
589
590 /* Advance one */
591 Pfn++;
592 }
593 }
594 else if (MemoryType == LoaderBad)
595 {
596 // FIXME: later
597 ASSERT(FALSE);
598 }
599 else
600 {
601 /* For now skip it */
602 Pfn = &MmPfnDatabase[BasePage];
603 while (PageCount--)
604 {
605 /* Make an active PFN */
606 Pfn->u3.e1.PageLocation = ActiveAndValid;
607
608 /* Advance one */
609 Pfn++;
610 }
611 }
612 }
613
614 CODE_SEG("INIT")
615 VOID
616 NTAPI
MiBuildPfnDatabase(IN PLOADER_PARAMETER_BLOCK LoaderBlock)617 MiBuildPfnDatabase(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
618 {
619 PLIST_ENTRY ListEntry;
620 PMEMORY_ALLOCATION_DESCRIPTOR Descriptor;
621 PFN_NUMBER BasePage, PageCount;
622 KIRQL OldIrql;
623
624 /* Lock the PFN Database */
625 OldIrql = MiAcquirePfnLock();
626
627 /* Map the PDEs and PPEs for the pfn database (ignore holes) */
628 #if (_MI_PAGING_LEVELS >= 3)
629 MiMapPPEs(MmPfnDatabase, (PUCHAR)MmPfnDatabase + (MxPfnAllocation * PAGE_SIZE) - 1);
630 #endif
631 MiMapPDEs(MmPfnDatabase, (PUCHAR)MmPfnDatabase + (MxPfnAllocation * PAGE_SIZE) - 1);
632
633 /* First initialize the color tables */
634 MiInitializeColorTables();
635
636 /* Loop the memory descriptors */
637 for (ListEntry = LoaderBlock->MemoryDescriptorListHead.Flink;
638 ListEntry != &LoaderBlock->MemoryDescriptorListHead;
639 ListEntry = ListEntry->Flink)
640 {
641 /* Get the descriptor */
642 Descriptor = CONTAINING_RECORD(ListEntry,
643 MEMORY_ALLOCATION_DESCRIPTOR,
644 ListEntry);
645
646 /* Skip invisible memory */
647 if (MiIsMemoryTypeInvisible(Descriptor->MemoryType)) continue;
648
649 /* If this is the free descriptor, use the copy instead */
650 if (Descriptor == MxFreeDescriptor) Descriptor = &MxOldFreeDescriptor;
651
652 /* Get the range for this descriptor */
653 BasePage = Descriptor->BasePage;
654 PageCount = Descriptor->PageCount;
655
656 /* Map the pages for the database */
657 MiMapPTEs(&MmPfnDatabase[BasePage],
658 (PUCHAR)(&MmPfnDatabase[BasePage + PageCount]) - 1);
659
660 /* If this was the free descriptor, skip the next step */
661 if (Descriptor == &MxOldFreeDescriptor) continue;
662
663 /* Add this descriptor to the database */
664 MiAddDescriptorToDatabase(BasePage, PageCount, Descriptor->MemoryType);
665 }
666
667 /* At this point the whole pfn database is mapped. We are about to add the
668 pages from the free descriptor to the database, so from now on we cannot
669 use it anymore. */
670
671 /* Now add the free descriptor */
672 BasePage = MxFreeDescriptor->BasePage;
673 PageCount = MxFreeDescriptor->PageCount;
674 MiAddDescriptorToDatabase(BasePage, PageCount, LoaderFree);
675
676 /* And finally the memory we used */
677 BasePage = MxOldFreeDescriptor.BasePage;
678 PageCount = MxFreeDescriptor->BasePage - BasePage;
679 MiAddDescriptorToDatabase(BasePage, PageCount, LoaderMemoryData);
680
681 /* Reset the descriptor back so we can create the correct memory blocks */
682 *MxFreeDescriptor = MxOldFreeDescriptor;
683
684 /* Now process the page tables */
685 MiBuildPfnDatabaseFromPageTables();
686
687 /* PFNs are initialized now! */
688 MiPfnsInitialized = TRUE;
689
690 /* Release PFN database */
691 MiReleasePfnLock(OldIrql);
692 }
693
694 CODE_SEG("INIT")
695 NTSTATUS
696 NTAPI
MiInitMachineDependent(IN PLOADER_PARAMETER_BLOCK LoaderBlock)697 MiInitMachineDependent(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
698 {
699 NTSTATUS Status;
700 ULONG Flags;
701
702 ASSERT(MxPfnAllocation != 0);
703
704 /* Set some hardcoded addresses */
705 MmHyperSpaceEnd = (PVOID)HYPER_SPACE_END;
706 MmNonPagedSystemStart = (PVOID)MM_SYSTEM_SPACE_START;
707 MmPfnDatabase = (PVOID)MI_PFN_DATABASE;
708 MmWorkingSetList = (PVOID)MI_WORKING_SET_LIST;
709
710
711 // PrototypePte.u.Proto.Valid = 1
712 // PrototypePte.u.ReadOnly
713 // PrototypePte.u.Prototype
714 // PrototypePte.u.Protection = MM_READWRITE;
715 // PrototypePte.u.ProtoAddress
716 PrototypePte.u.Soft.PageFileHigh = MI_PTE_LOOKUP_NEEDED;
717
718 MiInitializePageTable();
719
720 MiBuildNonPagedPool();
721
722 MiBuildSystemPteSpace();
723
724 /* Map the PFN database pages */
725 MiBuildPfnDatabase(LoaderBlock);
726
727 /* Reset the ref/share count so that MmInitializeProcessAddressSpace works */
728 PMMPFN Pfn = MiGetPfnEntry(PFN_FROM_PTE((PMMPTE)PXE_SELFMAP));
729 Pfn->u2.ShareCount = 0;
730 Pfn->u3.e2.ReferenceCount = 0;
731
732 Pfn = MiGetPfnEntry(PFN_FROM_PDE(MiAddressToPde((PVOID)HYPER_SPACE)));
733 Pfn->u2.ShareCount = 0;
734 Pfn->u3.e2.ReferenceCount = 0;
735
736 Pfn = MiGetPfnEntry(PFN_FROM_PPE(MiAddressToPpe((PVOID)HYPER_SPACE)));
737 Pfn->u2.ShareCount = 0;
738 Pfn->u3.e2.ReferenceCount = 0;
739
740 Pfn = MiGetPfnEntry(PFN_FROM_PXE(MiAddressToPxe((PVOID)HYPER_SPACE)));
741 Pfn->u2.ShareCount = 0;
742 Pfn->u3.e2.ReferenceCount = 0;
743
744 Pfn = MiGetPfnEntry(PFN_FROM_PTE(MiAddressToPte(MmWorkingSetList)));
745 Pfn->u2.ShareCount = 0;
746 Pfn->u3.e2.ReferenceCount = 0;
747
748 /* Initialize the nonpaged pool */
749 InitializePool(NonPagedPool, 0);
750
751 /* Initialize the bogus address space */
752 Flags = 0;
753 Status = MmInitializeProcessAddressSpace(PsGetCurrentProcess(), NULL, NULL, &Flags, NULL);
754 if (!NT_SUCCESS(Status))
755 {
756 DPRINT1("MmInitializeProcessAddressSpace(9 failed: 0x%lx\n", Status);
757 return Status;
758 }
759
760 /* Initialize the balancer */
761 MmInitializeBalancer((ULONG)MmAvailablePages, 0);
762
763 /* Make sure we have everything we need */
764 ASSERT(MmPfnDatabase);
765 ASSERT(MmNonPagedSystemStart);
766 ASSERT(MmNonPagedPoolStart);
767 ASSERT(MmSizeOfNonPagedPoolInBytes);
768 ASSERT(MmMaximumNonPagedPoolInBytes);
769 ASSERT(MmNonPagedPoolExpansionStart);
770 ASSERT(MmHyperSpaceEnd);
771 ASSERT(MmNumberOfSystemPtes);
772 ASSERT(MiAddressToPde(MmNonPagedPoolStart)->u.Hard.Valid);
773 ASSERT(MiAddressToPte(MmNonPagedPoolStart)->u.Hard.Valid);
774
775 return STATUS_SUCCESS;
776 }
777