diff --git a/reactos/ntoskrnl/mm/amd64/init.c b/reactos/ntoskrnl/mm/amd64/init.c index b539a4f0924..89d7d1e5778 100644 --- a/reactos/ntoskrnl/mm/amd64/init.c +++ b/reactos/ntoskrnl/mm/amd64/init.c @@ -77,6 +77,10 @@ BOOLEAN MiIncludeType[LoaderMaximum]; PFN_NUMBER MxFreePageBase; ULONG64 MxFreePageCount = 0; +ULONG MxPhase = 0; + +PFN_NUMBER MmSystemPageDirectory; +PFN_NUMBER MmSizeOfPagedPoolInPages = MI_MIN_INIT_PAGED_POOLSIZE / PAGE_SIZE; /* FUNCTIONS *****************************************************************/ @@ -96,6 +100,10 @@ MiEvaluateMemoryDescriptors(IN PLOADER_PARAMETER_BLOCK LoaderBlock) PFN_NUMBER LastPage; ULONG i; + /* Get the size of the boot loader's image allocations */ + MmBootImageSize = KeLoaderBlock->Extension->LoaderPagesSpanned * PAGE_SIZE; + MmBootImageSize = ROUND_UP(MmBootImageSize, 4 * 1024 * 1024); + /* Instantiate memory that we don't consider RAM/usable */ for (i = 0; i < LoaderMaximum; i++) MiIncludeType[i] = TRUE; MiIncludeType[LoaderBad] = FALSE; @@ -160,47 +168,54 @@ MiEvaluateMemoryDescriptors(IN PLOADER_PARAMETER_BLOCK LoaderBlock) PFN_NUMBER NTAPI -MxGetNextPage(IN PFN_NUMBER PageCount) +MxAllocEarlyPage() { PFN_NUMBER Pfn; /* Make sure we have enough pages */ - if (PageCount > MxFreePageCount) + if (!MxFreePageCount) { /* Crash the system */ KeBugCheckEx(INSTALL_MORE_MEMORY, MmNumberOfPhysicalPages, MxFreeDescriptor->PageCount, MxOldFreeDescriptor.PageCount, - PageCount); + 1); } /* Use our lowest usable free pages */ Pfn = MxFreePageBase; - MxFreePageBase += PageCount; - MxFreePageCount -= PageCount; + MxFreePageBase++; + MxFreePageCount--; return Pfn; } +PFN_NUMBER +NTAPI +MxAllocPage() +{ + return (MxPhase == 0) ? MxAllocEarlyPage() : MmAllocPage(MC_SYSTEM, 0); +} + PMMPTE NTAPI MxGetPte(PVOID Address) { PMMPTE Pte; - MMPTE TmpPte; + MMPTE TmplPte; /* Setup template pte */ - TmpPte.u.Long = 0; - TmpPte.u.Flush.Valid = 1; - TmpPte.u.Flush.Write = 1; + TmplPte.u.Long = 0; + TmplPte.u.Flush.Valid = 1; + TmplPte.u.Flush.Write = 1; /* Get a pointer to the PXE */ Pte = MiAddressToPxe(Address); if (!Pte->u.Hard.Valid) { /* It's not valid, map it! */ - TmpPte.u.Hard.PageFrameNumber = MxGetNextPage(1); - *Pte = TmpPte; + TmplPte.u.Hard.PageFrameNumber = MxAllocPage(); + *Pte = TmplPte; } /* Get a pointer to the PPE */ @@ -208,8 +223,8 @@ MxGetPte(PVOID Address) if (!Pte->u.Hard.Valid) { /* It's not valid, map it! */ - TmpPte.u.Hard.PageFrameNumber = MxGetNextPage(1); - *Pte = TmpPte; + TmplPte.u.Hard.PageFrameNumber = MxAllocPage(); + *Pte = TmplPte; } /* Get a pointer to the PDE */ @@ -217,8 +232,8 @@ MxGetPte(PVOID Address) if (!Pte->u.Hard.Valid) { /* It's not valid, map it! */ - TmpPte.u.Hard.PageFrameNumber = MxGetNextPage(1); - *Pte = TmpPte; + TmplPte.u.Hard.PageFrameNumber = MxAllocPage(); + *Pte = TmplPte; } /* Get a pointer to the PTE */ @@ -227,24 +242,31 @@ MxGetPte(PVOID Address) } VOID -MxMapPageRange(PVOID Address, ULONG64 PageCount) +MxMapPage(PVOID Address) { - MMPTE TmpPte, *Pte; + MMPTE TmplPte, *Pte; /* Setup template pte */ - TmpPte.u.Long = 0; - TmpPte.u.Flush.Valid = 1; - TmpPte.u.Flush.Write = 1; + TmplPte.u.Long = 0; + TmplPte.u.Flush.Valid = 1; + TmplPte.u.Flush.Write = 1; + TmplPte.u.Hard.PageFrameNumber = MxAllocPage(); + /* Get the PTE for that page */ + Pte = MxGetPte(Address); + ASSERT(Pte->u.Hard.Valid == 0); + + /* Map a physical page */ + *Pte = TmplPte; +} + +VOID +MxMapPageRange(PVOID Address, ULONG64 PageCount) +{ while (PageCount--) { - /* Get the PTE for that page */ - Pte = MxGetPte(Address); - ASSERT(Pte->u.Hard.Valid == 0); - - /* Map a physical page */ - TmpPte.u.Hard.PageFrameNumber = MxGetNextPage(1); - *Pte = TmpPte; + /* Map the page */ + MxMapPage(Address); /* Goto next page */ Address = (PVOID)((ULONG64)Address + PAGE_SIZE); @@ -253,11 +275,170 @@ MxMapPageRange(PVOID Address, ULONG64 PageCount) VOID NTAPI -MiArmConfigureMemorySizes(IN PLOADER_PARAMETER_BLOCK LoaderBloc) +MiPreparePfnDatabse(IN PLOADER_PARAMETER_BLOCK LoaderBlock) { - /* Get the size of the boot loader's image allocations */ - MmBootImageSize = KeLoaderBlock->Extension->LoaderPagesSpanned * PAGE_SIZE; - MmBootImageSize = ROUND_UP(MmBootImageSize, 4 * 1024 * 1024); + PMEMORY_ALLOCATION_DESCRIPTOR MdBlock; + PLIST_ENTRY ListEntry; + PUCHAR Page, FirstPage; + SIZE_T Size; + + /* Calculate the size of the PFN database and convert to pages */ + MxPfnSizeInBytes = ROUND_TO_PAGES((MmHighestPhysicalPage + 1) * sizeof(MMPFN)); + MxPfnAllocation = MxPfnSizeInBytes >> PAGE_SHIFT; + + /* Sinply start at hardcoded address */ + MmPfnDatabase = (PVOID)0xFFFFFAC000000000ULL; + + /* Loop the memory descriptors */ + for (ListEntry = LoaderBlock->MemoryDescriptorListHead.Flink; + ListEntry != &LoaderBlock->MemoryDescriptorListHead; + ListEntry = ListEntry->Flink) + { + /* Get the memory descriptor */ + MdBlock = CONTAINING_RECORD(ListEntry, + MEMORY_ALLOCATION_DESCRIPTOR, + ListEntry); + + /* Skip pages that are not part of the PFN database */ + if (MiIncludeType[MdBlock->MemoryType]) + { + /* Get the base and size of this pfn database entry */ + FirstPage = PAGE_ALIGN(&MmPfnDatabase[MdBlock->BasePage]); + Size = ROUND_TO_PAGES(MdBlock->PageCount * sizeof(MMPFN)); + + /* Loop the pages of this Pfn database entry */ + for (Page = FirstPage; Page < FirstPage + Size; Page += PAGE_SIZE) + { + /* Is the page already mapped? */ + if (!MmIsAddressValid(Page)) + { + /* It's not, map it now */ + MxMapPage(Page); + } + } + + /* Zero out the pages */ + RtlZeroMemory(FirstPage, Size); + } + } +} + + +VOID +NTAPI +MiInitializeSessionSpace(IN PLOADER_PARAMETER_BLOCK LoaderBlock) +{ + /* Set up session space */ + MiSessionSpaceEnd = (PVOID)MI_SESSION_SPACE_END; + + /* This is where we will load Win32k.sys and the video driver */ + MiSessionImageEnd = MiSessionSpaceEnd; + MiSessionImageStart = (PCHAR)MiSessionImageEnd - MmSessionImageSize; + + /* The view starts right below the session working set (itself below + * the image area) */ + MiSessionViewEnd = MI_SESSION_VIEW_END; + MiSessionViewStart = (PCHAR)MiSessionViewEnd - MmSessionViewSize; + ASSERT(IS_PAGE_ALIGNED(MiSessionViewStart)); + + /* Session pool follows */ + MiSessionPoolEnd = MiSessionViewStart; + MiSessionPoolStart = (PCHAR)MiSessionPoolEnd - MmSessionPoolSize; + ASSERT(IS_PAGE_ALIGNED(MiSessionPoolStart)); + + /* And it all begins here */ + MmSessionBase = MiSessionPoolStart; + + /* System view space ends at session space, so now that we know where + * this is, we can compute the base address of system view space itself. */ + MiSystemViewStart = (PCHAR)MmSessionBase - MmSystemViewSize; + ASSERT(IS_PAGE_ALIGNED(MiSystemViewStart)); + + /* Sanity checks */ + ASSERT(MiSessionViewEnd <= MiSessionImageStart); + ASSERT(MmSessionBase <= MiSessionPoolStart); +} + +VOID +MiInitializePageTable() +{ + ULONG64 PageFrameOffset; + PMMPTE Pte, StartPte, EndPte; + MMPTE TmplPte; + PFN_NUMBER PageCount; + + /* HACK: don't use freeldr debug print anymore */ + FrLdrDbgPrint = NoDbgPrint; + + /* Get current directory base */ + MmSystemPageDirectory = ((PMMPTE)PXE_SELFMAP)->u.Hard.PageFrameNumber; + PageFrameOffset = MmSystemPageDirectory << PAGE_SHIFT; + ASSERT(PageFrameOffset == __readcr3()); + + /* Set directory base for the system process */ + PsGetCurrentProcess()->Pcb.DirectoryTableBase[0] = PageFrameOffset; + + /* Enable global pages */ + __writecr4(__readcr4() | CR4_PGE); + ASSERT(__readcr4() & CR4_PGE); + + /* Set user mode address range */ + StartPte = MiAddressToPxe(0); + EndPte = MiAddressToPxe(MmHighestUserAddress); + + /* Loop the user mode PXEs */ + for (Pte = StartPte; Pte <= EndPte; Pte++) + { + /* Zero the PXE, clear all mappings */ + Pte->u.Long = 0; + } + + /* Flush the TLB */ + KeFlushCurrentTb(); + + /* Set up a template PTE */ + TmplPte.u.Long = 0; + TmplPte.u.Flush.Valid = 1; + TmplPte.u.Flush.Write = 1; + HyperTemplatePte = TmplPte; + + /* Create PDPTs (72 KB) for shared system address space, + * skip page tables and hyperspace */ + + /* Set the range */ + StartPte = MiAddressToPxe((PVOID)(HYPER_SPACE_END + 1)); + EndPte = MiAddressToPxe(MI_HIGHEST_SYSTEM_ADDRESS); + + /* Loop the PXEs */ + for (Pte = StartPte; Pte <= EndPte; Pte++) + { + /* Is the PXE already valid? */ + if (!Pte->u.Hard.Valid) + { + /* It's not Initialize it */ + TmplPte.u.Flush.PageFrameNumber = MxAllocPage(); + *Pte = TmplPte; + + /* Zero the page. The PXE is the PTE for the PDPT. */ + RtlZeroMemory(MiPteToAddress(Pte), PAGE_SIZE); + } + } + + /* Setup the mapping PTEs */ + MmFirstReservedMappingPte = MxGetPte((PVOID)MI_MAPPING_RANGE_START); + MmFirstReservedMappingPte->u.Hard.PageFrameNumber = MI_HYPERSPACE_PTES; + MmLastReservedMappingPte = MiAddressToPte((PVOID)MI_MAPPING_RANGE_END); + + /* Setup debug mapping PTE */ + MmDebugPte = MxGetPte(MI_DEBUG_MAPPING); +} + +VOID +NTAPI +MiBuildNonPagedPool(VOID) +{ + PMMPTE Pte; + PFN_COUNT PageCount; /* Check if this is a machine with less than 256MB of RAM, and no overide */ if ((MmNumberOfPhysicalPages <= MI_MIN_PAGES_FOR_NONPAGED_POOL_TUNING) && @@ -317,52 +498,18 @@ MiArmConfigureMemorySizes(IN PLOADER_PARAMETER_BLOCK LoaderBloc) MmMaximumNonPagedPoolInBytes = MI_MAX_NONPAGED_POOL_SIZE; } - // MmSessionImageSize -} + /* Put non paged pool to the end of the region */ + MmNonPagedPoolStart = (PCHAR)MmNonPagedPoolEnd - MmMaximumNonPagedPoolInBytes; -VOID -NTAPI -MiArmInitializeMemoryLayout(IN PLOADER_PARAMETER_BLOCK LoaderBlock) -{ - /* Set up session space */ - MiSessionSpaceEnd = (PVOID)MI_SESSION_SPACE_END; + /* Make sure it doesn't collide with the PFN database */ + if ((PCHAR)MmNonPagedPoolStart < (PCHAR)MmPfnDatabase + MxPfnSizeInBytes) + { + /* Put non paged pool after the PFN database */ + MmNonPagedPoolStart = (PCHAR)MmPfnDatabase + MxPfnSizeInBytes; + MmMaximumNonPagedPoolInBytes = (ULONG64)MmNonPagedPoolEnd - + (ULONG64)MmNonPagedPoolStart; + } - /* This is where we will load Win32k.sys and the video driver */ - MiSessionImageEnd = MiSessionSpaceEnd; - MiSessionImageStart = (PCHAR)MiSessionImageEnd - MmSessionImageSize; - - /* The view starts right below the session working set (itself below - * the image area) */ - MiSessionViewEnd = MI_SESSION_VIEW_END; - MiSessionViewStart = (PCHAR)MiSessionViewEnd - MmSessionViewSize; - ASSERT(IS_PAGE_ALIGNED(MiSessionViewStart)); - - /* Session pool follows */ - MiSessionPoolEnd = MiSessionViewStart; - MiSessionPoolStart = (PCHAR)MiSessionPoolEnd - MmSessionPoolSize; - ASSERT(IS_PAGE_ALIGNED(MiSessionPoolStart)); - - /* And it all begins here */ - MmSessionBase = MiSessionPoolStart; - - /* System view space ends at session space, so now that we know where - * this is, we can compute the base address of system view space itself. */ - MiSystemViewStart = (PCHAR)MmSessionBase - MmSystemViewSize; - ASSERT(IS_PAGE_ALIGNED(MiSystemViewStart)); - - /* Calculate the size of the PFN database and convert to pages */ - MxPfnSizeInBytes = ROUND_TO_PAGES((MmHighestPhysicalPage + 1) * sizeof(MMPFN)); - MxPfnAllocation = MxPfnSizeInBytes >> PAGE_SHIFT; - - /* The PFN database is at the start of the non paged region */ - MmPfnDatabase = (PVOID)((PCHAR)MmNonPagedPoolEnd - - MmMaximumNonPagedPoolInBytes); - - /* Reduce maximum non paged pool size */ - MmMaximumNonPagedPoolInBytes -= MxPfnSizeInBytes; - - /* Put non paged pool after the PFN database */ - MmNonPagedPoolStart = (PCHAR)MmPfnDatabase + MxPfnSizeInBytes; ASSERT(IS_PAGE_ALIGNED(MmNonPagedPoolStart)); /* Calculate the nonpaged pool expansion start region */ @@ -370,6 +517,32 @@ MiArmInitializeMemoryLayout(IN PLOADER_PARAMETER_BLOCK LoaderBlock) MmSizeOfNonPagedPoolInBytes; ASSERT(IS_PAGE_ALIGNED(MmNonPagedPoolExpansionStart)); + /* Map the nonpaged pool */ + PageCount = (MmSizeOfNonPagedPoolInBytes + PAGE_SIZE - 1) / PAGE_SIZE; + MxMapPageRange(MmNonPagedPoolStart, PageCount); + + /* Loop the non paged pool extension PTEs */ + for (Pte = MiAddressToPte(MmNonPagedPoolExpansionStart); + Pte <= MiAddressToPte(MmNonPagedPoolEnd); + Pte++) + { + /* Create PXE, PPE, PDE and zero the PTE */ + MxGetPte(MiPteToAddress(Pte))->u.Long = 0; + } + + /* Initialize the ARM3 nonpaged pool */ + MiInitializeArmPool(); + + /* Initialize the nonpaged pool */ + InitializePool(NonPagedPool, 0); +} + +VOID +NTAPI +MiBuildSystemPteSpace() +{ + PMMPTE Pte, StartPte, EndPte; + /* Use the default numer of system PTEs */ MmNumberOfSystemPtes = MI_NUMBER_SYSTEM_PTES; @@ -392,122 +565,6 @@ MiArmInitializeMemoryLayout(IN PLOADER_PARAMETER_BLOCK LoaderBlock) ASSERT(MmNumberOfSystemPtes > 1000); } - /* Sanity checks */ - ASSERT(MiSessionViewEnd <= MiSessionImageStart); - ASSERT(MmSessionBase <= MiSessionPoolStart); -} - -VOID -NTAPI -MiArmPreparePfnDatabse(IN PLOADER_PARAMETER_BLOCK LoaderBlock) -{ - PMEMORY_ALLOCATION_DESCRIPTOR MdBlock; - PLIST_ENTRY ListEntry; - PUCHAR Page, FirstPage; - SIZE_T Size; - - /* Loop the memory descriptors */ - for (ListEntry = LoaderBlock->MemoryDescriptorListHead.Flink; - ListEntry != &LoaderBlock->MemoryDescriptorListHead; - ListEntry = ListEntry->Flink) - { - /* Get the memory descriptor */ - MdBlock = CONTAINING_RECORD(ListEntry, - MEMORY_ALLOCATION_DESCRIPTOR, - ListEntry); - - /* Skip pages that are not part of the PFN database */ - if (!MiIncludeType[MdBlock->MemoryType]) - { - continue; - } - - /* Get the base and size of this pfn database entry */ - FirstPage = PAGE_ALIGN(&MmPfnDatabase[MdBlock->BasePage]); - Size = ROUND_TO_PAGES(MdBlock->PageCount * sizeof(MMPFN)); - - /* Loop the pages of this Pfn database entry */ - for (Page = FirstPage; Page < FirstPage + Size; Page += PAGE_SIZE) - { - /* Is the page already mapped? */ - if (!MmIsAddressValid(Page)) - { - /* It's not, map it now */ - MxMapPageRange(Page, 1); - } - } - - /* Zero out the pages */ - RtlZeroMemory(FirstPage, Size); - } -} - - -VOID -MiArmInitializePageTable() -{ - ULONG64 PageFrameOffset; - PMMPTE Pte, StartPte, EndPte; - MMPTE TmplPte; - PFN_NUMBER PageCount; - PVOID Address; - - /* HACK: don't use freeldr debug print anymore */ - FrLdrDbgPrint = NoDbgPrint; - - /* Get current directory base */ - PageFrameOffset = ((PMMPTE)PXE_SELFMAP)->u.Hard.PageFrameNumber << PAGE_SHIFT; - ASSERT(PageFrameOffset == __readcr3()); - - /* Set directory base for the system process */ - PsGetCurrentProcess()->Pcb.DirectoryTableBase[0] = PageFrameOffset; - - /* Enable global pages */ - __writecr4(__readcr4() | CR4_PGE); - ASSERT(__readcr4() & CR4_PGE); - - /* Set user mode address range */ - StartPte = MiAddressToPxe(0); - EndPte = MiAddressToPxe(MmHighestUserAddress); - - /* Loop the user mode PXEs */ - for (Pte = StartPte; Pte <= EndPte; Pte++) - { - /* Zero the PXE, clear all mappings */ - Pte->u.Long = 0; - } - - /* Flush the TLB */ - KeFlushCurrentTb(); - - /* Set up a template PTE */ - TmplPte.u.Long = 0; - TmplPte.u.Flush.Valid = 1; - TmplPte.u.Flush.Write = 1; - HyperTemplatePte = TmplPte; - - /* Create PDPTs (72 KB) for shared system address space, - * skip page tables and hyperspace */ - - /* Set the range */ - StartPte = MiAddressToPxe((PVOID)(HYPER_SPACE_END + 1)); - EndPte = MiAddressToPxe(MI_HIGHEST_SYSTEM_ADDRESS); - - /* Loop the PXEs */ - for (Pte = StartPte; Pte <= EndPte; Pte++) - { - /* Is the PXE already valid? */ - if (!Pte->u.Hard.Valid) - { - /* It's not Initialize it */ - TmplPte.u.Flush.PageFrameNumber = MxGetNextPage(1); - *Pte = TmplPte; - - /* Zero the page. The PXE is the PTE for the PDPT. */ - RtlZeroMemory(MiPteToAddress(Pte), PAGE_SIZE); - } - } - /* Set the range of system PTEs */ StartPte = MiAddressToPte(MI_SYSTEM_PTE_START); EndPte = StartPte + MmNumberOfSystemPtes - 1; @@ -515,33 +572,22 @@ MiArmInitializePageTable() /* Loop the system PTEs */ for (Pte = StartPte; Pte <= EndPte; Pte++) { - /* Make sure the PTE is valid */ - MxGetPte(MiPteToAddress(Pte)); + /* Create PXE, PPE, PDE and zero the PTE */ + MxGetPte(MiPteToAddress(Pte))->u.Long = 0; } - /* Map the nonpaged pool */ - PageCount = (MmSizeOfNonPagedPoolInBytes + PAGE_SIZE - 1) / PAGE_SIZE; - MxMapPageRange(MmNonPagedPoolStart, PageCount); + /* Create the system PTE space */ + Pte = MiAddressToPte(MI_SYSTEM_PTE_START); + MiInitializeSystemPtes(Pte, MmNumberOfSystemPtes, SystemPteSpace); - /* Create PTEs for the paged pool extension */ - for (Address = MmNonPagedPoolExpansionStart; - Address < MmNonPagedPoolEnd; - Address = (PVOID)((ULONG64)Address + PAGE_SIZE)) - { - /* Create PXE, PPE, PDE and set PTE to 0*/ - MxGetPte(Address)->u.Long = 0; - } + /* Reserve system PTEs for zeroing PTEs and clear them */ + MiFirstReservedZeroingPte = MiReserveSystemPtes(MI_ZERO_PTES, SystemPteSpace); + RtlZeroMemory(MiFirstReservedZeroingPte, MI_ZERO_PTES * sizeof(MMPTE)); - /* Setup the mapping PTEs */ - MmFirstReservedMappingPte = MxGetPte((PVOID)MI_MAPPING_RANGE_START); - MmFirstReservedMappingPte->u.Hard.PageFrameNumber = MI_HYPERSPACE_PTES; - MmLastReservedMappingPte = MiAddressToPte((PVOID)MI_MAPPING_RANGE_END); - - /* Setup debug mapping PTE */ - MmDebugPte = MxGetPte(MI_DEBUG_MAPPING); + /* Set the counter to maximum */ + MiFirstReservedZeroingPte->u.Hard.PageFrameNumber = MI_ZERO_PTES - 1; } - VOID NTAPI MiBuildPhysicalMemoryBlock(IN PLOADER_PARAMETER_BLOCK LoaderBlock) @@ -555,7 +601,7 @@ MiBuildPhysicalMemoryBlock(IN PLOADER_PARAMETER_BLOCK LoaderBlock) ULONG Size, i; /* Calculate size for the PFN bitmap */ - Size = (MmHighestPhysicalPage + sizeof(ULONG)) / sizeof(ULONG); + Size = ROUND_UP(MmHighestPhysicalPage + 1, sizeof(ULONG)); /* Allocate the PFN bitmap */ Bitmap = ExAllocatePoolWithTag(NonPagedPool, Size, ' mM'); @@ -628,7 +674,129 @@ VOID NTAPI MiBuildPagedPool(VOID) { - UNIMPLEMENTED; + PMMPTE Pte; + MMPTE TmplPte; + PFN_NUMBER PageFrameIndex; + KIRQL OldIrql; + ULONG Size, BitMapSize; + + /* Default size for paged pool is 4 times non paged pool */ + MmSizeOfPagedPoolInBytes = 4 * MmMaximumNonPagedPoolInBytes; + + /* Make sure it doesn't overflow */ + if (MmSizeOfPagedPoolInBytes > ((ULONG64)MmNonPagedSystemStart - + (ULONG64)MmPagedPoolStart)) + { + MmSizeOfPagedPoolInBytes = (ULONG64)MmNonPagedSystemStart - + (ULONG64)MmPagedPoolStart; + } + + /* Make sure paged pool is big enough */ + if (MmSizeOfPagedPoolInBytes < MI_MIN_INIT_PAGED_POOLSIZE) + { + MmSizeOfPagedPoolInBytes = MI_MIN_INIT_PAGED_POOLSIZE; + } + + /* Align down to a PDE boundary */ + MmSizeOfPagedPoolInBytes = ROUND_DOWN(MmSizeOfPagedPoolInBytes, + 512 * PAGE_SIZE); + MmSizeOfPagedPoolInPages = MmSizeOfPagedPoolInBytes >> PAGE_SHIFT; + + /* This is where paged pool ends */ + MmPagedPoolEnd = (PCHAR)MmPagedPoolStart + MmSizeOfPagedPoolInBytes - 1; + + /* Sanity check */ + ASSERT(MmPagedPoolEnd < MmNonPagedSystemStart); + + /* setup a template PTE */ + TmplPte.u.Long = 0; + TmplPte.u.Flush.Valid = 1; + TmplPte.u.Flush.Write = 1; + + /* Make sure the PXE is valid */ + Pte = MiAddressToPxe(MmPagedPoolStart); + if (!Pte->u.Flush.Valid) + { + /* Map it! */ + TmplPte.u.Flush.PageFrameNumber = MmAllocPage(MC_SYSTEM, 0); + *Pte = TmplPte; + } + + /* Map all page directories (max 128) */ + for (Pte = MiAddressToPpe(MmPagedPoolStart); + Pte <= MiAddressToPpe(MmPagedPoolEnd); + Pte++) + { + if (!Pte->u.Flush.Valid) + { + /* Map it! */ + TmplPte.u.Flush.PageFrameNumber = MxAllocPage(); + *Pte = TmplPte; + } + } + + /* Create and map the first PTE for paged pool */ + Pte = MxGetPte(MmPagedPoolStart); + TmplPte.u.Flush.PageFrameNumber = MxAllocPage(); + *Pte = TmplPte; + + /* Save the first and last paged pool PTE */ + MmPagedPoolInfo.FirstPteForPagedPool = MiAddressToPte(MmPagedPoolStart); + MmPagedPoolInfo.LastPteForPagedPool = MiAddressToPte(MmPagedPoolEnd); + + MmPagedPoolInfo.NextPdeForPagedPoolExpansion = + MiAddressToPde(MmPagedPoolStart) + 1; + + // We keep track of each page via a bit, so check how big the bitmap will + // have to be (make sure to align our page count such that it fits nicely + // into a 4-byte aligned bitmap. + + /* The size of the bitmap in bits is the size in pages */ + BitMapSize = MmSizeOfPagedPoolInPages; + + /* Calculate buffer size in bytes, aligned to 32 bits */ + Size = sizeof(RTL_BITMAP) + ROUND_UP(BitMapSize, 32) / 8; + + // Allocate the allocation bitmap, which tells us which regions have not yet + // been mapped into memory + + MmPagedPoolInfo.PagedPoolAllocationMap = + ExAllocatePoolWithTag(NonPagedPool, Size, ' mM'); + ASSERT(MmPagedPoolInfo.PagedPoolAllocationMap); + + DPRINT1("BitMapSize = 0x%lx, Size = 0x%lx\n", BitMapSize, Size); + // Initialize it such that at first, only the first page's worth of PTEs is + // marked as allocated (incidentially, the first PDE we allocated earlier). + RtlInitializeBitMap(MmPagedPoolInfo.PagedPoolAllocationMap, + (PULONG)(MmPagedPoolInfo.PagedPoolAllocationMap + 1), + BitMapSize); + RtlSetAllBits(MmPagedPoolInfo.PagedPoolAllocationMap); + RtlClearBits(MmPagedPoolInfo.PagedPoolAllocationMap, 0, 512); + + // We have a second bitmap, which keeps track of where allocations end. + // Given the allocation bitmap and a base address, we can therefore figure + // out which page is the last page of that allocation, and thus how big the + // entire allocation is. + MmPagedPoolInfo.EndOfPagedPoolBitmap = + ExAllocatePoolWithTag(NonPagedPool, Size, ' mM'); + ASSERT(MmPagedPoolInfo.EndOfPagedPoolBitmap); + + DPRINT1("PagedPoolAllocationMap=%p, EndOfPagedPoolBitmap=%p\n", + MmPagedPoolInfo.PagedPoolAllocationMap, MmPagedPoolInfo.EndOfPagedPoolBitmap); + + /* Initialize the bitmap */ + RtlInitializeBitMap(MmPagedPoolInfo.EndOfPagedPoolBitmap, + (PULONG)(MmPagedPoolInfo.EndOfPagedPoolBitmap + 1), + BitMapSize); + + /* No allocations, no allocation ends; clear all bits. */ + RtlClearAllBits(MmPagedPoolInfo.EndOfPagedPoolBitmap); + + /* Initialize the paged pool mutex */ + KeInitializeGuardedMutex(&MmPagedPoolMutex); + + /* Initialize the paged pool */ + InitializePool(PagedPool, 0); } @@ -642,20 +810,14 @@ MmArmInitSystem(IN ULONG Phase, /* Parse memory descriptors */ MiEvaluateMemoryDescriptors(LoaderBlock); - /* Configure the memory sizes */ - MiArmConfigureMemorySizes(LoaderBlock); - - /* Initialize the memory layout */ - MiArmInitializeMemoryLayout(LoaderBlock); - /* Prepare PFN database mappings */ - MiArmPreparePfnDatabse(LoaderBlock); + MiPreparePfnDatabse(LoaderBlock); + + /* Initialize the session space */ + MiInitializeSessionSpace(LoaderBlock); /* Initialize some mappings */ - MiArmInitializePageTable(); - - /* Initialize the ARM3 nonpaged pool */ - MiInitializeArmPool(); + MiInitializePageTable(); /* Update the memory descriptor, to make sure the pages we used won't get inserted into the PFN database */ @@ -672,20 +834,14 @@ MmArmInitSystem(IN ULONG Phase, /* The PFN database was created, restore the free descriptor */ *MxFreeDescriptor = MxOldFreeDescriptor; + /* Switch to phase 1 */ + MxPhase = 1; + /* Initialize the nonpaged pool */ - InitializePool(NonPagedPool, 0); + MiBuildNonPagedPool(); - /* Create the system PTE space */ - Pte = MiAddressToPte(MI_SYSTEM_PTE_START); - MiInitializeSystemPtes(Pte, MmNumberOfSystemPtes, SystemPteSpace); - - /* Reserve system PTEs for zeroing PTEs and clear them */ - MiFirstReservedZeroingPte = MiReserveSystemPtes(MI_ZERO_PTES, - SystemPteSpace); - RtlZeroMemory(MiFirstReservedZeroingPte, MI_ZERO_PTES * sizeof(MMPTE)); - - /* Set the counter to maximum */ - MiFirstReservedZeroingPte->u.Hard.PageFrameNumber = MI_ZERO_PTES - 1; + /* Initialize system PTE handling */ + MiBuildSystemPteSpace(); /* Build the physical memory block */ MiBuildPhysicalMemoryBlock(LoaderBlock);