From 581d7a7972ba2123cd87a1ddc0e91c905b2a34a0 Mon Sep 17 00:00:00 2001 From: ReactOS Portable Systems Group Date: Fri, 15 Feb 2008 06:53:49 +0000 Subject: [PATCH] Remove Ke386Pae conditionals from page code -- PAE is never turned on ReactOS (and probably wouldn't work), so there's no point in wasting CPU cycles and making the code even more unredable by having the PAE code there. The PAE implementation has been moved to pagepae.c, in case someone will ever want to revive. Even in that case, it's recommended to build two kernels like everyone else does, instead of doing runtime conditionals for every single page operation. svn path=/trunk/; revision=32374 --- reactos/ntoskrnl/mm/i386/page.c | 1103 +------------ reactos/ntoskrnl/mm/i386/pagepae.c | 2472 ++++++++++++++++++++++++++++ 2 files changed, 2527 insertions(+), 1048 deletions(-) create mode 100644 reactos/ntoskrnl/mm/i386/pagepae.c diff --git a/reactos/ntoskrnl/mm/i386/page.c b/reactos/ntoskrnl/mm/i386/page.c index 182abac3394..67ed2494e6c 100644 --- a/reactos/ntoskrnl/mm/i386/page.c +++ b/reactos/ntoskrnl/mm/i386/page.c @@ -43,20 +43,14 @@ #define PAGETABLE_MAP (0xc0000000) #define PAGEDIRECTORY_MAP (0xc0000000 + (PAGETABLE_MAP / (1024))) -#define PAE_PAGEDIRECTORY_MAP (0xc0000000 + (PAGETABLE_MAP / (512))) - -#define HYPERSPACE (Ke386Pae ? 0xc0800000 : 0xc0400000) +#define HYPERSPACE (0xc0400000) #define IS_HYPERSPACE(v) (((ULONG)(v) >= HYPERSPACE && (ULONG)(v) < HYPERSPACE + 0x400000)) ULONG MmGlobalKernelPageDirectory[1024]; -ULONGLONG MmGlobalKernelPageDirectoryForPAE[2048]; #define PTE_TO_PFN(X) ((X) >> PAGE_SHIFT) #define PFN_TO_PTE(X) ((X) << PAGE_SHIFT) -#define PAE_PTE_TO_PFN(X) (PAE_PAGE_MASK(X) >> PAGE_SHIFT) -#define PAE_PFN_TO_PTE(X) ((X) << PAGE_SHIFT) - #if defined(__GNUC__) #define PTE_TO_PAGE(X) ((LARGE_INTEGER)(LONGLONG)(PAGE_MASK(X))) #else @@ -68,9 +62,6 @@ __inline LARGE_INTEGER PTE_TO_PAGE(ULONG npage) } #endif -extern BOOLEAN Ke386Pae; -extern BOOLEAN Ke386NoExecute; - /* FUNCTIONS ***************************************************************/ BOOLEAN MmUnmapPageTable(PULONG Pt); @@ -97,25 +88,10 @@ MiFlushTlbIpiRoutine(ULONG_PTR Address) VOID MiFlushTlb(PULONG Pt, PVOID Address) { -#ifdef CONFIG_SMP - if (Pt) - { - MmUnmapPageTable(Pt); - } - if (KeNumberProcessors > 1) - { - KeIpiGenericCall(MiFlushTlbIpiRoutine, (ULONG_PTR)Address); - } - else - { - MiFlushTlbIpiRoutine((ULONG_PTR)Address); - } -#else if ((Pt && MmUnmapPageTable(Pt)) || Address >= MmSystemRangeStart) { __invlpg(Address); } -#endif } @@ -148,12 +124,7 @@ ProtectToPTE(ULONG flProtect) DPRINT1("Unknown main protection type.\n"); KEBUGCHECK(0); } - if (Ke386NoExecute && - !(flProtect & PAGE_IS_EXECUTABLE)) - { - Attributes = Attributes | 0x80000000; - } - + if (flProtect & PAGE_SYSTEM) { } @@ -183,22 +154,6 @@ ProtectToPTE(ULONG flProtect) #define ADDR_TO_PTE_OFFSET(v) ((((ULONG)(v)) % (1024 * PAGE_SIZE)) / PAGE_SIZE) -#define PAE_ADDR_TO_PAGE_TABLE(v) (((ULONG)(v)) / (512 * PAGE_SIZE)) - -#define PAE_ADDR_TO_PDE(v) (PULONGLONG) (PAE_PAGEDIRECTORY_MAP + \ - ((((ULONG_PTR)(v)) / (512 * 512))&(~0x7))) -#define PAE_ADDR_TO_PTE(v) (PULONGLONG) (PAGETABLE_MAP + ((((ULONG_PTR)(v) / 512))&(~0x7))) - - -#define PAE_ADDR_TO_PDTE_OFFSET(v) (((ULONG_PTR)(v)) / (512 * 512 * PAGE_SIZE)) - -#define PAE_ADDR_TO_PDE_PAGE_OFFSET(v) ((((ULONG_PTR)(v)) % (512 * 512 * PAGE_SIZE)) / (512 * PAGE_SIZE)) - -#define PAE_ADDR_TO_PDE_OFFSET(v) (((ULONG_PTR)(v))/ (512 * PAGE_SIZE)) - -#define PAE_ADDR_TO_PTE_OFFSET(v) ((((ULONG_PTR)(v)) % (512 * PAGE_SIZE)) / PAGE_SIZE) - - NTSTATUS NTAPI Mmi386ReleaseMmInfo(PEPROCESS Process) @@ -221,98 +176,43 @@ Mmi386ReleaseMmInfo(PEPROCESS Process) ExFreePool((PVOID) LdtBase); } - if (Ke386Pae) - { - PULONGLONG PageDirTable; - PULONGLONG PageDir; - PULONGLONG Pde; - ULONG k; - PageDirTable = (PULONGLONG)MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart)); - for (i = 0; i < 4; i++) - { - PageDir = (PULONGLONG)MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(PageDirTable[i])); - if (i < PAE_ADDR_TO_PDTE_OFFSET(MmSystemRangeStart)) - { - for (j = 0; j < 512; j++) - { - if (PageDir[j] != 0LL) - { - DPRINT1("ProcessId %d, Pde for %08x - %08x is not freed, RefCount %d\n", - Process->UniqueProcessId, - (i * 512 + j) * 512 * PAGE_SIZE, (i * 512 + j + 1) * 512 * PAGE_SIZE - 1, - ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable[i*512 + j]); - Pde = MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(PageDir[j])); - for (k = 0; k < 512; k++) - { - if(Pde[k] != 0) - { - if (Pde[k] & PA_PRESENT) - { - DPRINT1("Page at %08x is not freed\n", - (i * 512 + j) * 512 * PAGE_SIZE + k * PAGE_SIZE); - } - else - { - DPRINT1("Swapentry %x at %x is not freed\n", - (i * 512 + j) * 512 * PAGE_SIZE + k * PAGE_SIZE); - } - } - } - MmDeleteHyperspaceMapping(Pde); - MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(PageDir[j])); - } - } - } - if (i == PAE_ADDR_TO_PDTE_OFFSET(HYPERSPACE)) - { - MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE)])); - MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE)+1])); - } - MmDeleteHyperspaceMapping(PageDir); - MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(PageDirTable[i])); - } - MmDeleteHyperspaceMapping((PVOID)PageDirTable); - MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart)); - } - else - { - PULONG Pde; - PULONG PageDir; - PageDir = MmCreateHyperspaceMapping(PTE_TO_PFN(Process->Pcb.DirectoryTableBase.u.LowPart)); - for (i = 0; i < ADDR_TO_PDE_OFFSET(MmSystemRangeStart); i++) - { - if (PageDir[i] != 0) - { + PULONG Pde; + PULONG PageDir; + PageDir = MmCreateHyperspaceMapping(PTE_TO_PFN(Process->Pcb.DirectoryTableBase.u.LowPart)); + for (i = 0; i < ADDR_TO_PDE_OFFSET(MmSystemRangeStart); i++) + { + if (PageDir[i] != 0) + { DPRINT1("Pde for %08x - %08x is not freed, RefCount %d\n", - i * 4 * 1024 * 1024, (i + 1) * 4 * 1024 * 1024 - 1, - ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable[i]); - Pde = MmCreateHyperspaceMapping(PTE_TO_PFN(PageDir[i])); - for (j = 0; j < 1024; j++) - { - if(Pde[j] != 0) - { - if (Pde[j] & PA_PRESENT) - { - DPRINT1("Page at %08x is not freed\n", - i * 4 * 1024 * 1024 + j * PAGE_SIZE); - } - else - { - DPRINT1("Swapentry %x at %x is not freed\n", - Pde[j], i * 4 * 1024 * 1024 + j * PAGE_SIZE); - } - } - } - MmDeleteHyperspaceMapping(Pde); - MmReleasePageMemoryConsumer(MC_NPPOOL, PTE_TO_PFN(PageDir[i])); - } - } - MmReleasePageMemoryConsumer(MC_NPPOOL, PTE_TO_PFN(PageDir[ADDR_TO_PDE_OFFSET(HYPERSPACE)])); - MmDeleteHyperspaceMapping(PageDir); - MmReleasePageMemoryConsumer(MC_NPPOOL, PTE_TO_PFN(Process->Pcb.DirectoryTableBase.u.LowPart)); - } - + i * 4 * 1024 * 1024, (i + 1) * 4 * 1024 * 1024 - 1, + ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable[i]); + Pde = MmCreateHyperspaceMapping(PTE_TO_PFN(PageDir[i])); + for (j = 0; j < 1024; j++) + { + if(Pde[j] != 0) + { + if (Pde[j] & PA_PRESENT) + { + DPRINT1("Page at %08x is not freed\n", + i * 4 * 1024 * 1024 + j * PAGE_SIZE); + } + else + { + DPRINT1("Swapentry %x at %x is not freed\n", + Pde[j], i * 4 * 1024 * 1024 + j * PAGE_SIZE); + + } + } + } + MmDeleteHyperspaceMapping(Pde); + MmReleasePageMemoryConsumer(MC_NPPOOL, PTE_TO_PFN(PageDir[i])); + } + } + MmReleasePageMemoryConsumer(MC_NPPOOL, PTE_TO_PFN(PageDir[ADDR_TO_PDE_OFFSET(HYPERSPACE)])); + MmDeleteHyperspaceMapping(PageDir); + MmReleasePageMemoryConsumer(MC_NPPOOL, PTE_TO_PFN(Process->Pcb.DirectoryTableBase.u.LowPart)); + #if defined(__GNUC__) Process->Pcb.DirectoryTableBase.QuadPart = 0LL; @@ -349,14 +249,11 @@ MmCreateProcessAddressSpace(IN ULONG MinWs, { NTSTATUS Status; ULONG i, j; - PFN_TYPE Pfn[7]; - ULONG Count; + PFN_TYPE Pfn[2]; DPRINT("MmCopyMmInfo(Src %x, Dest %x)\n", MinWs, Process); - Count = Ke386Pae ? 7 : 2; - - for (i = 0; i < Count; i++) + for (i = 0; i < 2; i++) { Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn[i]); if (!NT_SUCCESS(Status)) @@ -370,38 +267,6 @@ MmCreateProcessAddressSpace(IN ULONG MinWs, } } - if (Ke386Pae) - { - PULONGLONG PageDirTable; - PULONGLONG PageDir; - - PageDirTable = MmCreateHyperspaceMapping(Pfn[0]); - for (i = 0; i < 4; i++) - { - PageDirTable[i] = PAE_PFN_TO_PTE(Pfn[1+i]) | PA_PRESENT; - } - MmDeleteHyperspaceMapping(PageDirTable); - for (i = PAE_ADDR_TO_PDTE_OFFSET(MmSystemRangeStart); i < 4; i++) - { - PageDir = (PULONGLONG)MmCreateHyperspaceMapping(Pfn[i+1]); - memcpy(PageDir, &MmGlobalKernelPageDirectoryForPAE[i * 512], 512 * sizeof(ULONGLONG)); - if (PAE_ADDR_TO_PDTE_OFFSET(PAGETABLE_MAP) == i) - { - for (j = 0; j < 4; j++) - { - PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(PAGETABLE_MAP) + j] = PAE_PFN_TO_PTE(Pfn[1+j]) | PA_PRESENT | PA_READWRITE; - } - } - if (PAE_ADDR_TO_PDTE_OFFSET(HYPERSPACE) == i) - { - PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE)] = PAE_PFN_TO_PTE(Pfn[5]) | PA_PRESENT | PA_READWRITE; - PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE)+1] = PAE_PFN_TO_PTE(Pfn[6]) | PA_PRESENT | PA_READWRITE; - } - MmDeleteHyperspaceMapping(PageDir); - } - } - else - { PULONG PageDirectory; PageDirectory = MmCreateHyperspaceMapping(Pfn[0]); @@ -414,7 +279,6 @@ MmCreateProcessAddressSpace(IN ULONG MinWs, PageDirectory[ADDR_TO_PDE_OFFSET(HYPERSPACE)] = PFN_TO_PTE(Pfn[1]) | PA_PRESENT | PA_READWRITE; MmDeleteHyperspaceMapping(PageDirectory); - } DirectoryTableBase->QuadPart = PFN_TO_PTE(Pfn[0]); DPRINT("Finished MmCopyMmInfo(): %I64x\n", DirectoryTableBase->QuadPart); @@ -432,17 +296,9 @@ MmDeletePageTable(PEPROCESS Process, PVOID Address) KeAttachProcess(&Process->Pcb); } - if (Ke386Pae) - { - ULONGLONG ZeroPde = 0LL; - (void)ExfpInterlockedExchange64UL(PAE_ADDR_TO_PDE(Address), &ZeroPde); - MiFlushTlb((PULONG)PAE_ADDR_TO_PDE(Address), PAE_ADDR_TO_PTE(Address)); - } - else - { *(ADDR_TO_PDE(Address)) = 0; MiFlushTlb(ADDR_TO_PDE(Address), ADDR_TO_PTE(Address)); - } + if (Address >= MmSystemRangeStart) { KEBUGCHECK(0); @@ -467,26 +323,7 @@ MmFreePageTable(PEPROCESS Process, PVOID Address) { KeAttachProcess(&Process->Pcb); } - if (Ke386Pae) - { - PULONGLONG PageTable; - ULONGLONG ZeroPte = 0LL; - PageTable = (PULONGLONG)PAGE_ROUND_DOWN((PVOID)PAE_ADDR_TO_PTE(Address)); - for (i = 0; i < 512; i++) - { - if (PageTable[i] != 0LL) - { - DbgPrint("Page table entry not clear at %x/%x (is %I64x)\n", - ((ULONG)Address / (4*1024*1024)), i, PageTable[i]); - KEBUGCHECK(0); - } - } - Pfn = PAE_PTE_TO_PFN(*(PAE_ADDR_TO_PDE(Address))); - (void)ExfpInterlockedExchange64UL(PAE_ADDR_TO_PDE(Address), &ZeroPte); - MiFlushTlb((PULONG)PAE_ADDR_TO_PDE(Address), PAE_ADDR_TO_PTE(Address)); - } - else - { + PULONG PageTable; PageTable = (PULONG)PAGE_ROUND_DOWN((PVOID)ADDR_TO_PTE(Address)); for (i = 0; i < 1024; i++) @@ -501,7 +338,6 @@ MmFreePageTable(PEPROCESS Process, PVOID Address) Pfn = PTE_TO_PFN(*(ADDR_TO_PDE(Address))); *(ADDR_TO_PDE(Address)) = 0; MiFlushTlb(ADDR_TO_PDE(Address), ADDR_TO_PTE(Address)); - } if (Address >= MmSystemRangeStart) { @@ -518,120 +354,6 @@ MmFreePageTable(PEPROCESS Process, PVOID Address) } } -static PULONGLONG -MmGetPageTableForProcessForPAE(PEPROCESS Process, PVOID Address, BOOLEAN Create) -{ - NTSTATUS Status; - PFN_TYPE Pfn; - ULONGLONG Entry; - ULONGLONG ZeroEntry = 0LL; - PULONGLONG Pt; - PULONGLONG PageDir; - PULONGLONG PageDirTable; - - DPRINT("MmGetPageTableForProcessForPAE(%x %x %d)\n", - Process, Address, Create); - if (Address >= (PVOID)PAGETABLE_MAP && Address < (PVOID)((ULONG_PTR)PAGETABLE_MAP + 0x800000)) - { - KEBUGCHECK(0); - } - if (Address < MmSystemRangeStart && Process && Process != PsGetCurrentProcess()) - { - PageDirTable = MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart)); - if (PageDirTable == NULL) - { - KEBUGCHECK(0); - } - PageDir = MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(PageDirTable[PAE_ADDR_TO_PDTE_OFFSET(Address)])); - MmDeleteHyperspaceMapping(PageDirTable); - if (PageDir == NULL) - { - KEBUGCHECK(0); - } - PageDir += PAE_ADDR_TO_PDE_PAGE_OFFSET(Address); - Entry = ExfInterlockedCompareExchange64UL(PageDir, &ZeroEntry, &ZeroEntry); - if (Entry == 0LL) - { - if (Create == FALSE) - { - MmDeleteHyperspaceMapping(PageDir); - return NULL; - } - Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn); - if (!NT_SUCCESS(Status)) - { - KEBUGCHECK(0); - } - Entry = PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE | PA_USER; - Entry = ExfInterlockedCompareExchange64UL(PageDir, &Entry, &ZeroEntry); - if (Entry != 0LL) - { - MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn); - Pfn = PAE_PTE_TO_PFN(Entry); - } - } - else - { - Pfn = PAE_PTE_TO_PFN(Entry); - } - MmDeleteHyperspaceMapping(PageDir); - Pt = MmCreateHyperspaceMapping(Pfn); - if (Pt == NULL) - { - KEBUGCHECK(0); - } - return Pt + PAE_ADDR_TO_PTE_OFFSET(Address); - } - PageDir = PAE_ADDR_TO_PDE(Address); - if (0LL == ExfInterlockedCompareExchange64UL(PageDir, &ZeroEntry, &ZeroEntry)) - { - if (Address >= MmSystemRangeStart) - { - if (MmGlobalKernelPageDirectoryForPAE[PAE_ADDR_TO_PDE_OFFSET(Address)] == 0LL) - { - if (Create == FALSE) - { - return NULL; - } - Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn); - if (!NT_SUCCESS(Status)) - { - KEBUGCHECK(0); - } - Entry = PAE_PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE; - if (Ke386GlobalPagesEnabled) - { - Entry |= PA_GLOBAL; - } - if (0LL != ExfInterlockedCompareExchange64UL(&MmGlobalKernelPageDirectoryForPAE[PAE_ADDR_TO_PDE_OFFSET(Address)], &Entry, &ZeroEntry)) - { - MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn); - } - } - (void)ExfInterlockedCompareExchange64UL(PageDir, &MmGlobalKernelPageDirectoryForPAE[PAE_ADDR_TO_PDE_OFFSET(Address)], &ZeroEntry); - } - else - { - if (Create == FALSE) - { - return NULL; - } - Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn); - if (!NT_SUCCESS(Status)) - { - KEBUGCHECK(0); - } - Entry = PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE | PA_USER; - Entry = ExfInterlockedCompareExchange64UL(PageDir, &Entry, &ZeroEntry); - if (Entry != 0LL) - { - MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn); - } - } - } - return (PULONGLONG)PAE_ADDR_TO_PTE(Address); -} - static PULONG MmGetPageTableForProcess(PEPROCESS Process, PVOID Address, BOOLEAN Create) { @@ -730,20 +452,11 @@ MmGetPageTableForProcess(PEPROCESS Process, PVOID Address, BOOLEAN Create) BOOLEAN MmUnmapPageTable(PULONG Pt) { - if (Ke386Pae) - { - if ((PULONGLONG)Pt >= (PULONGLONG)PAGETABLE_MAP && (PULONGLONG)Pt < (PULONGLONG)PAGETABLE_MAP + 4*512*512) - { - return TRUE; - } - } - else - { if (Pt >= (PULONG)PAGETABLE_MAP && Pt < (PULONG)PAGETABLE_MAP + 1024*1024) { return TRUE; } - } + if (Pt) { MmDeleteHyperspaceMapping((PVOID)PAGE_ROUND_DOWN(Pt)); @@ -751,21 +464,6 @@ BOOLEAN MmUnmapPageTable(PULONG Pt) return FALSE; } -static ULONGLONG MmGetPageEntryForProcessForPAE(PEPROCESS Process, PVOID Address) -{ - ULONGLONG Pte; - PULONGLONG Pt; - - Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE); - if (Pt) - { - Pte = *Pt; - MmUnmapPageTable((PULONG)Pt); - return Pte; - } - return 0; -} - static ULONG MmGetPageEntryForProcess(PEPROCESS Process, PVOID Address) { ULONG Pte; @@ -786,19 +484,6 @@ NTAPI MmGetPfnForProcess(PEPROCESS Process, PVOID Address) { - - if (Ke386Pae) - { - ULONGLONG Entry; - Entry = MmGetPageEntryForProcessForPAE(Process, Address); - if (!(Entry & PA_PRESENT)) - { - return 0; - } - return(PAE_PTE_TO_PFN(Entry)); - } - else - { ULONG Entry; Entry = MmGetPageEntryForProcess(Process, Address); if (!(Entry & PA_PRESENT)) @@ -806,7 +491,6 @@ MmGetPfnForProcess(PEPROCESS Process, return 0; } return(PTE_TO_PFN(Entry)); - } } VOID @@ -817,47 +501,6 @@ MmDisableVirtualMapping(PEPROCESS Process, PVOID Address, BOOLEAN* WasDirty, PPF */ { BOOLEAN WasValid; - if (Ke386Pae) - { - ULONGLONG Pte; - ULONGLONG tmpPte; - PULONGLONG Pt; - - Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE); - if (Pt == NULL) - { - KEBUGCHECK(0); - } - /* - * Atomically disable the present bit and get the old value. - */ - do - { - Pte = *Pt; - tmpPte = Pte & ~PA_PRESENT; - } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte)); - - MiFlushTlb((PULONG)Pt, Address); - WasValid = PAE_PAGE_MASK(Pte) != 0LL ? TRUE : FALSE; - if (!WasValid) - { - KEBUGCHECK(0); - } - - /* - * Return some information to the caller - */ - if (WasDirty != NULL) - { - *WasDirty = Pte & PA_DIRTY ? TRUE : FALSE; - } - if (Page != NULL) - { - *Page = PAE_PTE_TO_PFN(Pte); - } - } - else - { ULONG Pte; PULONG Pt; @@ -892,29 +535,12 @@ MmDisableVirtualMapping(PEPROCESS Process, PVOID Address, BOOLEAN* WasDirty, PPF { *Page = PTE_TO_PFN(Pte); } - } } VOID NTAPI MmRawDeleteVirtualMapping(PVOID Address) { - if (Ke386Pae) - { - PULONGLONG Pt; - ULONGLONG ZeroPte = 0LL; - Pt = MmGetPageTableForProcessForPAE(NULL, Address, FALSE); - if (Pt) - { - /* - * Set the entry to zero - */ - (void)ExfpInterlockedExchange64UL(Pt, &ZeroPte); - MiFlushTlb((PULONG)Pt, Address); - } - } - else - { PULONG Pt; Pt = MmGetPageTableForProcess(NULL, Address, FALSE); @@ -926,7 +552,6 @@ MmRawDeleteVirtualMapping(PVOID Address) (void)InterlockedExchangeUL(Pt, 0); MiFlushTlb(Pt, Address); } - } } VOID @@ -942,63 +567,6 @@ MmDeleteVirtualMapping(PEPROCESS Process, PVOID Address, BOOLEAN FreePage, DPRINT("MmDeleteVirtualMapping(%x, %x, %d, %x, %x)\n", Process, Address, FreePage, WasDirty, Page); - if (Ke386Pae) - { - ULONGLONG Pte; - PULONGLONG Pt; - - Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE); - if (Pt == NULL) - { - if (WasDirty != NULL) - { - *WasDirty = FALSE; - } - if (Page != NULL) - { - *Page = 0; - } - return; - } - - /* - * Atomically set the entry to zero and get the old value. - */ - Pte = 0LL; - Pte = ExfpInterlockedExchange64UL(Pt, &Pte); - - MiFlushTlb((PULONG)Pt, Address); - - WasValid = PAE_PAGE_MASK(Pte) != 0 ? TRUE : FALSE; - if (WasValid) - { - Pfn = PAE_PTE_TO_PFN(Pte); - MmMarkPageUnmapped(Pfn); - } - else - { - Pfn = 0; - } - - if (FreePage && WasValid) - { - MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn); - } - - /* - * Return some information to the caller - */ - if (WasDirty != NULL) - { - *WasDirty = Pte & PA_DIRTY ? TRUE : FALSE; - } - if (Page != NULL) - { - *Page = Pfn; - } - } - else - { ULONG Pte; PULONG Pt; @@ -1051,7 +619,6 @@ MmDeleteVirtualMapping(PEPROCESS Process, PVOID Address, BOOLEAN FreePage, { *Page = Pfn; } - } /* * Decrement the reference count for this page table. */ @@ -1063,7 +630,7 @@ MmDeleteVirtualMapping(PEPROCESS Process, PVOID Address, BOOLEAN FreePage, ULONG Idx; Ptrc = ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable; - Idx = Ke386Pae ? PAE_ADDR_TO_PAGE_TABLE(Address) : ADDR_TO_PAGE_TABLE(Address); + Idx = ADDR_TO_PAGE_TABLE(Address); Ptrc[Idx]--; if (Ptrc[Idx] == 0) @@ -1081,52 +648,6 @@ MmDeletePageFileMapping(PEPROCESS Process, PVOID Address, * FUNCTION: Delete a virtual mapping */ { - if (Ke386Pae) - { - ULONGLONG Pte; - PULONGLONG Pt; - - Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE); - if (Pt == NULL) - { - *SwapEntry = 0; - return; - } - - /* - * Atomically set the entry to zero and get the old value. - */ - Pte = 0LL; - Pte = ExfpInterlockedExchange64UL(Pt, &Pte); - - MiFlushTlb((PULONG)Pt, Address); - - /* - * Decrement the reference count for this page table. - */ - if (Process != NULL && Pte && - ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable != NULL && - Address < MmSystemRangeStart) - { - PUSHORT Ptrc; - - Ptrc = ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable; - - Ptrc[PAE_ADDR_TO_PAGE_TABLE(Address)]--; - if (Ptrc[PAE_ADDR_TO_PAGE_TABLE(Address)] == 0) - { - MmFreePageTable(Process, Address); - } - } - - - /* - * Return some information to the caller - */ - *SwapEntry = Pte >> 1; - } - else - { ULONG Pte; PULONG Pt; @@ -1168,47 +689,21 @@ MmDeletePageFileMapping(PEPROCESS Process, PVOID Address, * Return some information to the caller */ *SwapEntry = Pte >> 1; - } } BOOLEAN Mmi386MakeKernelPageTableGlobal(PVOID PAddress) { - if (Ke386Pae) - { - PULONGLONG Pt; - PULONGLONG Pde; - Pde = PAE_ADDR_TO_PDE(PAddress); - if (*Pde == 0LL) - { - Pt = MmGetPageTableForProcessForPAE(NULL, PAddress, FALSE); -#if 0 - /* Non existing mappings are not cached within the tlb. We must not invalidate this entry */ - FLASH_TLB_ONE(PAddress); -#endif - if (Pt != NULL) - { - return TRUE; - } - } - } - else - { PULONG Pt, Pde; Pde = ADDR_TO_PDE(PAddress); if (*Pde == 0) { Pt = MmGetPageTableForProcess(NULL, PAddress, FALSE); -#if 0 - /* Non existing mappings are not cached within the tlb. We must not invalidate this entry */ - FLASH_TLB_ONE(PAddress); -#endif if (Pt != NULL) { return TRUE; } } - } return(FALSE); } @@ -1216,14 +711,7 @@ BOOLEAN NTAPI MmIsDirtyPage(PEPROCESS Process, PVOID Address) { - if (Ke386Pae) - { - return MmGetPageEntryForProcessForPAE(Process, Address) & PA_DIRTY ? TRUE : FALSE; - } - else - { return MmGetPageEntryForProcess(Process, Address) & PA_DIRTY ? TRUE : FALSE; - } } BOOLEAN @@ -1235,37 +723,7 @@ MmIsAccessedAndResetAccessPage(PEPROCESS Process, PVOID Address) DPRINT1("MmIsAccessedAndResetAccessPage is called for user space without a process.\n"); KEBUGCHECK(0); } - if (Ke386Pae) - { - PULONGLONG Pt; - ULONGLONG Pte; - ULONGLONG tmpPte; - Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE); - if (Pt == NULL) - { - KEBUGCHECK(0); - } - - do - { - Pte = *Pt; - tmpPte = Pte & ~PA_ACCESSED; - } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte)); - - if (Pte & PA_ACCESSED) - { - MiFlushTlb((PULONG)Pt, Address); - return TRUE; - } - else - { - MmUnmapPageTable((PULONG)Pt); - return FALSE; - } - } - else - { PULONG Pt; ULONG Pte; @@ -1290,7 +748,6 @@ MmIsAccessedAndResetAccessPage(PEPROCESS Process, PVOID Address) MmUnmapPageTable(Pt); return FALSE; } - } } VOID @@ -1302,36 +759,7 @@ MmSetCleanPage(PEPROCESS Process, PVOID Address) DPRINT1("MmSetCleanPage is called for user space without a process.\n"); KEBUGCHECK(0); } - if (Ke386Pae) - { - PULONGLONG Pt; - ULONGLONG Pte; - ULONGLONG tmpPte; - Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE); - - if (Pt == NULL) - { - KEBUGCHECK(0); - } - - do - { - Pte = *Pt; - tmpPte = Pte & ~PA_DIRTY; - } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte)); - - if (Pte & PA_DIRTY) - { - MiFlushTlb((PULONG)Pt, Address); - } - else - { - MmUnmapPageTable((PULONG)Pt); - } - } - else - { PULONG Pt; ULONG Pte; @@ -1355,7 +783,6 @@ MmSetCleanPage(PEPROCESS Process, PVOID Address) { MmUnmapPageTable(Pt); } - } } VOID @@ -1367,34 +794,6 @@ MmSetDirtyPage(PEPROCESS Process, PVOID Address) DPRINT1("MmSetDirtyPage is called for user space without a process.\n"); KEBUGCHECK(0); } - if (Ke386Pae) - { - PULONGLONG Pt; - ULONGLONG Pte; - ULONGLONG tmpPte; - - Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE); - if (Pt == NULL) - { - KEBUGCHECK(0); - } - - do - { - Pte = *Pt; - tmpPte = Pte | PA_DIRTY; - } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte)); - if (!(Pte & PA_DIRTY)) - { - MiFlushTlb((PULONG)Pt, Address); - } - else - { - MmUnmapPageTable((PULONG)Pt); - } - } - else - { PULONG Pt; ULONG Pte; @@ -1416,41 +815,12 @@ MmSetDirtyPage(PEPROCESS Process, PVOID Address) { MmUnmapPageTable(Pt); } - } } VOID NTAPI MmEnableVirtualMapping(PEPROCESS Process, PVOID Address) { - if (Ke386Pae) - { - PULONGLONG Pt; - ULONGLONG Pte; - ULONGLONG tmpPte; - - Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE); - if (Pt == NULL) - { - KEBUGCHECK(0); - } - - do - { - Pte = *Pt; - tmpPte = Pte | PA_PRESENT; - } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte)); - if (!(Pte & PA_PRESENT)) - { - MiFlushTlb((PULONG)Pt, Address); - } - else - { - MmUnmapPageTable((PULONG)Pt); - } - } - else - { PULONG Pt; ULONG Pte; @@ -1472,39 +842,22 @@ MmEnableVirtualMapping(PEPROCESS Process, PVOID Address) { MmUnmapPageTable(Pt); } - } } BOOLEAN NTAPI MmIsPagePresent(PEPROCESS Process, PVOID Address) { - if (Ke386Pae) - { - return MmGetPageEntryForProcessForPAE(Process, Address) & PA_PRESENT ? TRUE : FALSE; - } - else - { return MmGetPageEntryForProcess(Process, Address) & PA_PRESENT ? TRUE : FALSE; - } } BOOLEAN NTAPI MmIsPageSwapEntry(PEPROCESS Process, PVOID Address) { - if (Ke386Pae) - { - ULONGLONG Entry; - Entry = MmGetPageEntryForProcessForPAE(Process, Address); - return !(Entry & PA_PRESENT) && Entry != 0 ? TRUE : FALSE; - } - else - { ULONG Entry; Entry = MmGetPageEntryForProcess(Process, Address); return !(Entry & PA_PRESENT) && Entry != 0 ? TRUE : FALSE; - } } NTSTATUS @@ -1542,51 +895,6 @@ MmCreateVirtualMappingForKernel(PVOID Address, Addr = Address; - if (Ke386Pae) - { - PULONGLONG Pt = NULL; - ULONGLONG Pte; - - oldPdeOffset = PAE_ADDR_TO_PDE_OFFSET(Addr) + 1; - for (i = 0; i < PageCount; i++, Addr = (PVOID)((ULONG_PTR)Addr + PAGE_SIZE)) - { - if (!(Attributes & PA_PRESENT) && Pages[i] != 0) - { - DPRINT1("Setting physical address but not allowing access at address " - "0x%.8X with attributes %x/%x.\n", - Addr, Attributes, flProtect); - KEBUGCHECK(0); - } - - PdeOffset = PAE_ADDR_TO_PDE_OFFSET(Addr); - if (oldPdeOffset != PdeOffset) - { - Pt = MmGetPageTableForProcessForPAE(NULL, Addr, TRUE); - if (Pt == NULL) - { - KEBUGCHECK(0); - } - } - else - { - Pt++; - } - oldPdeOffset = PdeOffset; - - Pte = PFN_TO_PTE(Pages[i]) | Attributes; - if (NoExecute) - { - Pte |= 0x8000000000000000LL; - } - Pte = ExfpInterlockedExchange64UL(Pt, &Pte); - if (Pte != 0LL) - { - KEBUGCHECK(0); - } - } - } - else - { PULONG Pt; ULONG Pte; @@ -1630,7 +938,6 @@ MmCreateVirtualMappingForKernel(PVOID Address, } (void)InterlockedExchangeUL(Pt, PFN_TO_PTE(Pages[i]) | Attributes); } - } return(STATUS_SUCCESS); } @@ -1656,35 +963,6 @@ MmCreatePageFileMapping(PEPROCESS Process, KEBUGCHECK(0); } - if (Ke386Pae) - { - PULONGLONG Pt; - ULONGLONG Pte; - ULONGLONG tmpPte; - - Pt = MmGetPageTableForProcessForPAE(Process, Address, TRUE); - if (Pt == NULL) - { - KEBUGCHECK(0); - } - tmpPte = SwapEntry << 1; - Pte = ExfpInterlockedExchange64UL(Pt, &tmpPte); - if (PAE_PAGE_MASK((Pte)) != 0) - { - MmMarkPageUnmapped(PAE_PTE_TO_PFN((Pte))); - } - - if (Pte != 0) - { - MiFlushTlb((PULONG)Pt, Address); - } - else - { - MmUnmapPageTable((PULONG)Pt); - } - } - else - { PULONG Pt; ULONG Pte; @@ -1707,7 +985,7 @@ MmCreatePageFileMapping(PEPROCESS Process, { MmUnmapPageTable(Pt); } - } + if (Process != NULL && ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable != NULL && Address < MmSystemRangeStart) @@ -1716,7 +994,7 @@ MmCreatePageFileMapping(PEPROCESS Process, ULONG Idx; Ptrc = ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable; - Idx = Ke386Pae ? PAE_ADDR_TO_PAGE_TABLE(Address) : ADDR_TO_PAGE_TABLE(Address); + Idx = ADDR_TO_PAGE_TABLE(Address); Ptrc[Idx]++; } return(STATUS_SUCCESS); @@ -1791,79 +1069,7 @@ MmCreateVirtualMappingUnsafe(PEPROCESS Process, Addr = Address; - if (Ke386Pae) - { - ULONGLONG Pte, tmpPte; - PULONGLONG Pt = NULL; - - oldPdeOffset = PAE_ADDR_TO_PDE_OFFSET(Addr) + 1; - for (i = 0; i < PageCount; i++, Addr = (PVOID)((ULONG_PTR)Addr + PAGE_SIZE)) - { - if (!(Attributes & PA_PRESENT) && Pages[i] != 0) - { - DPRINT1("Setting physical address but not allowing access at address " - "0x%.8X with attributes %x/%x.\n", - Addr, Attributes, flProtect); - KEBUGCHECK(0); - } - PdeOffset = PAE_ADDR_TO_PDE_OFFSET(Addr); - if (oldPdeOffset != PdeOffset) - { - MmUnmapPageTable((PULONG)Pt); - Pt = MmGetPageTableForProcessForPAE(Process, Addr, TRUE); - if (Pt == NULL) - { - KEBUGCHECK(0); - } - } - else - { - Pt++; - } - oldPdeOffset = PdeOffset; - - MmMarkPageMapped(Pages[i]); - tmpPte = PAE_PFN_TO_PTE(Pages[i]) | Attributes; - if (NoExecute) - { - tmpPte |= 0x8000000000000000LL; - } - Pte = ExfpInterlockedExchange64UL(Pt, &tmpPte); - if (PAE_PAGE_MASK((Pte)) != 0LL && !((Pte) & PA_PRESENT)) - { - KEBUGCHECK(0); - } - if (PAE_PAGE_MASK((Pte)) != 0LL) - { - MmMarkPageUnmapped(PAE_PTE_TO_PFN((Pte))); - } - if (Address < MmSystemRangeStart && - ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable != NULL && - Attributes & PA_PRESENT) - { - PUSHORT Ptrc; - - Ptrc = ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable; - - Ptrc[PAE_ADDR_TO_PAGE_TABLE(Addr)]++; - } - if (Pte != 0LL) - { - if (Address > MmSystemRangeStart || - (Pt >= (PULONGLONG)PAGETABLE_MAP && Pt < (PULONGLONG)PAGETABLE_MAP + 4*512*512)) - { - MiFlushTlb((PULONG)Pt, Address); - } - } - } - if (Addr > Address) - { - MmUnmapPageTable((PULONG)Pt); - } - } - else - { - PULONG Pt = NULL; + PULONG Pt = NULL; ULONG Pte; oldPdeOffset = ADDR_TO_PDE_OFFSET(Addr) + 1; for (i = 0; i < PageCount; i++, Addr = (PVOID)((ULONG_PTR)Addr + PAGE_SIZE)) @@ -1925,7 +1131,7 @@ MmCreateVirtualMappingUnsafe(PEPROCESS Process, { MmUnmapPageTable(Pt); } - } + return(STATUS_SUCCESS); } @@ -1961,14 +1167,9 @@ MmGetPageProtect(PEPROCESS Process, PVOID Address) { ULONG Entry; ULONG Protect; - if (Ke386Pae) - { - Entry = MmGetPageEntryForProcessForPAE(Process, Address); - } - else - { + Entry = MmGetPageEntryForProcess(Process, Address); - } + if (!(Entry & PA_PRESENT)) { @@ -2029,35 +1230,6 @@ MmSetPageProtect(PEPROCESS Process, PVOID Address, ULONG flProtect) { Attributes |= PA_USER; } - if (Ke386Pae) - { - PULONGLONG Pt; - ULONGLONG tmpPte, Pte; - - Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE); - if (Pt == NULL) - { - DPRINT1("Address %x\n", Address); - KEBUGCHECK(0); - } - do - { - Pte = *Pt; - tmpPte = PAE_PAGE_MASK(Pte) | Attributes | (Pte & (PA_ACCESSED|PA_DIRTY)); - if (NoExecute) - { - tmpPte |= 0x8000000000000000LL; - } - else - { - tmpPte &= ~0x8000000000000000LL; - } - } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte)); - - MiFlushTlb((PULONG)Pt, Address); - } - else - { PULONG Pt; Pt = MmGetPageTableForProcess(Process, Address, FALSE); @@ -2067,7 +1239,6 @@ MmSetPageProtect(PEPROCESS Process, PVOID Address, ULONG flProtect) } InterlockedExchange((PLONG)Pt, PAGE_MASK(*Pt) | Attributes | (*Pt & (PA_ACCESSED|PA_DIRTY))); MiFlushTlb(Pt, Address); - } } /* @@ -2082,22 +1253,6 @@ MmGetPhysicalAddress(PVOID vaddr) PHYSICAL_ADDRESS p; DPRINT("MmGetPhysicalAddress(vaddr %x)\n", vaddr); - if (Ke386Pae) - { - ULONGLONG Pte; - Pte = MmGetPageEntryForProcessForPAE(NULL, vaddr); - if (Pte != 0 && Pte & PA_PRESENT) - { - p.QuadPart = PAE_PAGE_MASK(Pte); - p.u.LowPart |= (ULONG_PTR)vaddr & (PAGE_SIZE - 1); - } - else - { - p.QuadPart = 0; - } - } - else - { ULONG Pte; Pte = MmGetPageEntryForProcess(NULL, vaddr); if (Pte != 0 && Pte & PA_PRESENT) @@ -2109,7 +1264,6 @@ MmGetPhysicalAddress(PVOID vaddr) { p.QuadPart = 0; } - } return p; } @@ -2120,68 +1274,6 @@ MmCreateHyperspaceMapping(PFN_TYPE Page) PVOID Address; ULONG i; - if (Ke386Pae) - { - ULONGLONG Entry; - ULONGLONG ZeroEntry = 0LL; - PULONGLONG Pte; - - Entry = PFN_TO_PTE(Page) | PA_PRESENT | PA_READWRITE; - Pte = PAE_ADDR_TO_PTE(HYPERSPACE) + Page % 1024; - - if (Page & 1024) - { - for (i = Page %1024; i < 1024; i++, Pte++) - { - if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry)) - { - break; - } - } - if (i >= 1024) - { - Pte = PAE_ADDR_TO_PTE(HYPERSPACE); - for (i = 0; i < Page % 1024; i++, Pte++) - { - if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry)) - { - break; - } - } - if (i >= Page % 1024) - { - KEBUGCHECK(0); - } - } - } - else - { - for (i = Page %1024; (LONG)i >= 0; i--, Pte--) - { - if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry)) - { - break; - } - } - if ((LONG)i < 0) - { - Pte = PAE_ADDR_TO_PTE(HYPERSPACE) + 1023; - for (i = 1023; i > Page % 1024; i--, Pte--) - { - if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry)) - { - break; - } - } - if (i <= Page % 1024) - { - KEBUGCHECK(0); - } - } - } - } - else - { ULONG Entry; PULONG Pte; Entry = PFN_TO_PTE(Page) | PA_PRESENT | PA_READWRITE; @@ -2236,7 +1328,6 @@ MmCreateHyperspaceMapping(PFN_TYPE Page) } } } - } Address = (PVOID)((ULONG_PTR)HYPERSPACE + i * PAGE_SIZE); __invlpg(Address); return Address; @@ -2248,18 +1339,11 @@ MmChangeHyperspaceMapping(PVOID Address, PFN_TYPE NewPage) { PFN_TYPE Pfn; ASSERT (IS_HYPERSPACE(Address)); - if (Ke386Pae) - { - ULONGLONG Entry = PAE_PFN_TO_PTE(NewPage) | PA_PRESENT | PA_READWRITE; - Entry = (ULONG)ExfpInterlockedExchange64UL(PAE_ADDR_TO_PTE(Address), &Entry); - Pfn = PAE_PTE_TO_PFN(Entry); - } - else - { + ULONG Entry; Entry = InterlockedExchange((PLONG)ADDR_TO_PTE(Address), PFN_TO_PTE(NewPage) | PA_PRESENT | PA_READWRITE); Pfn = PTE_TO_PFN(Entry); - } + __invlpg(Address); return Pfn; } @@ -2270,18 +1354,11 @@ MmDeleteHyperspaceMapping(PVOID Address) { PFN_TYPE Pfn; ASSERT (IS_HYPERSPACE(Address)); - if (Ke386Pae) - { - ULONGLONG Entry = 0LL; - Entry = (ULONG)ExfpInterlockedExchange64UL(PAE_ADDR_TO_PTE(Address), &Entry); - Pfn = PAE_PTE_TO_PFN(Entry); - } - else - { + ULONG Entry; Entry = InterlockedExchange((PLONG)ADDR_TO_PTE(Address), 0); Pfn = PTE_TO_PFN(Entry); - } + __invlpg(Address); return Pfn; } @@ -2296,55 +1373,7 @@ MmUpdatePageDir(PEPROCESS Process, PVOID Address, ULONG Size) { KEBUGCHECK(0); } - if (Ke386Pae) - { - PULONGLONG PageDirTable; - PULONGLONG Pde; - ULONGLONG ZeroPde = 0LL; - ULONG i; - for (i = PAE_ADDR_TO_PDTE_OFFSET(Address); i <= PAE_ADDR_TO_PDTE_OFFSET((PVOID)((ULONG_PTR)Address + Size)); i++) - { - if (i == PAE_ADDR_TO_PDTE_OFFSET(Address)) - { - StartOffset = PAE_ADDR_TO_PDE_PAGE_OFFSET(Address); - } - else - { - StartOffset = 0; - } - if (i == PAE_ADDR_TO_PDTE_OFFSET((PVOID)((ULONG_PTR)Address + Size))) - { - EndOffset = PAE_ADDR_TO_PDE_PAGE_OFFSET((PVOID)((ULONG_PTR)Address + Size)); - } - else - { - EndOffset = 511; - } - - if (Process != NULL && Process != PsGetCurrentProcess()) - { - PageDirTable = MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart)); - Pde = (PULONGLONG)MmCreateHyperspaceMapping(PTE_TO_PFN(PageDirTable[i])); - MmDeleteHyperspaceMapping(PageDirTable); - } - else - { - Pde = (PULONGLONG)PAE_PAGEDIRECTORY_MAP + i*512; - } - - for (Offset = StartOffset; Offset <= EndOffset; Offset++) - { - if (i * 512 + Offset < PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) || i * 512 + Offset >= PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP)+4) - { - (void)ExfInterlockedCompareExchange64UL(&Pde[Offset], &MmGlobalKernelPageDirectoryForPAE[i*512 + Offset], &ZeroPde); - } - } - MmUnmapPageTable((PULONG)Pde); - } - } - else - { PULONG Pde; StartOffset = ADDR_TO_PDE_OFFSET(Address); EndOffset = ADDR_TO_PDE_OFFSET((PVOID)((ULONG_PTR)Address + Size)); @@ -2368,7 +1397,6 @@ MmUpdatePageDir(PEPROCESS Process, PVOID Address, ULONG Size) { MmDeleteHyperspaceMapping(Pde); } - } } VOID @@ -2380,26 +1408,6 @@ MmInitGlobalKernelPageDirectory(VOID) DPRINT("MmInitGlobalKernelPageDirectory()\n"); - if (Ke386Pae) - { - PULONGLONG CurrentPageDirectory = (PULONGLONG)PAE_PAGEDIRECTORY_MAP; - for (i = PAE_ADDR_TO_PDE_OFFSET(MmSystemRangeStart); i < 4 * 512; i++) - { - if (!(i >= PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) && i < PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) + 4) && - !(i >= PAE_ADDR_TO_PDE_OFFSET(HYPERSPACE) && i < PAE_ADDR_TO_PDE_OFFSET(HYPERSPACE) + 2) && - 0LL == MmGlobalKernelPageDirectoryForPAE[i] && 0LL != CurrentPageDirectory[i]) - { - (void)ExfpInterlockedExchange64UL(&MmGlobalKernelPageDirectoryForPAE[i], &CurrentPageDirectory[i]); - if (Ke386GlobalPagesEnabled) - { - MmGlobalKernelPageDirectoryForPAE[i] |= PA_GLOBAL; - CurrentPageDirectory[i] |= PA_GLOBAL; - } - } - } - } - else - { PULONG CurrentPageDirectory = (PULONG)PAGEDIRECTORY_MAP; for (i = ADDR_TO_PDE_OFFSET(MmSystemRangeStart); i < 1024; i++) { @@ -2415,14 +1423,13 @@ MmInitGlobalKernelPageDirectory(VOID) } } } - } } ULONG NTAPI MiGetUserPageDirectoryCount(VOID) { - return Ke386Pae ? PAE_ADDR_TO_PDE_OFFSET(MmSystemRangeStart) : ADDR_TO_PDE_OFFSET(MmSystemRangeStart); + return ADDR_TO_PDE_OFFSET(MmSystemRangeStart); } VOID @@ -2443,7 +1450,7 @@ MiInitPageDirectoryMap(VOID) Status = MmCreateMemoryArea(MmGetKernelAddressSpace(), MEMORY_AREA_SYSTEM, &BaseAddress, - Ke386Pae ? 0x800000 : 0x400000, + 0x400000, PAGE_READWRITE, &kernel_map_desc, TRUE, diff --git a/reactos/ntoskrnl/mm/i386/pagepae.c b/reactos/ntoskrnl/mm/i386/pagepae.c new file mode 100644 index 00000000000..5e13e7830aa --- /dev/null +++ b/reactos/ntoskrnl/mm/i386/pagepae.c @@ -0,0 +1,2472 @@ +/* $Id: page.c 32364 2008-02-14 20:30:31Z ros-arm-bringup $ + * + * COPYRIGHT: See COPYING in the top level directory + * PROJECT: ReactOS kernel + * FILE: ntoskrnl/mm/i386/page.c + * PURPOSE: Low level memory managment manipulation + * + * PROGRAMMERS: David Welch (welch@cwcom.net) + */ + +/* INCLUDES ***************************************************************/ + +#include +#define NDEBUG +#include + +#if defined (ALLOC_PRAGMA) +#pragma alloc_text(INIT, MmInitGlobalKernelPageDirectory) +#pragma alloc_text(INIT, MiInitPageDirectoryMap) +#endif + + +/* GLOBALS *****************************************************************/ + +#define PA_BIT_PRESENT (0) +#define PA_BIT_READWRITE (1) +#define PA_BIT_USER (2) +#define PA_BIT_WT (3) +#define PA_BIT_CD (4) +#define PA_BIT_ACCESSED (5) +#define PA_BIT_DIRTY (6) +#define PA_BIT_GLOBAL (8) + +#define PA_PRESENT (1 << PA_BIT_PRESENT) +#define PA_READWRITE (1 << PA_BIT_READWRITE) +#define PA_USER (1 << PA_BIT_USER) +#define PA_DIRTY (1 << PA_BIT_DIRTY) +#define PA_WT (1 << PA_BIT_WT) +#define PA_CD (1 << PA_BIT_CD) +#define PA_ACCESSED (1 << PA_BIT_ACCESSED) +#define PA_GLOBAL (1 << PA_BIT_GLOBAL) + +#define PAGETABLE_MAP (0xc0000000) +#define PAGEDIRECTORY_MAP (0xc0000000 + (PAGETABLE_MAP / (1024))) + +#define PAE_PAGEDIRECTORY_MAP (0xc0000000 + (PAGETABLE_MAP / (512))) + +#define HYPERSPACE (Ke386Pae ? 0xc0800000 : 0xc0400000) +#define IS_HYPERSPACE(v) (((ULONG)(v) >= HYPERSPACE && (ULONG)(v) < HYPERSPACE + 0x400000)) + +ULONG MmGlobalKernelPageDirectory[1024]; +ULONGLONG MmGlobalKernelPageDirectoryForPAE[2048]; + +#define PTE_TO_PFN(X) ((X) >> PAGE_SHIFT) +#define PFN_TO_PTE(X) ((X) << PAGE_SHIFT) + +#define PAE_PTE_TO_PFN(X) (PAE_PAGE_MASK(X) >> PAGE_SHIFT) +#define PAE_PFN_TO_PTE(X) ((X) << PAGE_SHIFT) + +#if defined(__GNUC__) +#define PTE_TO_PAGE(X) ((LARGE_INTEGER)(LONGLONG)(PAGE_MASK(X))) +#else +__inline LARGE_INTEGER PTE_TO_PAGE(ULONG npage) +{ + LARGE_INTEGER dummy; + dummy.QuadPart = (LONGLONG)(PAGE_MASK(npage)); + return dummy; +} +#endif + +extern BOOLEAN Ke386Pae; +extern BOOLEAN Ke386NoExecute; + +/* FUNCTIONS ***************************************************************/ + +BOOLEAN MmUnmapPageTable(PULONG Pt); + +ULONG_PTR +NTAPI +MiFlushTlbIpiRoutine(ULONG_PTR Address) +{ + if (Address == (ULONG_PTR)-1) + { + KeFlushCurrentTb(); + } + else if (Address == (ULONG_PTR)-2) + { + KeFlushCurrentTb(); + } + else + { + __invlpg((PVOID)Address); + } + return 0; +} + +VOID +MiFlushTlb(PULONG Pt, PVOID Address) +{ +#ifdef CONFIG_SMP + if (Pt) + { + MmUnmapPageTable(Pt); + } + if (KeNumberProcessors > 1) + { + KeIpiGenericCall(MiFlushTlbIpiRoutine, (ULONG_PTR)Address); + } + else + { + MiFlushTlbIpiRoutine((ULONG_PTR)Address); + } +#else + if ((Pt && MmUnmapPageTable(Pt)) || Address >= MmSystemRangeStart) + { + __invlpg(Address); + } +#endif +} + + + +PULONG +MmGetPageDirectory(VOID) +{ + return (PULONG)__readcr3(); +} + +static ULONG +ProtectToPTE(ULONG flProtect) +{ + ULONG Attributes = 0; + + if (flProtect & (PAGE_NOACCESS|PAGE_GUARD)) + { + Attributes = 0; + } + else if (flProtect & PAGE_IS_WRITABLE) + { + Attributes = PA_PRESENT | PA_READWRITE; + } + else if (flProtect & (PAGE_IS_READABLE | PAGE_IS_EXECUTABLE)) + { + Attributes = PA_PRESENT; + } + else + { + DPRINT1("Unknown main protection type.\n"); + KEBUGCHECK(0); + } + if (Ke386NoExecute && + !(flProtect & PAGE_IS_EXECUTABLE)) + { + Attributes = Attributes | 0x80000000; + } + + if (flProtect & PAGE_SYSTEM) + { + } + else + { + Attributes = Attributes | PA_USER; + } + if (flProtect & PAGE_NOCACHE) + { + Attributes = Attributes | PA_CD; + } + if (flProtect & PAGE_WRITETHROUGH) + { + Attributes = Attributes | PA_WT; + } + return(Attributes); +} + +#define ADDR_TO_PAGE_TABLE(v) (((ULONG)(v)) / (1024 * PAGE_SIZE)) + +#define ADDR_TO_PDE(v) (PULONG)(PAGEDIRECTORY_MAP + \ + ((((ULONG)(v)) / (1024 * 1024))&(~0x3))) +#define ADDR_TO_PTE(v) (PULONG)(PAGETABLE_MAP + ((((ULONG)(v) / 1024))&(~0x3))) + +#define ADDR_TO_PDE_OFFSET(v) ((((ULONG)(v)) / (1024 * PAGE_SIZE))) + +#define ADDR_TO_PTE_OFFSET(v) ((((ULONG)(v)) % (1024 * PAGE_SIZE)) / PAGE_SIZE) + + +#define PAE_ADDR_TO_PAGE_TABLE(v) (((ULONG)(v)) / (512 * PAGE_SIZE)) + +#define PAE_ADDR_TO_PDE(v) (PULONGLONG) (PAE_PAGEDIRECTORY_MAP + \ + ((((ULONG_PTR)(v)) / (512 * 512))&(~0x7))) +#define PAE_ADDR_TO_PTE(v) (PULONGLONG) (PAGETABLE_MAP + ((((ULONG_PTR)(v) / 512))&(~0x7))) + + +#define PAE_ADDR_TO_PDTE_OFFSET(v) (((ULONG_PTR)(v)) / (512 * 512 * PAGE_SIZE)) + +#define PAE_ADDR_TO_PDE_PAGE_OFFSET(v) ((((ULONG_PTR)(v)) % (512 * 512 * PAGE_SIZE)) / (512 * PAGE_SIZE)) + +#define PAE_ADDR_TO_PDE_OFFSET(v) (((ULONG_PTR)(v))/ (512 * PAGE_SIZE)) + +#define PAE_ADDR_TO_PTE_OFFSET(v) ((((ULONG_PTR)(v)) % (512 * PAGE_SIZE)) / PAGE_SIZE) + + +NTSTATUS +NTAPI +Mmi386ReleaseMmInfo(PEPROCESS Process) +{ + PUSHORT LdtDescriptor; + ULONG LdtBase; + ULONG i, j; + + DPRINT("Mmi386ReleaseMmInfo(Process %x)\n",Process); + + LdtDescriptor = (PUSHORT) &Process->Pcb.LdtDescriptor; + LdtBase = LdtDescriptor[1] | + ((LdtDescriptor[2] & 0xff) << 16) | + ((LdtDescriptor[3] & ~0xff) << 16); + + DPRINT("LdtBase: %x\n", LdtBase); + + if (LdtBase) + { + ExFreePool((PVOID) LdtBase); + } + + if (Ke386Pae) + { + PULONGLONG PageDirTable; + PULONGLONG PageDir; + PULONGLONG Pde; + ULONG k; + + PageDirTable = (PULONGLONG)MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart)); + for (i = 0; i < 4; i++) + { + PageDir = (PULONGLONG)MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(PageDirTable[i])); + if (i < PAE_ADDR_TO_PDTE_OFFSET(MmSystemRangeStart)) + { + for (j = 0; j < 512; j++) + { + if (PageDir[j] != 0LL) + { + DPRINT1("ProcessId %d, Pde for %08x - %08x is not freed, RefCount %d\n", + Process->UniqueProcessId, + (i * 512 + j) * 512 * PAGE_SIZE, (i * 512 + j + 1) * 512 * PAGE_SIZE - 1, + ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable[i*512 + j]); + Pde = MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(PageDir[j])); + for (k = 0; k < 512; k++) + { + if(Pde[k] != 0) + { + if (Pde[k] & PA_PRESENT) + { + DPRINT1("Page at %08x is not freed\n", + (i * 512 + j) * 512 * PAGE_SIZE + k * PAGE_SIZE); + } + else + { + DPRINT1("Swapentry %x at %x is not freed\n", + (i * 512 + j) * 512 * PAGE_SIZE + k * PAGE_SIZE); + } + } + } + MmDeleteHyperspaceMapping(Pde); + MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(PageDir[j])); + } + } + } + if (i == PAE_ADDR_TO_PDTE_OFFSET(HYPERSPACE)) + { + MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE)])); + MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE)+1])); + } + MmDeleteHyperspaceMapping(PageDir); + MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(PageDirTable[i])); + } + MmDeleteHyperspaceMapping((PVOID)PageDirTable); + MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart)); + } + else + { + PULONG Pde; + PULONG PageDir; + PageDir = MmCreateHyperspaceMapping(PTE_TO_PFN(Process->Pcb.DirectoryTableBase.u.LowPart)); + for (i = 0; i < ADDR_TO_PDE_OFFSET(MmSystemRangeStart); i++) + { + if (PageDir[i] != 0) + { + DPRINT1("Pde for %08x - %08x is not freed, RefCount %d\n", + i * 4 * 1024 * 1024, (i + 1) * 4 * 1024 * 1024 - 1, + ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable[i]); + Pde = MmCreateHyperspaceMapping(PTE_TO_PFN(PageDir[i])); + for (j = 0; j < 1024; j++) + { + if(Pde[j] != 0) + { + if (Pde[j] & PA_PRESENT) + { + DPRINT1("Page at %08x is not freed\n", + i * 4 * 1024 * 1024 + j * PAGE_SIZE); + } + else + { + DPRINT1("Swapentry %x at %x is not freed\n", + Pde[j], i * 4 * 1024 * 1024 + j * PAGE_SIZE); + } + } + } + MmDeleteHyperspaceMapping(Pde); + MmReleasePageMemoryConsumer(MC_NPPOOL, PTE_TO_PFN(PageDir[i])); + } + } + MmReleasePageMemoryConsumer(MC_NPPOOL, PTE_TO_PFN(PageDir[ADDR_TO_PDE_OFFSET(HYPERSPACE)])); + MmDeleteHyperspaceMapping(PageDir); + MmReleasePageMemoryConsumer(MC_NPPOOL, PTE_TO_PFN(Process->Pcb.DirectoryTableBase.u.LowPart)); + } + +#if defined(__GNUC__) + + Process->Pcb.DirectoryTableBase.QuadPart = 0LL; +#else + + Process->Pcb.DirectoryTableBase.QuadPart = 0; +#endif + + DPRINT("Finished Mmi386ReleaseMmInfo()\n"); + return(STATUS_SUCCESS); +} + +NTSTATUS +NTAPI +MmInitializeHandBuiltProcess(IN PEPROCESS Process, + IN PLARGE_INTEGER DirectoryTableBase) +{ + /* Share the directory base with the idle process */ + *DirectoryTableBase = PsGetCurrentProcess()->Pcb.DirectoryTableBase; + + /* Initialize the Addresss Space */ + MmInitializeAddressSpace(Process, (PMADDRESS_SPACE)&Process->VadRoot); + + /* The process now has an address space */ + Process->HasAddressSpace = TRUE; + return STATUS_SUCCESS; +} + +BOOLEAN +STDCALL +MmCreateProcessAddressSpace(IN ULONG MinWs, + IN PEPROCESS Process, + IN PLARGE_INTEGER DirectoryTableBase) +{ + NTSTATUS Status; + ULONG i, j; + PFN_TYPE Pfn[7]; + ULONG Count; + + DPRINT("MmCopyMmInfo(Src %x, Dest %x)\n", MinWs, Process); + + Count = Ke386Pae ? 7 : 2; + + for (i = 0; i < Count; i++) + { + Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn[i]); + if (!NT_SUCCESS(Status)) + { + for (j = 0; j < i; j++) + { + MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn[j]); + } + + return FALSE; + } + } + + if (Ke386Pae) + { + PULONGLONG PageDirTable; + PULONGLONG PageDir; + + PageDirTable = MmCreateHyperspaceMapping(Pfn[0]); + for (i = 0; i < 4; i++) + { + PageDirTable[i] = PAE_PFN_TO_PTE(Pfn[1+i]) | PA_PRESENT; + } + MmDeleteHyperspaceMapping(PageDirTable); + for (i = PAE_ADDR_TO_PDTE_OFFSET(MmSystemRangeStart); i < 4; i++) + { + PageDir = (PULONGLONG)MmCreateHyperspaceMapping(Pfn[i+1]); + memcpy(PageDir, &MmGlobalKernelPageDirectoryForPAE[i * 512], 512 * sizeof(ULONGLONG)); + if (PAE_ADDR_TO_PDTE_OFFSET(PAGETABLE_MAP) == i) + { + for (j = 0; j < 4; j++) + { + PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(PAGETABLE_MAP) + j] = PAE_PFN_TO_PTE(Pfn[1+j]) | PA_PRESENT | PA_READWRITE; + } + } + if (PAE_ADDR_TO_PDTE_OFFSET(HYPERSPACE) == i) + { + PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE)] = PAE_PFN_TO_PTE(Pfn[5]) | PA_PRESENT | PA_READWRITE; + PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE)+1] = PAE_PFN_TO_PTE(Pfn[6]) | PA_PRESENT | PA_READWRITE; + } + MmDeleteHyperspaceMapping(PageDir); + } + } + else + { + PULONG PageDirectory; + PageDirectory = MmCreateHyperspaceMapping(Pfn[0]); + + memcpy(PageDirectory + ADDR_TO_PDE_OFFSET(MmSystemRangeStart), + MmGlobalKernelPageDirectory + ADDR_TO_PDE_OFFSET(MmSystemRangeStart), + (1024 - ADDR_TO_PDE_OFFSET(MmSystemRangeStart)) * sizeof(ULONG)); + + DPRINT("Addr %x\n",ADDR_TO_PDE_OFFSET(PAGETABLE_MAP)); + PageDirectory[ADDR_TO_PDE_OFFSET(PAGETABLE_MAP)] = PFN_TO_PTE(Pfn[0]) | PA_PRESENT | PA_READWRITE; + PageDirectory[ADDR_TO_PDE_OFFSET(HYPERSPACE)] = PFN_TO_PTE(Pfn[1]) | PA_PRESENT | PA_READWRITE; + + MmDeleteHyperspaceMapping(PageDirectory); + } + + DirectoryTableBase->QuadPart = PFN_TO_PTE(Pfn[0]); + DPRINT("Finished MmCopyMmInfo(): %I64x\n", DirectoryTableBase->QuadPart); + return TRUE; +} + +VOID +NTAPI +MmDeletePageTable(PEPROCESS Process, PVOID Address) +{ + PEPROCESS CurrentProcess = PsGetCurrentProcess(); + + if (Process != NULL && Process != CurrentProcess) + { + KeAttachProcess(&Process->Pcb); + } + + if (Ke386Pae) + { + ULONGLONG ZeroPde = 0LL; + (void)ExfpInterlockedExchange64UL(PAE_ADDR_TO_PDE(Address), &ZeroPde); + MiFlushTlb((PULONG)PAE_ADDR_TO_PDE(Address), PAE_ADDR_TO_PTE(Address)); + } + else + { + *(ADDR_TO_PDE(Address)) = 0; + MiFlushTlb(ADDR_TO_PDE(Address), ADDR_TO_PTE(Address)); + } + if (Address >= MmSystemRangeStart) + { + KEBUGCHECK(0); + // MmGlobalKernelPageDirectory[ADDR_TO_PDE_OFFSET(Address)] = 0; + } + if (Process != NULL && Process != CurrentProcess) + { + KeDetachProcess(); + } +} + +VOID +NTAPI +MmFreePageTable(PEPROCESS Process, PVOID Address) +{ + PEPROCESS CurrentProcess = PsGetCurrentProcess(); + ULONG i; + PFN_TYPE Pfn; + + DPRINT("ProcessId %d, Address %x\n", Process->UniqueProcessId, Address); + if (Process != NULL && Process != CurrentProcess) + { + KeAttachProcess(&Process->Pcb); + } + if (Ke386Pae) + { + PULONGLONG PageTable; + ULONGLONG ZeroPte = 0LL; + PageTable = (PULONGLONG)PAGE_ROUND_DOWN((PVOID)PAE_ADDR_TO_PTE(Address)); + for (i = 0; i < 512; i++) + { + if (PageTable[i] != 0LL) + { + DbgPrint("Page table entry not clear at %x/%x (is %I64x)\n", + ((ULONG)Address / (4*1024*1024)), i, PageTable[i]); + KEBUGCHECK(0); + } + } + Pfn = PAE_PTE_TO_PFN(*(PAE_ADDR_TO_PDE(Address))); + (void)ExfpInterlockedExchange64UL(PAE_ADDR_TO_PDE(Address), &ZeroPte); + MiFlushTlb((PULONG)PAE_ADDR_TO_PDE(Address), PAE_ADDR_TO_PTE(Address)); + } + else + { + PULONG PageTable; + PageTable = (PULONG)PAGE_ROUND_DOWN((PVOID)ADDR_TO_PTE(Address)); + for (i = 0; i < 1024; i++) + { + if (PageTable[i] != 0) + { + DbgPrint("Page table entry not clear at %x/%x (is %x)\n", + ((ULONG)Address / (4*1024*1024)), i, PageTable[i]); + KEBUGCHECK(0); + } + } + Pfn = PTE_TO_PFN(*(ADDR_TO_PDE(Address))); + *(ADDR_TO_PDE(Address)) = 0; + MiFlushTlb(ADDR_TO_PDE(Address), ADDR_TO_PTE(Address)); + } + + if (Address >= MmSystemRangeStart) + { + // MmGlobalKernelPageDirectory[ADDR_TO_PDE_OFFSET(Address)] = 0; + KEBUGCHECK(0); + } + else + { + MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn); + } + if (Process != NULL && Process != CurrentProcess) + { + KeDetachProcess(); + } +} + +static PULONGLONG +MmGetPageTableForProcessForPAE(PEPROCESS Process, PVOID Address, BOOLEAN Create) +{ + NTSTATUS Status; + PFN_TYPE Pfn; + ULONGLONG Entry; + ULONGLONG ZeroEntry = 0LL; + PULONGLONG Pt; + PULONGLONG PageDir; + PULONGLONG PageDirTable; + + DPRINT("MmGetPageTableForProcessForPAE(%x %x %d)\n", + Process, Address, Create); + if (Address >= (PVOID)PAGETABLE_MAP && Address < (PVOID)((ULONG_PTR)PAGETABLE_MAP + 0x800000)) + { + KEBUGCHECK(0); + } + if (Address < MmSystemRangeStart && Process && Process != PsGetCurrentProcess()) + { + PageDirTable = MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart)); + if (PageDirTable == NULL) + { + KEBUGCHECK(0); + } + PageDir = MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(PageDirTable[PAE_ADDR_TO_PDTE_OFFSET(Address)])); + MmDeleteHyperspaceMapping(PageDirTable); + if (PageDir == NULL) + { + KEBUGCHECK(0); + } + PageDir += PAE_ADDR_TO_PDE_PAGE_OFFSET(Address); + Entry = ExfInterlockedCompareExchange64UL(PageDir, &ZeroEntry, &ZeroEntry); + if (Entry == 0LL) + { + if (Create == FALSE) + { + MmDeleteHyperspaceMapping(PageDir); + return NULL; + } + Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn); + if (!NT_SUCCESS(Status)) + { + KEBUGCHECK(0); + } + Entry = PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE | PA_USER; + Entry = ExfInterlockedCompareExchange64UL(PageDir, &Entry, &ZeroEntry); + if (Entry != 0LL) + { + MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn); + Pfn = PAE_PTE_TO_PFN(Entry); + } + } + else + { + Pfn = PAE_PTE_TO_PFN(Entry); + } + MmDeleteHyperspaceMapping(PageDir); + Pt = MmCreateHyperspaceMapping(Pfn); + if (Pt == NULL) + { + KEBUGCHECK(0); + } + return Pt + PAE_ADDR_TO_PTE_OFFSET(Address); + } + PageDir = PAE_ADDR_TO_PDE(Address); + if (0LL == ExfInterlockedCompareExchange64UL(PageDir, &ZeroEntry, &ZeroEntry)) + { + if (Address >= MmSystemRangeStart) + { + if (MmGlobalKernelPageDirectoryForPAE[PAE_ADDR_TO_PDE_OFFSET(Address)] == 0LL) + { + if (Create == FALSE) + { + return NULL; + } + Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn); + if (!NT_SUCCESS(Status)) + { + KEBUGCHECK(0); + } + Entry = PAE_PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE; + if (Ke386GlobalPagesEnabled) + { + Entry |= PA_GLOBAL; + } + if (0LL != ExfInterlockedCompareExchange64UL(&MmGlobalKernelPageDirectoryForPAE[PAE_ADDR_TO_PDE_OFFSET(Address)], &Entry, &ZeroEntry)) + { + MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn); + } + } + (void)ExfInterlockedCompareExchange64UL(PageDir, &MmGlobalKernelPageDirectoryForPAE[PAE_ADDR_TO_PDE_OFFSET(Address)], &ZeroEntry); + } + else + { + if (Create == FALSE) + { + return NULL; + } + Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn); + if (!NT_SUCCESS(Status)) + { + KEBUGCHECK(0); + } + Entry = PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE | PA_USER; + Entry = ExfInterlockedCompareExchange64UL(PageDir, &Entry, &ZeroEntry); + if (Entry != 0LL) + { + MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn); + } + } + } + return (PULONGLONG)PAE_ADDR_TO_PTE(Address); +} + +static PULONG +MmGetPageTableForProcess(PEPROCESS Process, PVOID Address, BOOLEAN Create) +{ + ULONG PdeOffset = ADDR_TO_PDE_OFFSET(Address); + NTSTATUS Status; + PFN_TYPE Pfn; + ULONG Entry; + PULONG Pt, PageDir; + + if (Address < MmSystemRangeStart && Process && Process != PsGetCurrentProcess()) + { + PageDir = MmCreateHyperspaceMapping(PTE_TO_PFN(Process->Pcb.DirectoryTableBase.LowPart)); + if (PageDir == NULL) + { + KEBUGCHECK(0); + } + if (0 == InterlockedCompareExchangeUL(&PageDir[PdeOffset], 0, 0)) + { + if (Create == FALSE) + { + MmDeleteHyperspaceMapping(PageDir); + return NULL; + } + Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn); + if (!NT_SUCCESS(Status) || Pfn == 0) + { + KEBUGCHECK(0); + } + Entry = InterlockedCompareExchangeUL(&PageDir[PdeOffset], PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE | PA_USER, 0); + if (Entry != 0) + { + MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn); + Pfn = PTE_TO_PFN(Entry); + } + } + else + { + Pfn = PTE_TO_PFN(PageDir[PdeOffset]); + } + MmDeleteHyperspaceMapping(PageDir); + Pt = MmCreateHyperspaceMapping(Pfn); + if (Pt == NULL) + { + KEBUGCHECK(0); + } + return Pt + ADDR_TO_PTE_OFFSET(Address); + } + PageDir = ADDR_TO_PDE(Address); + if (0 == InterlockedCompareExchangeUL(PageDir, 0, 0)) + { + if (Address >= MmSystemRangeStart) + { + if (0 == InterlockedCompareExchangeUL(&MmGlobalKernelPageDirectory[PdeOffset], 0, 0)) + { + if (Create == FALSE) + { + return NULL; + } + Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn); + if (!NT_SUCCESS(Status) || Pfn == 0) + { + KEBUGCHECK(0); + } + Entry = PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE; + if (Ke386GlobalPagesEnabled) + { + Entry |= PA_GLOBAL; + } + if(0 != InterlockedCompareExchangeUL(&MmGlobalKernelPageDirectory[PdeOffset], Entry, 0)) + { + MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn); + } + } + (void)InterlockedExchangeUL(PageDir, MmGlobalKernelPageDirectory[PdeOffset]); + } + else + { + if (Create == FALSE) + { + return NULL; + } + Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn); + if (!NT_SUCCESS(Status) || Pfn == 0) + { + KEBUGCHECK(0); + } + Entry = InterlockedCompareExchangeUL(PageDir, PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE | PA_USER, 0); + if (Entry != 0) + { + MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn); + } + } + } + return (PULONG)ADDR_TO_PTE(Address); +} + +BOOLEAN MmUnmapPageTable(PULONG Pt) +{ + if (Ke386Pae) + { + if ((PULONGLONG)Pt >= (PULONGLONG)PAGETABLE_MAP && (PULONGLONG)Pt < (PULONGLONG)PAGETABLE_MAP + 4*512*512) + { + return TRUE; + } + } + else + { + if (Pt >= (PULONG)PAGETABLE_MAP && Pt < (PULONG)PAGETABLE_MAP + 1024*1024) + { + return TRUE; + } + } + if (Pt) + { + MmDeleteHyperspaceMapping((PVOID)PAGE_ROUND_DOWN(Pt)); + } + return FALSE; +} + +static ULONGLONG MmGetPageEntryForProcessForPAE(PEPROCESS Process, PVOID Address) +{ + ULONGLONG Pte; + PULONGLONG Pt; + + Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE); + if (Pt) + { + Pte = *Pt; + MmUnmapPageTable((PULONG)Pt); + return Pte; + } + return 0; +} + +static ULONG MmGetPageEntryForProcess(PEPROCESS Process, PVOID Address) +{ + ULONG Pte; + PULONG Pt; + + Pt = MmGetPageTableForProcess(Process, Address, FALSE); + if (Pt) + { + Pte = *Pt; + MmUnmapPageTable(Pt); + return Pte; + } + return 0; +} + +PFN_TYPE +NTAPI +MmGetPfnForProcess(PEPROCESS Process, + PVOID Address) +{ + + if (Ke386Pae) + { + ULONGLONG Entry; + Entry = MmGetPageEntryForProcessForPAE(Process, Address); + if (!(Entry & PA_PRESENT)) + { + return 0; + } + return(PAE_PTE_TO_PFN(Entry)); + } + else + { + ULONG Entry; + Entry = MmGetPageEntryForProcess(Process, Address); + if (!(Entry & PA_PRESENT)) + { + return 0; + } + return(PTE_TO_PFN(Entry)); + } +} + +VOID +NTAPI +MmDisableVirtualMapping(PEPROCESS Process, PVOID Address, BOOLEAN* WasDirty, PPFN_TYPE Page) +/* + * FUNCTION: Delete a virtual mapping + */ +{ + BOOLEAN WasValid; + if (Ke386Pae) + { + ULONGLONG Pte; + ULONGLONG tmpPte; + PULONGLONG Pt; + + Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE); + if (Pt == NULL) + { + KEBUGCHECK(0); + } + /* + * Atomically disable the present bit and get the old value. + */ + do + { + Pte = *Pt; + tmpPte = Pte & ~PA_PRESENT; + } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte)); + + MiFlushTlb((PULONG)Pt, Address); + WasValid = PAE_PAGE_MASK(Pte) != 0LL ? TRUE : FALSE; + if (!WasValid) + { + KEBUGCHECK(0); + } + + /* + * Return some information to the caller + */ + if (WasDirty != NULL) + { + *WasDirty = Pte & PA_DIRTY ? TRUE : FALSE; + } + if (Page != NULL) + { + *Page = PAE_PTE_TO_PFN(Pte); + } + } + else + { + ULONG Pte; + PULONG Pt; + + Pt = MmGetPageTableForProcess(Process, Address, FALSE); + if (Pt == NULL) + { + KEBUGCHECK(0); + } + /* + * Atomically disable the present bit and get the old value. + */ + do + { + Pte = *Pt; + } while (Pte != InterlockedCompareExchangeUL(Pt, Pte & ~PA_PRESENT, Pte)); + + MiFlushTlb(Pt, Address); + WasValid = (PAGE_MASK(Pte) != 0); + if (!WasValid) + { + KEBUGCHECK(0); + } + + /* + * Return some information to the caller + */ + if (WasDirty != NULL) + { + *WasDirty = Pte & PA_DIRTY; + } + if (Page != NULL) + { + *Page = PTE_TO_PFN(Pte); + } + } +} + +VOID +NTAPI +MmRawDeleteVirtualMapping(PVOID Address) +{ + if (Ke386Pae) + { + PULONGLONG Pt; + ULONGLONG ZeroPte = 0LL; + Pt = MmGetPageTableForProcessForPAE(NULL, Address, FALSE); + if (Pt) + { + /* + * Set the entry to zero + */ + (void)ExfpInterlockedExchange64UL(Pt, &ZeroPte); + MiFlushTlb((PULONG)Pt, Address); + } + } + else + { + PULONG Pt; + + Pt = MmGetPageTableForProcess(NULL, Address, FALSE); + if (Pt && *Pt) + { + /* + * Set the entry to zero + */ + (void)InterlockedExchangeUL(Pt, 0); + MiFlushTlb(Pt, Address); + } + } +} + +VOID +NTAPI +MmDeleteVirtualMapping(PEPROCESS Process, PVOID Address, BOOLEAN FreePage, + BOOLEAN* WasDirty, PPFN_TYPE Page) +/* + * FUNCTION: Delete a virtual mapping + */ +{ + BOOLEAN WasValid = FALSE; + PFN_TYPE Pfn; + + DPRINT("MmDeleteVirtualMapping(%x, %x, %d, %x, %x)\n", + Process, Address, FreePage, WasDirty, Page); + if (Ke386Pae) + { + ULONGLONG Pte; + PULONGLONG Pt; + + Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE); + if (Pt == NULL) + { + if (WasDirty != NULL) + { + *WasDirty = FALSE; + } + if (Page != NULL) + { + *Page = 0; + } + return; + } + + /* + * Atomically set the entry to zero and get the old value. + */ + Pte = 0LL; + Pte = ExfpInterlockedExchange64UL(Pt, &Pte); + + MiFlushTlb((PULONG)Pt, Address); + + WasValid = PAE_PAGE_MASK(Pte) != 0 ? TRUE : FALSE; + if (WasValid) + { + Pfn = PAE_PTE_TO_PFN(Pte); + MmMarkPageUnmapped(Pfn); + } + else + { + Pfn = 0; + } + + if (FreePage && WasValid) + { + MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn); + } + + /* + * Return some information to the caller + */ + if (WasDirty != NULL) + { + *WasDirty = Pte & PA_DIRTY ? TRUE : FALSE; + } + if (Page != NULL) + { + *Page = Pfn; + } + } + else + { + ULONG Pte; + PULONG Pt; + + Pt = MmGetPageTableForProcess(Process, Address, FALSE); + + if (Pt == NULL) + { + if (WasDirty != NULL) + { + *WasDirty = FALSE; + } + if (Page != NULL) + { + *Page = 0; + } + return; + } + + /* + * Atomically set the entry to zero and get the old value. + */ + Pte = InterlockedExchangeUL(Pt, 0); + + MiFlushTlb(Pt, Address); + + WasValid = (PAGE_MASK(Pte) != 0); + if (WasValid) + { + Pfn = PTE_TO_PFN(Pte); + MmMarkPageUnmapped(Pfn); + } + else + { + Pfn = 0; + } + + if (FreePage && WasValid) + { + MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn); + } + + /* + * Return some information to the caller + */ + if (WasDirty != NULL) + { + *WasDirty = Pte & PA_DIRTY ? TRUE : FALSE; + } + if (Page != NULL) + { + *Page = Pfn; + } + } + /* + * Decrement the reference count for this page table. + */ + if (Process != NULL && WasValid && + ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable != NULL && + Address < MmSystemRangeStart) + { + PUSHORT Ptrc; + ULONG Idx; + + Ptrc = ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable; + Idx = Ke386Pae ? PAE_ADDR_TO_PAGE_TABLE(Address) : ADDR_TO_PAGE_TABLE(Address); + + Ptrc[Idx]--; + if (Ptrc[Idx] == 0) + { + MmFreePageTable(Process, Address); + } + } +} + +VOID +NTAPI +MmDeletePageFileMapping(PEPROCESS Process, PVOID Address, + SWAPENTRY* SwapEntry) +/* + * FUNCTION: Delete a virtual mapping + */ +{ + if (Ke386Pae) + { + ULONGLONG Pte; + PULONGLONG Pt; + + Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE); + if (Pt == NULL) + { + *SwapEntry = 0; + return; + } + + /* + * Atomically set the entry to zero and get the old value. + */ + Pte = 0LL; + Pte = ExfpInterlockedExchange64UL(Pt, &Pte); + + MiFlushTlb((PULONG)Pt, Address); + + /* + * Decrement the reference count for this page table. + */ + if (Process != NULL && Pte && + ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable != NULL && + Address < MmSystemRangeStart) + { + PUSHORT Ptrc; + + Ptrc = ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable; + + Ptrc[PAE_ADDR_TO_PAGE_TABLE(Address)]--; + if (Ptrc[PAE_ADDR_TO_PAGE_TABLE(Address)] == 0) + { + MmFreePageTable(Process, Address); + } + } + + + /* + * Return some information to the caller + */ + *SwapEntry = Pte >> 1; + } + else + { + ULONG Pte; + PULONG Pt; + + Pt = MmGetPageTableForProcess(Process, Address, FALSE); + + if (Pt == NULL) + { + *SwapEntry = 0; + return; + } + + /* + * Atomically set the entry to zero and get the old value. + */ + Pte = InterlockedExchangeUL(Pt, 0); + + MiFlushTlb(Pt, Address); + + /* + * Decrement the reference count for this page table. + */ + if (Process != NULL && Pte && + ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable != NULL && + Address < MmSystemRangeStart) + { + PUSHORT Ptrc; + + Ptrc = ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable; + + Ptrc[ADDR_TO_PAGE_TABLE(Address)]--; + if (Ptrc[ADDR_TO_PAGE_TABLE(Address)] == 0) + { + MmFreePageTable(Process, Address); + } + } + + + /* + * Return some information to the caller + */ + *SwapEntry = Pte >> 1; + } +} + +BOOLEAN +Mmi386MakeKernelPageTableGlobal(PVOID PAddress) +{ + if (Ke386Pae) + { + PULONGLONG Pt; + PULONGLONG Pde; + Pde = PAE_ADDR_TO_PDE(PAddress); + if (*Pde == 0LL) + { + Pt = MmGetPageTableForProcessForPAE(NULL, PAddress, FALSE); +#if 0 + /* Non existing mappings are not cached within the tlb. We must not invalidate this entry */ + FLASH_TLB_ONE(PAddress); +#endif + if (Pt != NULL) + { + return TRUE; + } + } + } + else + { + PULONG Pt, Pde; + Pde = ADDR_TO_PDE(PAddress); + if (*Pde == 0) + { + Pt = MmGetPageTableForProcess(NULL, PAddress, FALSE); +#if 0 + /* Non existing mappings are not cached within the tlb. We must not invalidate this entry */ + FLASH_TLB_ONE(PAddress); +#endif + if (Pt != NULL) + { + return TRUE; + } + } + } + return(FALSE); +} + +BOOLEAN +NTAPI +MmIsDirtyPage(PEPROCESS Process, PVOID Address) +{ + if (Ke386Pae) + { + return MmGetPageEntryForProcessForPAE(Process, Address) & PA_DIRTY ? TRUE : FALSE; + } + else + { + return MmGetPageEntryForProcess(Process, Address) & PA_DIRTY ? TRUE : FALSE; + } +} + +BOOLEAN +NTAPI +MmIsAccessedAndResetAccessPage(PEPROCESS Process, PVOID Address) +{ + if (Address < MmSystemRangeStart && Process == NULL) + { + DPRINT1("MmIsAccessedAndResetAccessPage is called for user space without a process.\n"); + KEBUGCHECK(0); + } + if (Ke386Pae) + { + PULONGLONG Pt; + ULONGLONG Pte; + ULONGLONG tmpPte; + + Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE); + if (Pt == NULL) + { + KEBUGCHECK(0); + } + + do + { + Pte = *Pt; + tmpPte = Pte & ~PA_ACCESSED; + } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte)); + + if (Pte & PA_ACCESSED) + { + MiFlushTlb((PULONG)Pt, Address); + return TRUE; + } + else + { + MmUnmapPageTable((PULONG)Pt); + return FALSE; + } + } + else + { + PULONG Pt; + ULONG Pte; + + Pt = MmGetPageTableForProcess(Process, Address, FALSE); + if (Pt == NULL) + { + KEBUGCHECK(0); + } + + do + { + Pte = *Pt; + } while (Pte != InterlockedCompareExchangeUL(Pt, Pte & ~PA_ACCESSED, Pte)); + + if (Pte & PA_ACCESSED) + { + MiFlushTlb(Pt, Address); + return TRUE; + } + else + { + MmUnmapPageTable(Pt); + return FALSE; + } + } +} + +VOID +NTAPI +MmSetCleanPage(PEPROCESS Process, PVOID Address) +{ + if (Address < MmSystemRangeStart && Process == NULL) + { + DPRINT1("MmSetCleanPage is called for user space without a process.\n"); + KEBUGCHECK(0); + } + if (Ke386Pae) + { + PULONGLONG Pt; + ULONGLONG Pte; + ULONGLONG tmpPte; + + Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE); + + if (Pt == NULL) + { + KEBUGCHECK(0); + } + + do + { + Pte = *Pt; + tmpPte = Pte & ~PA_DIRTY; + } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte)); + + if (Pte & PA_DIRTY) + { + MiFlushTlb((PULONG)Pt, Address); + } + else + { + MmUnmapPageTable((PULONG)Pt); + } + } + else + { + PULONG Pt; + ULONG Pte; + + Pt = MmGetPageTableForProcess(Process, Address, FALSE); + + if (Pt == NULL) + { + KEBUGCHECK(0); + } + + do + { + Pte = *Pt; + } while (Pte != InterlockedCompareExchangeUL(Pt, Pte & ~PA_DIRTY, Pte)); + + if (Pte & PA_DIRTY) + { + MiFlushTlb(Pt, Address); + } + else + { + MmUnmapPageTable(Pt); + } + } +} + +VOID +NTAPI +MmSetDirtyPage(PEPROCESS Process, PVOID Address) +{ + if (Address < MmSystemRangeStart && Process == NULL) + { + DPRINT1("MmSetDirtyPage is called for user space without a process.\n"); + KEBUGCHECK(0); + } + if (Ke386Pae) + { + PULONGLONG Pt; + ULONGLONG Pte; + ULONGLONG tmpPte; + + Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE); + if (Pt == NULL) + { + KEBUGCHECK(0); + } + + do + { + Pte = *Pt; + tmpPte = Pte | PA_DIRTY; + } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte)); + if (!(Pte & PA_DIRTY)) + { + MiFlushTlb((PULONG)Pt, Address); + } + else + { + MmUnmapPageTable((PULONG)Pt); + } + } + else + { + PULONG Pt; + ULONG Pte; + + Pt = MmGetPageTableForProcess(Process, Address, FALSE); + if (Pt == NULL) + { + KEBUGCHECK(0); + } + + do + { + Pte = *Pt; + } while (Pte != InterlockedCompareExchangeUL(Pt, Pte | PA_DIRTY, Pte)); + if (!(Pte & PA_DIRTY)) + { + MiFlushTlb(Pt, Address); + } + else + { + MmUnmapPageTable(Pt); + } + } +} + +VOID +NTAPI +MmEnableVirtualMapping(PEPROCESS Process, PVOID Address) +{ + if (Ke386Pae) + { + PULONGLONG Pt; + ULONGLONG Pte; + ULONGLONG tmpPte; + + Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE); + if (Pt == NULL) + { + KEBUGCHECK(0); + } + + do + { + Pte = *Pt; + tmpPte = Pte | PA_PRESENT; + } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte)); + if (!(Pte & PA_PRESENT)) + { + MiFlushTlb((PULONG)Pt, Address); + } + else + { + MmUnmapPageTable((PULONG)Pt); + } + } + else + { + PULONG Pt; + ULONG Pte; + + Pt = MmGetPageTableForProcess(Process, Address, FALSE); + if (Pt == NULL) + { + KEBUGCHECK(0); + } + + do + { + Pte = *Pt; + } while (Pte != InterlockedCompareExchangeUL(Pt, Pte | PA_PRESENT, Pte)); + if (!(Pte & PA_PRESENT)) + { + MiFlushTlb(Pt, Address); + } + else + { + MmUnmapPageTable(Pt); + } + } +} + +BOOLEAN +NTAPI +MmIsPagePresent(PEPROCESS Process, PVOID Address) +{ + if (Ke386Pae) + { + return MmGetPageEntryForProcessForPAE(Process, Address) & PA_PRESENT ? TRUE : FALSE; + } + else + { + return MmGetPageEntryForProcess(Process, Address) & PA_PRESENT ? TRUE : FALSE; + } +} + +BOOLEAN +NTAPI +MmIsPageSwapEntry(PEPROCESS Process, PVOID Address) +{ + if (Ke386Pae) + { + ULONGLONG Entry; + Entry = MmGetPageEntryForProcessForPAE(Process, Address); + return !(Entry & PA_PRESENT) && Entry != 0 ? TRUE : FALSE; + } + else + { + ULONG Entry; + Entry = MmGetPageEntryForProcess(Process, Address); + return !(Entry & PA_PRESENT) && Entry != 0 ? TRUE : FALSE; + } +} + +NTSTATUS +NTAPI +MmCreateVirtualMappingForKernel(PVOID Address, + ULONG flProtect, + PPFN_TYPE Pages, + ULONG PageCount) +{ + ULONG Attributes; + ULONG i; + PVOID Addr; + ULONG PdeOffset, oldPdeOffset; + BOOLEAN NoExecute = FALSE; + + DPRINT("MmCreateVirtualMappingForKernel(%x, %x, %x, %d)\n", + Address, flProtect, Pages, PageCount); + + if (Address < MmSystemRangeStart) + { + DPRINT1("MmCreateVirtualMappingForKernel is called for user space\n"); + KEBUGCHECK(0); + } + + Attributes = ProtectToPTE(flProtect); + if (Attributes & 0x80000000) + { + NoExecute = TRUE; + } + Attributes &= 0xfff; + if (Ke386GlobalPagesEnabled) + { + Attributes |= PA_GLOBAL; + } + + Addr = Address; + + if (Ke386Pae) + { + PULONGLONG Pt = NULL; + ULONGLONG Pte; + + oldPdeOffset = PAE_ADDR_TO_PDE_OFFSET(Addr) + 1; + for (i = 0; i < PageCount; i++, Addr = (PVOID)((ULONG_PTR)Addr + PAGE_SIZE)) + { + if (!(Attributes & PA_PRESENT) && Pages[i] != 0) + { + DPRINT1("Setting physical address but not allowing access at address " + "0x%.8X with attributes %x/%x.\n", + Addr, Attributes, flProtect); + KEBUGCHECK(0); + } + + PdeOffset = PAE_ADDR_TO_PDE_OFFSET(Addr); + if (oldPdeOffset != PdeOffset) + { + Pt = MmGetPageTableForProcessForPAE(NULL, Addr, TRUE); + if (Pt == NULL) + { + KEBUGCHECK(0); + } + } + else + { + Pt++; + } + oldPdeOffset = PdeOffset; + + Pte = PFN_TO_PTE(Pages[i]) | Attributes; + if (NoExecute) + { + Pte |= 0x8000000000000000LL; + } + Pte = ExfpInterlockedExchange64UL(Pt, &Pte); + if (Pte != 0LL) + { + KEBUGCHECK(0); + } + } + } + else + { + PULONG Pt; + ULONG Pte; + + oldPdeOffset = ADDR_TO_PDE_OFFSET(Addr); + Pt = MmGetPageTableForProcess(NULL, Addr, TRUE); + if (Pt == NULL) + { + KEBUGCHECK(0); + } + Pt--; + + for (i = 0; i < PageCount; i++, Addr = (PVOID)((ULONG_PTR)Addr + PAGE_SIZE)) + { + if (!(Attributes & PA_PRESENT) && Pages[i] != 0) + { + DPRINT1("Setting physical address but not allowing access at address " + "0x%.8X with attributes %x/%x.\n", + Addr, Attributes, flProtect); + KEBUGCHECK(0); + } + + PdeOffset = ADDR_TO_PDE_OFFSET(Addr); + if (oldPdeOffset != PdeOffset) + { + Pt = MmGetPageTableForProcess(NULL, Addr, TRUE); + if (Pt == NULL) + { + KEBUGCHECK(0); + } + } + else + { + Pt++; + } + oldPdeOffset = PdeOffset; + + Pte = *Pt; + if (Pte != 0) + { + KEBUGCHECK(0); + } + (void)InterlockedExchangeUL(Pt, PFN_TO_PTE(Pages[i]) | Attributes); + } + } + + return(STATUS_SUCCESS); +} + +NTSTATUS +NTAPI +MmCreatePageFileMapping(PEPROCESS Process, + PVOID Address, + SWAPENTRY SwapEntry) +{ + if (Process == NULL && Address < MmSystemRangeStart) + { + DPRINT1("No process\n"); + KEBUGCHECK(0); + } + if (Process != NULL && Address >= MmSystemRangeStart) + { + DPRINT1("Setting kernel address with process context\n"); + KEBUGCHECK(0); + } + if (SwapEntry & (1 << 31)) + { + KEBUGCHECK(0); + } + + if (Ke386Pae) + { + PULONGLONG Pt; + ULONGLONG Pte; + ULONGLONG tmpPte; + + Pt = MmGetPageTableForProcessForPAE(Process, Address, TRUE); + if (Pt == NULL) + { + KEBUGCHECK(0); + } + tmpPte = SwapEntry << 1; + Pte = ExfpInterlockedExchange64UL(Pt, &tmpPte); + if (PAE_PAGE_MASK((Pte)) != 0) + { + MmMarkPageUnmapped(PAE_PTE_TO_PFN((Pte))); + } + + if (Pte != 0) + { + MiFlushTlb((PULONG)Pt, Address); + } + else + { + MmUnmapPageTable((PULONG)Pt); + } + } + else + { + PULONG Pt; + ULONG Pte; + + Pt = MmGetPageTableForProcess(Process, Address, TRUE); + if (Pt == NULL) + { + KEBUGCHECK(0); + } + Pte = *Pt; + if (PAGE_MASK((Pte)) != 0) + { + MmMarkPageUnmapped(PTE_TO_PFN((Pte))); + } + (void)InterlockedExchangeUL(Pt, SwapEntry << 1); + if (Pte != 0) + { + MiFlushTlb(Pt, Address); + } + else + { + MmUnmapPageTable(Pt); + } + } + if (Process != NULL && + ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable != NULL && + Address < MmSystemRangeStart) + { + PUSHORT Ptrc; + ULONG Idx; + + Ptrc = ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable; + Idx = Ke386Pae ? PAE_ADDR_TO_PAGE_TABLE(Address) : ADDR_TO_PAGE_TABLE(Address); + Ptrc[Idx]++; + } + return(STATUS_SUCCESS); +} + + +NTSTATUS +NTAPI +MmCreateVirtualMappingUnsafe(PEPROCESS Process, + PVOID Address, + ULONG flProtect, + PPFN_TYPE Pages, + ULONG PageCount) +{ + ULONG Attributes; + PVOID Addr; + ULONG i; + ULONG oldPdeOffset, PdeOffset; + BOOLEAN NoExecute = FALSE; + + DPRINT("MmCreateVirtualMappingUnsafe(%x, %x, %x, %x (%x), %d)\n", + Process, Address, flProtect, Pages, *Pages, PageCount); + + if (Process == NULL) + { + if (Address < MmSystemRangeStart) + { + DPRINT1("No process\n"); + KEBUGCHECK(0); + } + if (PageCount > 0x10000 || + (ULONG_PTR) Address / PAGE_SIZE + PageCount > 0x100000) + { + DPRINT1("Page count to large\n"); + KEBUGCHECK(0); + } + } + else + { + if (Address >= MmSystemRangeStart) + { + DPRINT1("Setting kernel address with process context\n"); + KEBUGCHECK(0); + } + if (PageCount > (ULONG_PTR)MmSystemRangeStart / PAGE_SIZE || + (ULONG_PTR) Address / PAGE_SIZE + PageCount > + (ULONG_PTR)MmSystemRangeStart / PAGE_SIZE) + { + DPRINT1("Page Count to large\n"); + KEBUGCHECK(0); + } + } + + Attributes = ProtectToPTE(flProtect); + if (Attributes & 0x80000000) + { + NoExecute = TRUE; + } + Attributes &= 0xfff; + if (Address >= MmSystemRangeStart) + { + Attributes &= ~PA_USER; + if (Ke386GlobalPagesEnabled) + { + Attributes |= PA_GLOBAL; + } + } + else + { + Attributes |= PA_USER; + } + + Addr = Address; + + if (Ke386Pae) + { + ULONGLONG Pte, tmpPte; + PULONGLONG Pt = NULL; + + oldPdeOffset = PAE_ADDR_TO_PDE_OFFSET(Addr) + 1; + for (i = 0; i < PageCount; i++, Addr = (PVOID)((ULONG_PTR)Addr + PAGE_SIZE)) + { + if (!(Attributes & PA_PRESENT) && Pages[i] != 0) + { + DPRINT1("Setting physical address but not allowing access at address " + "0x%.8X with attributes %x/%x.\n", + Addr, Attributes, flProtect); + KEBUGCHECK(0); + } + PdeOffset = PAE_ADDR_TO_PDE_OFFSET(Addr); + if (oldPdeOffset != PdeOffset) + { + MmUnmapPageTable((PULONG)Pt); + Pt = MmGetPageTableForProcessForPAE(Process, Addr, TRUE); + if (Pt == NULL) + { + KEBUGCHECK(0); + } + } + else + { + Pt++; + } + oldPdeOffset = PdeOffset; + + MmMarkPageMapped(Pages[i]); + tmpPte = PAE_PFN_TO_PTE(Pages[i]) | Attributes; + if (NoExecute) + { + tmpPte |= 0x8000000000000000LL; + } + Pte = ExfpInterlockedExchange64UL(Pt, &tmpPte); + if (PAE_PAGE_MASK((Pte)) != 0LL && !((Pte) & PA_PRESENT)) + { + KEBUGCHECK(0); + } + if (PAE_PAGE_MASK((Pte)) != 0LL) + { + MmMarkPageUnmapped(PAE_PTE_TO_PFN((Pte))); + } + if (Address < MmSystemRangeStart && + ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable != NULL && + Attributes & PA_PRESENT) + { + PUSHORT Ptrc; + + Ptrc = ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable; + + Ptrc[PAE_ADDR_TO_PAGE_TABLE(Addr)]++; + } + if (Pte != 0LL) + { + if (Address > MmSystemRangeStart || + (Pt >= (PULONGLONG)PAGETABLE_MAP && Pt < (PULONGLONG)PAGETABLE_MAP + 4*512*512)) + { + MiFlushTlb((PULONG)Pt, Address); + } + } + } + if (Addr > Address) + { + MmUnmapPageTable((PULONG)Pt); + } + } + else + { + PULONG Pt = NULL; + ULONG Pte; + oldPdeOffset = ADDR_TO_PDE_OFFSET(Addr) + 1; + for (i = 0; i < PageCount; i++, Addr = (PVOID)((ULONG_PTR)Addr + PAGE_SIZE)) + { + if (!(Attributes & PA_PRESENT) && Pages[i] != 0) + { + DPRINT1("Setting physical address but not allowing access at address " + "0x%.8X with attributes %x/%x.\n", + Addr, Attributes, flProtect); + KEBUGCHECK(0); + } + PdeOffset = ADDR_TO_PDE_OFFSET(Addr); + if (oldPdeOffset != PdeOffset) + { + MmUnmapPageTable(Pt); + Pt = MmGetPageTableForProcess(Process, Addr, TRUE); + if (Pt == NULL) + { + KEBUGCHECK(0); + } + } + else + { + Pt++; + } + oldPdeOffset = PdeOffset; + + Pte = *Pt; + MmMarkPageMapped(Pages[i]); + if (PAGE_MASK((Pte)) != 0 && !((Pte) & PA_PRESENT)) + { + KEBUGCHECK(0); + } + if (PAGE_MASK((Pte)) != 0) + { + MmMarkPageUnmapped(PTE_TO_PFN((Pte))); + } + (void)InterlockedExchangeUL(Pt, PFN_TO_PTE(Pages[i]) | Attributes); + if (Address < MmSystemRangeStart && + ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable != NULL && + Attributes & PA_PRESENT) + { + PUSHORT Ptrc; + + Ptrc = ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable; + + Ptrc[ADDR_TO_PAGE_TABLE(Addr)]++; + } + if (Pte != 0) + { + if (Address > MmSystemRangeStart || + (Pt >= (PULONG)PAGETABLE_MAP && Pt < (PULONG)PAGETABLE_MAP + 1024*1024)) + { + MiFlushTlb(Pt, Address); + } + } + } + if (Addr > Address) + { + MmUnmapPageTable(Pt); + } + } + return(STATUS_SUCCESS); +} + +NTSTATUS +NTAPI +MmCreateVirtualMapping(PEPROCESS Process, + PVOID Address, + ULONG flProtect, + PPFN_TYPE Pages, + ULONG PageCount) +{ + ULONG i; + + for (i = 0; i < PageCount; i++) + { + if (!MmIsPageInUse(Pages[i])) + { + DPRINT1("Page at address %x not in use\n", PFN_TO_PTE(Pages[i])); + KEBUGCHECK(0); + } + } + + return(MmCreateVirtualMappingUnsafe(Process, + Address, + flProtect, + Pages, + PageCount)); +} + +ULONG +NTAPI +MmGetPageProtect(PEPROCESS Process, PVOID Address) +{ + ULONG Entry; + ULONG Protect; + if (Ke386Pae) + { + Entry = MmGetPageEntryForProcessForPAE(Process, Address); + } + else + { + Entry = MmGetPageEntryForProcess(Process, Address); + } + + if (!(Entry & PA_PRESENT)) + { + Protect = PAGE_NOACCESS; + } + else + { + if (Entry & PA_READWRITE) + { + Protect = PAGE_READWRITE; + } + else + { + Protect = PAGE_EXECUTE_READ; + } + if (Entry & PA_CD) + { + Protect |= PAGE_NOCACHE; + } + if (Entry & PA_WT) + { + Protect |= PAGE_WRITETHROUGH; + } + if (!(Entry & PA_USER)) + { + Protect |= PAGE_SYSTEM; + } + + } + return(Protect); +} + +VOID +NTAPI +MmSetPageProtect(PEPROCESS Process, PVOID Address, ULONG flProtect) +{ + ULONG Attributes = 0; + BOOLEAN NoExecute = FALSE; + + DPRINT("MmSetPageProtect(Process %x Address %x flProtect %x)\n", + Process, Address, flProtect); + + Attributes = ProtectToPTE(flProtect); + if (Attributes & 0x80000000) + { + NoExecute = TRUE; + } + Attributes &= 0xfff; + if (Address >= MmSystemRangeStart) + { + Attributes &= ~PA_USER; + if (Ke386GlobalPagesEnabled) + { + Attributes |= PA_GLOBAL; + } + } + else + { + Attributes |= PA_USER; + } + if (Ke386Pae) + { + PULONGLONG Pt; + ULONGLONG tmpPte, Pte; + + Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE); + if (Pt == NULL) + { + DPRINT1("Address %x\n", Address); + KEBUGCHECK(0); + } + do + { + Pte = *Pt; + tmpPte = PAE_PAGE_MASK(Pte) | Attributes | (Pte & (PA_ACCESSED|PA_DIRTY)); + if (NoExecute) + { + tmpPte |= 0x8000000000000000LL; + } + else + { + tmpPte &= ~0x8000000000000000LL; + } + } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte)); + + MiFlushTlb((PULONG)Pt, Address); + } + else + { + PULONG Pt; + + Pt = MmGetPageTableForProcess(Process, Address, FALSE); + if (Pt == NULL) + { + KEBUGCHECK(0); + } + InterlockedExchange((PLONG)Pt, PAGE_MASK(*Pt) | Attributes | (*Pt & (PA_ACCESSED|PA_DIRTY))); + MiFlushTlb(Pt, Address); + } +} + +/* + * @implemented + */ +PHYSICAL_ADDRESS STDCALL +MmGetPhysicalAddress(PVOID vaddr) +/* + * FUNCTION: Returns the physical address corresponding to a virtual address + */ +{ + PHYSICAL_ADDRESS p; + + DPRINT("MmGetPhysicalAddress(vaddr %x)\n", vaddr); + if (Ke386Pae) + { + ULONGLONG Pte; + Pte = MmGetPageEntryForProcessForPAE(NULL, vaddr); + if (Pte != 0 && Pte & PA_PRESENT) + { + p.QuadPart = PAE_PAGE_MASK(Pte); + p.u.LowPart |= (ULONG_PTR)vaddr & (PAGE_SIZE - 1); + } + else + { + p.QuadPart = 0; + } + } + else + { + ULONG Pte; + Pte = MmGetPageEntryForProcess(NULL, vaddr); + if (Pte != 0 && Pte & PA_PRESENT) + { + p.QuadPart = PAGE_MASK(Pte); + p.u.LowPart |= (ULONG_PTR)vaddr & (PAGE_SIZE - 1); + } + else + { + p.QuadPart = 0; + } + } + return p; +} + +PVOID +NTAPI +MmCreateHyperspaceMapping(PFN_TYPE Page) +{ + PVOID Address; + ULONG i; + + if (Ke386Pae) + { + ULONGLONG Entry; + ULONGLONG ZeroEntry = 0LL; + PULONGLONG Pte; + + Entry = PFN_TO_PTE(Page) | PA_PRESENT | PA_READWRITE; + Pte = PAE_ADDR_TO_PTE(HYPERSPACE) + Page % 1024; + + if (Page & 1024) + { + for (i = Page %1024; i < 1024; i++, Pte++) + { + if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry)) + { + break; + } + } + if (i >= 1024) + { + Pte = PAE_ADDR_TO_PTE(HYPERSPACE); + for (i = 0; i < Page % 1024; i++, Pte++) + { + if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry)) + { + break; + } + } + if (i >= Page % 1024) + { + KEBUGCHECK(0); + } + } + } + else + { + for (i = Page %1024; (LONG)i >= 0; i--, Pte--) + { + if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry)) + { + break; + } + } + if ((LONG)i < 0) + { + Pte = PAE_ADDR_TO_PTE(HYPERSPACE) + 1023; + for (i = 1023; i > Page % 1024; i--, Pte--) + { + if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry)) + { + break; + } + } + if (i <= Page % 1024) + { + KEBUGCHECK(0); + } + } + } + } + else + { + ULONG Entry; + PULONG Pte; + Entry = PFN_TO_PTE(Page) | PA_PRESENT | PA_READWRITE; + Pte = ADDR_TO_PTE(HYPERSPACE) + Page % 1024; + if (Page & 1024) + { + for (i = Page % 1024; i < 1024; i++, Pte++) + { + if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0)) + { + break; + } + } + if (i >= 1024) + { + Pte = ADDR_TO_PTE(HYPERSPACE); + for (i = 0; i < Page % 1024; i++, Pte++) + { + if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0)) + { + break; + } + } + if (i >= Page % 1024) + { + KEBUGCHECK(0); + } + } + } + else + { + for (i = Page % 1024; (LONG)i >= 0; i--, Pte--) + { + if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0)) + { + break; + } + } + if ((LONG)i < 0) + { + Pte = ADDR_TO_PTE(HYPERSPACE) + 1023; + for (i = 1023; i > Page % 1024; i--, Pte--) + { + if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0)) + { + break; + } + } + if (i <= Page % 1024) + { + KEBUGCHECK(0); + } + } + } + } + Address = (PVOID)((ULONG_PTR)HYPERSPACE + i * PAGE_SIZE); + __invlpg(Address); + return Address; +} + +PFN_TYPE +NTAPI +MmChangeHyperspaceMapping(PVOID Address, PFN_TYPE NewPage) +{ + PFN_TYPE Pfn; + ASSERT (IS_HYPERSPACE(Address)); + if (Ke386Pae) + { + ULONGLONG Entry = PAE_PFN_TO_PTE(NewPage) | PA_PRESENT | PA_READWRITE; + Entry = (ULONG)ExfpInterlockedExchange64UL(PAE_ADDR_TO_PTE(Address), &Entry); + Pfn = PAE_PTE_TO_PFN(Entry); + } + else + { + ULONG Entry; + Entry = InterlockedExchange((PLONG)ADDR_TO_PTE(Address), PFN_TO_PTE(NewPage) | PA_PRESENT | PA_READWRITE); + Pfn = PTE_TO_PFN(Entry); + } + __invlpg(Address); + return Pfn; +} + +PFN_TYPE +NTAPI +MmDeleteHyperspaceMapping(PVOID Address) +{ + PFN_TYPE Pfn; + ASSERT (IS_HYPERSPACE(Address)); + if (Ke386Pae) + { + ULONGLONG Entry = 0LL; + Entry = (ULONG)ExfpInterlockedExchange64UL(PAE_ADDR_TO_PTE(Address), &Entry); + Pfn = PAE_PTE_TO_PFN(Entry); + } + else + { + ULONG Entry; + Entry = InterlockedExchange((PLONG)ADDR_TO_PTE(Address), 0); + Pfn = PTE_TO_PFN(Entry); + } + __invlpg(Address); + return Pfn; +} + +VOID +NTAPI +MmUpdatePageDir(PEPROCESS Process, PVOID Address, ULONG Size) +{ + ULONG StartOffset, EndOffset, Offset; + + if (Address < MmSystemRangeStart) + { + KEBUGCHECK(0); + } + if (Ke386Pae) + { + PULONGLONG PageDirTable; + PULONGLONG Pde; + ULONGLONG ZeroPde = 0LL; + ULONG i; + + for (i = PAE_ADDR_TO_PDTE_OFFSET(Address); i <= PAE_ADDR_TO_PDTE_OFFSET((PVOID)((ULONG_PTR)Address + Size)); i++) + { + if (i == PAE_ADDR_TO_PDTE_OFFSET(Address)) + { + StartOffset = PAE_ADDR_TO_PDE_PAGE_OFFSET(Address); + } + else + { + StartOffset = 0; + } + if (i == PAE_ADDR_TO_PDTE_OFFSET((PVOID)((ULONG_PTR)Address + Size))) + { + EndOffset = PAE_ADDR_TO_PDE_PAGE_OFFSET((PVOID)((ULONG_PTR)Address + Size)); + } + else + { + EndOffset = 511; + } + + if (Process != NULL && Process != PsGetCurrentProcess()) + { + PageDirTable = MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart)); + Pde = (PULONGLONG)MmCreateHyperspaceMapping(PTE_TO_PFN(PageDirTable[i])); + MmDeleteHyperspaceMapping(PageDirTable); + } + else + { + Pde = (PULONGLONG)PAE_PAGEDIRECTORY_MAP + i*512; + } + + for (Offset = StartOffset; Offset <= EndOffset; Offset++) + { + if (i * 512 + Offset < PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) || i * 512 + Offset >= PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP)+4) + { + (void)ExfInterlockedCompareExchange64UL(&Pde[Offset], &MmGlobalKernelPageDirectoryForPAE[i*512 + Offset], &ZeroPde); + } + } + MmUnmapPageTable((PULONG)Pde); + } + } + else + { + PULONG Pde; + StartOffset = ADDR_TO_PDE_OFFSET(Address); + EndOffset = ADDR_TO_PDE_OFFSET((PVOID)((ULONG_PTR)Address + Size)); + + if (Process != NULL && Process != PsGetCurrentProcess()) + { + Pde = MmCreateHyperspaceMapping(PTE_TO_PFN(Process->Pcb.DirectoryTableBase.u.LowPart)); + } + else + { + Pde = (PULONG)PAGEDIRECTORY_MAP; + } + for (Offset = StartOffset; Offset <= EndOffset; Offset++) + { + if (Offset != ADDR_TO_PDE_OFFSET(PAGETABLE_MAP)) + { + (void)InterlockedCompareExchangeUL(&Pde[Offset], MmGlobalKernelPageDirectory[Offset], 0); + } + } + if (Pde != (PULONG)PAGEDIRECTORY_MAP) + { + MmDeleteHyperspaceMapping(Pde); + } + } +} + +VOID +INIT_FUNCTION +NTAPI +MmInitGlobalKernelPageDirectory(VOID) +{ + ULONG i; + + DPRINT("MmInitGlobalKernelPageDirectory()\n"); + + if (Ke386Pae) + { + PULONGLONG CurrentPageDirectory = (PULONGLONG)PAE_PAGEDIRECTORY_MAP; + for (i = PAE_ADDR_TO_PDE_OFFSET(MmSystemRangeStart); i < 4 * 512; i++) + { + if (!(i >= PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) && i < PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) + 4) && + !(i >= PAE_ADDR_TO_PDE_OFFSET(HYPERSPACE) && i < PAE_ADDR_TO_PDE_OFFSET(HYPERSPACE) + 2) && + 0LL == MmGlobalKernelPageDirectoryForPAE[i] && 0LL != CurrentPageDirectory[i]) + { + (void)ExfpInterlockedExchange64UL(&MmGlobalKernelPageDirectoryForPAE[i], &CurrentPageDirectory[i]); + if (Ke386GlobalPagesEnabled) + { + MmGlobalKernelPageDirectoryForPAE[i] |= PA_GLOBAL; + CurrentPageDirectory[i] |= PA_GLOBAL; + } + } + } + } + else + { + PULONG CurrentPageDirectory = (PULONG)PAGEDIRECTORY_MAP; + for (i = ADDR_TO_PDE_OFFSET(MmSystemRangeStart); i < 1024; i++) + { + if (i != ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) && + i != ADDR_TO_PDE_OFFSET(HYPERSPACE) && + 0 == MmGlobalKernelPageDirectory[i] && 0 != CurrentPageDirectory[i]) + { + MmGlobalKernelPageDirectory[i] = CurrentPageDirectory[i]; + if (Ke386GlobalPagesEnabled) + { + MmGlobalKernelPageDirectory[i] |= PA_GLOBAL; + CurrentPageDirectory[i] |= PA_GLOBAL; + } + } + } + } +} + +ULONG +NTAPI +MiGetUserPageDirectoryCount(VOID) +{ + return Ke386Pae ? PAE_ADDR_TO_PDE_OFFSET(MmSystemRangeStart) : ADDR_TO_PDE_OFFSET(MmSystemRangeStart); +} + +VOID +INIT_FUNCTION +NTAPI +MiInitPageDirectoryMap(VOID) +{ + MEMORY_AREA* kernel_map_desc = NULL; + MEMORY_AREA* hyperspace_desc = NULL; + PHYSICAL_ADDRESS BoundaryAddressMultiple; + PVOID BaseAddress; + NTSTATUS Status; + + DPRINT("MiInitPageDirectoryMap()\n"); + + BoundaryAddressMultiple.QuadPart = 0; + BaseAddress = (PVOID)PAGETABLE_MAP; + Status = MmCreateMemoryArea(MmGetKernelAddressSpace(), + MEMORY_AREA_SYSTEM, + &BaseAddress, + Ke386Pae ? 0x800000 : 0x400000, + PAGE_READWRITE, + &kernel_map_desc, + TRUE, + 0, + BoundaryAddressMultiple); + if (!NT_SUCCESS(Status)) + { + KEBUGCHECK(0); + } + BaseAddress = (PVOID)HYPERSPACE; + Status = MmCreateMemoryArea(MmGetKernelAddressSpace(), + MEMORY_AREA_SYSTEM, + &BaseAddress, + 0x400000, + PAGE_READWRITE, + &hyperspace_desc, + TRUE, + 0, + BoundaryAddressMultiple); + if (!NT_SUCCESS(Status)) + { + KEBUGCHECK(0); + } +} + +/* EOF */