diff --git a/reactos/ntoskrnl/mm/i386/page.c b/reactos/ntoskrnl/mm/i386/page.c index 182abac3394..67ed2494e6c 100644 --- a/reactos/ntoskrnl/mm/i386/page.c +++ b/reactos/ntoskrnl/mm/i386/page.c @@ -43,20 +43,14 @@ #define PAGETABLE_MAP (0xc0000000) #define PAGEDIRECTORY_MAP (0xc0000000 + (PAGETABLE_MAP / (1024))) -#define PAE_PAGEDIRECTORY_MAP (0xc0000000 + (PAGETABLE_MAP / (512))) - -#define HYPERSPACE (Ke386Pae ? 0xc0800000 : 0xc0400000) +#define HYPERSPACE (0xc0400000) #define IS_HYPERSPACE(v) (((ULONG)(v) >= HYPERSPACE && (ULONG)(v) < HYPERSPACE + 0x400000)) ULONG MmGlobalKernelPageDirectory[1024]; -ULONGLONG MmGlobalKernelPageDirectoryForPAE[2048]; #define PTE_TO_PFN(X) ((X) >> PAGE_SHIFT) #define PFN_TO_PTE(X) ((X) << PAGE_SHIFT) -#define PAE_PTE_TO_PFN(X) (PAE_PAGE_MASK(X) >> PAGE_SHIFT) -#define PAE_PFN_TO_PTE(X) ((X) << PAGE_SHIFT) - #if defined(__GNUC__) #define PTE_TO_PAGE(X) ((LARGE_INTEGER)(LONGLONG)(PAGE_MASK(X))) #else @@ -68,9 +62,6 @@ __inline LARGE_INTEGER PTE_TO_PAGE(ULONG npage) } #endif -extern BOOLEAN Ke386Pae; -extern BOOLEAN Ke386NoExecute; - /* FUNCTIONS ***************************************************************/ BOOLEAN MmUnmapPageTable(PULONG Pt); @@ -97,25 +88,10 @@ MiFlushTlbIpiRoutine(ULONG_PTR Address) VOID MiFlushTlb(PULONG Pt, PVOID Address) { -#ifdef CONFIG_SMP - if (Pt) - { - MmUnmapPageTable(Pt); - } - if (KeNumberProcessors > 1) - { - KeIpiGenericCall(MiFlushTlbIpiRoutine, (ULONG_PTR)Address); - } - else - { - MiFlushTlbIpiRoutine((ULONG_PTR)Address); - } -#else if ((Pt && MmUnmapPageTable(Pt)) || Address >= MmSystemRangeStart) { __invlpg(Address); } -#endif } @@ -148,12 +124,7 @@ ProtectToPTE(ULONG flProtect) DPRINT1("Unknown main protection type.\n"); KEBUGCHECK(0); } - if (Ke386NoExecute && - !(flProtect & PAGE_IS_EXECUTABLE)) - { - Attributes = Attributes | 0x80000000; - } - + if (flProtect & PAGE_SYSTEM) { } @@ -183,22 +154,6 @@ ProtectToPTE(ULONG flProtect) #define ADDR_TO_PTE_OFFSET(v) ((((ULONG)(v)) % (1024 * PAGE_SIZE)) / PAGE_SIZE) -#define PAE_ADDR_TO_PAGE_TABLE(v) (((ULONG)(v)) / (512 * PAGE_SIZE)) - -#define PAE_ADDR_TO_PDE(v) (PULONGLONG) (PAE_PAGEDIRECTORY_MAP + \ - ((((ULONG_PTR)(v)) / (512 * 512))&(~0x7))) -#define PAE_ADDR_TO_PTE(v) (PULONGLONG) (PAGETABLE_MAP + ((((ULONG_PTR)(v) / 512))&(~0x7))) - - -#define PAE_ADDR_TO_PDTE_OFFSET(v) (((ULONG_PTR)(v)) / (512 * 512 * PAGE_SIZE)) - -#define PAE_ADDR_TO_PDE_PAGE_OFFSET(v) ((((ULONG_PTR)(v)) % (512 * 512 * PAGE_SIZE)) / (512 * PAGE_SIZE)) - -#define PAE_ADDR_TO_PDE_OFFSET(v) (((ULONG_PTR)(v))/ (512 * PAGE_SIZE)) - -#define PAE_ADDR_TO_PTE_OFFSET(v) ((((ULONG_PTR)(v)) % (512 * PAGE_SIZE)) / PAGE_SIZE) - - NTSTATUS NTAPI Mmi386ReleaseMmInfo(PEPROCESS Process) @@ -221,98 +176,43 @@ Mmi386ReleaseMmInfo(PEPROCESS Process) ExFreePool((PVOID) LdtBase); } - if (Ke386Pae) - { - PULONGLONG PageDirTable; - PULONGLONG PageDir; - PULONGLONG Pde; - ULONG k; - PageDirTable = (PULONGLONG)MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart)); - for (i = 0; i < 4; i++) - { - PageDir = (PULONGLONG)MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(PageDirTable[i])); - if (i < PAE_ADDR_TO_PDTE_OFFSET(MmSystemRangeStart)) - { - for (j = 0; j < 512; j++) - { - if (PageDir[j] != 0LL) - { - DPRINT1("ProcessId %d, Pde for %08x - %08x is not freed, RefCount %d\n", - Process->UniqueProcessId, - (i * 512 + j) * 512 * PAGE_SIZE, (i * 512 + j + 1) * 512 * PAGE_SIZE - 1, - ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable[i*512 + j]); - Pde = MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(PageDir[j])); - for (k = 0; k < 512; k++) - { - if(Pde[k] != 0) - { - if (Pde[k] & PA_PRESENT) - { - DPRINT1("Page at %08x is not freed\n", - (i * 512 + j) * 512 * PAGE_SIZE + k * PAGE_SIZE); - } - else - { - DPRINT1("Swapentry %x at %x is not freed\n", - (i * 512 + j) * 512 * PAGE_SIZE + k * PAGE_SIZE); - } - } - } - MmDeleteHyperspaceMapping(Pde); - MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(PageDir[j])); - } - } - } - if (i == PAE_ADDR_TO_PDTE_OFFSET(HYPERSPACE)) - { - MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE)])); - MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE)+1])); - } - MmDeleteHyperspaceMapping(PageDir); - MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(PageDirTable[i])); - } - MmDeleteHyperspaceMapping((PVOID)PageDirTable); - MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart)); - } - else - { - PULONG Pde; - PULONG PageDir; - PageDir = MmCreateHyperspaceMapping(PTE_TO_PFN(Process->Pcb.DirectoryTableBase.u.LowPart)); - for (i = 0; i < ADDR_TO_PDE_OFFSET(MmSystemRangeStart); i++) - { - if (PageDir[i] != 0) - { + PULONG Pde; + PULONG PageDir; + PageDir = MmCreateHyperspaceMapping(PTE_TO_PFN(Process->Pcb.DirectoryTableBase.u.LowPart)); + for (i = 0; i < ADDR_TO_PDE_OFFSET(MmSystemRangeStart); i++) + { + if (PageDir[i] != 0) + { DPRINT1("Pde for %08x - %08x is not freed, RefCount %d\n", - i * 4 * 1024 * 1024, (i + 1) * 4 * 1024 * 1024 - 1, - ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable[i]); - Pde = MmCreateHyperspaceMapping(PTE_TO_PFN(PageDir[i])); - for (j = 0; j < 1024; j++) - { - if(Pde[j] != 0) - { - if (Pde[j] & PA_PRESENT) - { - DPRINT1("Page at %08x is not freed\n", - i * 4 * 1024 * 1024 + j * PAGE_SIZE); - } - else - { - DPRINT1("Swapentry %x at %x is not freed\n", - Pde[j], i * 4 * 1024 * 1024 + j * PAGE_SIZE); - } - } - } - MmDeleteHyperspaceMapping(Pde); - MmReleasePageMemoryConsumer(MC_NPPOOL, PTE_TO_PFN(PageDir[i])); - } - } - MmReleasePageMemoryConsumer(MC_NPPOOL, PTE_TO_PFN(PageDir[ADDR_TO_PDE_OFFSET(HYPERSPACE)])); - MmDeleteHyperspaceMapping(PageDir); - MmReleasePageMemoryConsumer(MC_NPPOOL, PTE_TO_PFN(Process->Pcb.DirectoryTableBase.u.LowPart)); - } - + i * 4 * 1024 * 1024, (i + 1) * 4 * 1024 * 1024 - 1, + ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable[i]); + Pde = MmCreateHyperspaceMapping(PTE_TO_PFN(PageDir[i])); + for (j = 0; j < 1024; j++) + { + if(Pde[j] != 0) + { + if (Pde[j] & PA_PRESENT) + { + DPRINT1("Page at %08x is not freed\n", + i * 4 * 1024 * 1024 + j * PAGE_SIZE); + } + else + { + DPRINT1("Swapentry %x at %x is not freed\n", + Pde[j], i * 4 * 1024 * 1024 + j * PAGE_SIZE); + + } + } + } + MmDeleteHyperspaceMapping(Pde); + MmReleasePageMemoryConsumer(MC_NPPOOL, PTE_TO_PFN(PageDir[i])); + } + } + MmReleasePageMemoryConsumer(MC_NPPOOL, PTE_TO_PFN(PageDir[ADDR_TO_PDE_OFFSET(HYPERSPACE)])); + MmDeleteHyperspaceMapping(PageDir); + MmReleasePageMemoryConsumer(MC_NPPOOL, PTE_TO_PFN(Process->Pcb.DirectoryTableBase.u.LowPart)); + #if defined(__GNUC__) Process->Pcb.DirectoryTableBase.QuadPart = 0LL; @@ -349,14 +249,11 @@ MmCreateProcessAddressSpace(IN ULONG MinWs, { NTSTATUS Status; ULONG i, j; - PFN_TYPE Pfn[7]; - ULONG Count; + PFN_TYPE Pfn[2]; DPRINT("MmCopyMmInfo(Src %x, Dest %x)\n", MinWs, Process); - Count = Ke386Pae ? 7 : 2; - - for (i = 0; i < Count; i++) + for (i = 0; i < 2; i++) { Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn[i]); if (!NT_SUCCESS(Status)) @@ -370,38 +267,6 @@ MmCreateProcessAddressSpace(IN ULONG MinWs, } } - if (Ke386Pae) - { - PULONGLONG PageDirTable; - PULONGLONG PageDir; - - PageDirTable = MmCreateHyperspaceMapping(Pfn[0]); - for (i = 0; i < 4; i++) - { - PageDirTable[i] = PAE_PFN_TO_PTE(Pfn[1+i]) | PA_PRESENT; - } - MmDeleteHyperspaceMapping(PageDirTable); - for (i = PAE_ADDR_TO_PDTE_OFFSET(MmSystemRangeStart); i < 4; i++) - { - PageDir = (PULONGLONG)MmCreateHyperspaceMapping(Pfn[i+1]); - memcpy(PageDir, &MmGlobalKernelPageDirectoryForPAE[i * 512], 512 * sizeof(ULONGLONG)); - if (PAE_ADDR_TO_PDTE_OFFSET(PAGETABLE_MAP) == i) - { - for (j = 0; j < 4; j++) - { - PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(PAGETABLE_MAP) + j] = PAE_PFN_TO_PTE(Pfn[1+j]) | PA_PRESENT | PA_READWRITE; - } - } - if (PAE_ADDR_TO_PDTE_OFFSET(HYPERSPACE) == i) - { - PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE)] = PAE_PFN_TO_PTE(Pfn[5]) | PA_PRESENT | PA_READWRITE; - PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE)+1] = PAE_PFN_TO_PTE(Pfn[6]) | PA_PRESENT | PA_READWRITE; - } - MmDeleteHyperspaceMapping(PageDir); - } - } - else - { PULONG PageDirectory; PageDirectory = MmCreateHyperspaceMapping(Pfn[0]); @@ -414,7 +279,6 @@ MmCreateProcessAddressSpace(IN ULONG MinWs, PageDirectory[ADDR_TO_PDE_OFFSET(HYPERSPACE)] = PFN_TO_PTE(Pfn[1]) | PA_PRESENT | PA_READWRITE; MmDeleteHyperspaceMapping(PageDirectory); - } DirectoryTableBase->QuadPart = PFN_TO_PTE(Pfn[0]); DPRINT("Finished MmCopyMmInfo(): %I64x\n", DirectoryTableBase->QuadPart); @@ -432,17 +296,9 @@ MmDeletePageTable(PEPROCESS Process, PVOID Address) KeAttachProcess(&Process->Pcb); } - if (Ke386Pae) - { - ULONGLONG ZeroPde = 0LL; - (void)ExfpInterlockedExchange64UL(PAE_ADDR_TO_PDE(Address), &ZeroPde); - MiFlushTlb((PULONG)PAE_ADDR_TO_PDE(Address), PAE_ADDR_TO_PTE(Address)); - } - else - { *(ADDR_TO_PDE(Address)) = 0; MiFlushTlb(ADDR_TO_PDE(Address), ADDR_TO_PTE(Address)); - } + if (Address >= MmSystemRangeStart) { KEBUGCHECK(0); @@ -467,26 +323,7 @@ MmFreePageTable(PEPROCESS Process, PVOID Address) { KeAttachProcess(&Process->Pcb); } - if (Ke386Pae) - { - PULONGLONG PageTable; - ULONGLONG ZeroPte = 0LL; - PageTable = (PULONGLONG)PAGE_ROUND_DOWN((PVOID)PAE_ADDR_TO_PTE(Address)); - for (i = 0; i < 512; i++) - { - if (PageTable[i] != 0LL) - { - DbgPrint("Page table entry not clear at %x/%x (is %I64x)\n", - ((ULONG)Address / (4*1024*1024)), i, PageTable[i]); - KEBUGCHECK(0); - } - } - Pfn = PAE_PTE_TO_PFN(*(PAE_ADDR_TO_PDE(Address))); - (void)ExfpInterlockedExchange64UL(PAE_ADDR_TO_PDE(Address), &ZeroPte); - MiFlushTlb((PULONG)PAE_ADDR_TO_PDE(Address), PAE_ADDR_TO_PTE(Address)); - } - else - { + PULONG PageTable; PageTable = (PULONG)PAGE_ROUND_DOWN((PVOID)ADDR_TO_PTE(Address)); for (i = 0; i < 1024; i++) @@ -501,7 +338,6 @@ MmFreePageTable(PEPROCESS Process, PVOID Address) Pfn = PTE_TO_PFN(*(ADDR_TO_PDE(Address))); *(ADDR_TO_PDE(Address)) = 0; MiFlushTlb(ADDR_TO_PDE(Address), ADDR_TO_PTE(Address)); - } if (Address >= MmSystemRangeStart) { @@ -518,120 +354,6 @@ MmFreePageTable(PEPROCESS Process, PVOID Address) } } -static PULONGLONG -MmGetPageTableForProcessForPAE(PEPROCESS Process, PVOID Address, BOOLEAN Create) -{ - NTSTATUS Status; - PFN_TYPE Pfn; - ULONGLONG Entry; - ULONGLONG ZeroEntry = 0LL; - PULONGLONG Pt; - PULONGLONG PageDir; - PULONGLONG PageDirTable; - - DPRINT("MmGetPageTableForProcessForPAE(%x %x %d)\n", - Process, Address, Create); - if (Address >= (PVOID)PAGETABLE_MAP && Address < (PVOID)((ULONG_PTR)PAGETABLE_MAP + 0x800000)) - { - KEBUGCHECK(0); - } - if (Address < MmSystemRangeStart && Process && Process != PsGetCurrentProcess()) - { - PageDirTable = MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart)); - if (PageDirTable == NULL) - { - KEBUGCHECK(0); - } - PageDir = MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(PageDirTable[PAE_ADDR_TO_PDTE_OFFSET(Address)])); - MmDeleteHyperspaceMapping(PageDirTable); - if (PageDir == NULL) - { - KEBUGCHECK(0); - } - PageDir += PAE_ADDR_TO_PDE_PAGE_OFFSET(Address); - Entry = ExfInterlockedCompareExchange64UL(PageDir, &ZeroEntry, &ZeroEntry); - if (Entry == 0LL) - { - if (Create == FALSE) - { - MmDeleteHyperspaceMapping(PageDir); - return NULL; - } - Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn); - if (!NT_SUCCESS(Status)) - { - KEBUGCHECK(0); - } - Entry = PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE | PA_USER; - Entry = ExfInterlockedCompareExchange64UL(PageDir, &Entry, &ZeroEntry); - if (Entry != 0LL) - { - MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn); - Pfn = PAE_PTE_TO_PFN(Entry); - } - } - else - { - Pfn = PAE_PTE_TO_PFN(Entry); - } - MmDeleteHyperspaceMapping(PageDir); - Pt = MmCreateHyperspaceMapping(Pfn); - if (Pt == NULL) - { - KEBUGCHECK(0); - } - return Pt + PAE_ADDR_TO_PTE_OFFSET(Address); - } - PageDir = PAE_ADDR_TO_PDE(Address); - if (0LL == ExfInterlockedCompareExchange64UL(PageDir, &ZeroEntry, &ZeroEntry)) - { - if (Address >= MmSystemRangeStart) - { - if (MmGlobalKernelPageDirectoryForPAE[PAE_ADDR_TO_PDE_OFFSET(Address)] == 0LL) - { - if (Create == FALSE) - { - return NULL; - } - Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn); - if (!NT_SUCCESS(Status)) - { - KEBUGCHECK(0); - } - Entry = PAE_PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE; - if (Ke386GlobalPagesEnabled) - { - Entry |= PA_GLOBAL; - } - if (0LL != ExfInterlockedCompareExchange64UL(&MmGlobalKernelPageDirectoryForPAE[PAE_ADDR_TO_PDE_OFFSET(Address)], &Entry, &ZeroEntry)) - { - MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn); - } - } - (void)ExfInterlockedCompareExchange64UL(PageDir, &MmGlobalKernelPageDirectoryForPAE[PAE_ADDR_TO_PDE_OFFSET(Address)], &ZeroEntry); - } - else - { - if (Create == FALSE) - { - return NULL; - } - Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn); - if (!NT_SUCCESS(Status)) - { - KEBUGCHECK(0); - } - Entry = PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE | PA_USER; - Entry = ExfInterlockedCompareExchange64UL(PageDir, &Entry, &ZeroEntry); - if (Entry != 0LL) - { - MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn); - } - } - } - return (PULONGLONG)PAE_ADDR_TO_PTE(Address); -} - static PULONG MmGetPageTableForProcess(PEPROCESS Process, PVOID Address, BOOLEAN Create) { @@ -730,20 +452,11 @@ MmGetPageTableForProcess(PEPROCESS Process, PVOID Address, BOOLEAN Create) BOOLEAN MmUnmapPageTable(PULONG Pt) { - if (Ke386Pae) - { - if ((PULONGLONG)Pt >= (PULONGLONG)PAGETABLE_MAP && (PULONGLONG)Pt < (PULONGLONG)PAGETABLE_MAP + 4*512*512) - { - return TRUE; - } - } - else - { if (Pt >= (PULONG)PAGETABLE_MAP && Pt < (PULONG)PAGETABLE_MAP + 1024*1024) { return TRUE; } - } + if (Pt) { MmDeleteHyperspaceMapping((PVOID)PAGE_ROUND_DOWN(Pt)); @@ -751,21 +464,6 @@ BOOLEAN MmUnmapPageTable(PULONG Pt) return FALSE; } -static ULONGLONG MmGetPageEntryForProcessForPAE(PEPROCESS Process, PVOID Address) -{ - ULONGLONG Pte; - PULONGLONG Pt; - - Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE); - if (Pt) - { - Pte = *Pt; - MmUnmapPageTable((PULONG)Pt); - return Pte; - } - return 0; -} - static ULONG MmGetPageEntryForProcess(PEPROCESS Process, PVOID Address) { ULONG Pte; @@ -786,19 +484,6 @@ NTAPI MmGetPfnForProcess(PEPROCESS Process, PVOID Address) { - - if (Ke386Pae) - { - ULONGLONG Entry; - Entry = MmGetPageEntryForProcessForPAE(Process, Address); - if (!(Entry & PA_PRESENT)) - { - return 0; - } - return(PAE_PTE_TO_PFN(Entry)); - } - else - { ULONG Entry; Entry = MmGetPageEntryForProcess(Process, Address); if (!(Entry & PA_PRESENT)) @@ -806,7 +491,6 @@ MmGetPfnForProcess(PEPROCESS Process, return 0; } return(PTE_TO_PFN(Entry)); - } } VOID @@ -817,47 +501,6 @@ MmDisableVirtualMapping(PEPROCESS Process, PVOID Address, BOOLEAN* WasDirty, PPF */ { BOOLEAN WasValid; - if (Ke386Pae) - { - ULONGLONG Pte; - ULONGLONG tmpPte; - PULONGLONG Pt; - - Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE); - if (Pt == NULL) - { - KEBUGCHECK(0); - } - /* - * Atomically disable the present bit and get the old value. - */ - do - { - Pte = *Pt; - tmpPte = Pte & ~PA_PRESENT; - } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte)); - - MiFlushTlb((PULONG)Pt, Address); - WasValid = PAE_PAGE_MASK(Pte) != 0LL ? TRUE : FALSE; - if (!WasValid) - { - KEBUGCHECK(0); - } - - /* - * Return some information to the caller - */ - if (WasDirty != NULL) - { - *WasDirty = Pte & PA_DIRTY ? TRUE : FALSE; - } - if (Page != NULL) - { - *Page = PAE_PTE_TO_PFN(Pte); - } - } - else - { ULONG Pte; PULONG Pt; @@ -892,29 +535,12 @@ MmDisableVirtualMapping(PEPROCESS Process, PVOID Address, BOOLEAN* WasDirty, PPF { *Page = PTE_TO_PFN(Pte); } - } } VOID NTAPI MmRawDeleteVirtualMapping(PVOID Address) { - if (Ke386Pae) - { - PULONGLONG Pt; - ULONGLONG ZeroPte = 0LL; - Pt = MmGetPageTableForProcessForPAE(NULL, Address, FALSE); - if (Pt) - { - /* - * Set the entry to zero - */ - (void)ExfpInterlockedExchange64UL(Pt, &ZeroPte); - MiFlushTlb((PULONG)Pt, Address); - } - } - else - { PULONG Pt; Pt = MmGetPageTableForProcess(NULL, Address, FALSE); @@ -926,7 +552,6 @@ MmRawDeleteVirtualMapping(PVOID Address) (void)InterlockedExchangeUL(Pt, 0); MiFlushTlb(Pt, Address); } - } } VOID @@ -942,63 +567,6 @@ MmDeleteVirtualMapping(PEPROCESS Process, PVOID Address, BOOLEAN FreePage, DPRINT("MmDeleteVirtualMapping(%x, %x, %d, %x, %x)\n", Process, Address, FreePage, WasDirty, Page); - if (Ke386Pae) - { - ULONGLONG Pte; - PULONGLONG Pt; - - Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE); - if (Pt == NULL) - { - if (WasDirty != NULL) - { - *WasDirty = FALSE; - } - if (Page != NULL) - { - *Page = 0; - } - return; - } - - /* - * Atomically set the entry to zero and get the old value. - */ - Pte = 0LL; - Pte = ExfpInterlockedExchange64UL(Pt, &Pte); - - MiFlushTlb((PULONG)Pt, Address); - - WasValid = PAE_PAGE_MASK(Pte) != 0 ? TRUE : FALSE; - if (WasValid) - { - Pfn = PAE_PTE_TO_PFN(Pte); - MmMarkPageUnmapped(Pfn); - } - else - { - Pfn = 0; - } - - if (FreePage && WasValid) - { - MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn); - } - - /* - * Return some information to the caller - */ - if (WasDirty != NULL) - { - *WasDirty = Pte & PA_DIRTY ? TRUE : FALSE; - } - if (Page != NULL) - { - *Page = Pfn; - } - } - else - { ULONG Pte; PULONG Pt; @@ -1051,7 +619,6 @@ MmDeleteVirtualMapping(PEPROCESS Process, PVOID Address, BOOLEAN FreePage, { *Page = Pfn; } - } /* * Decrement the reference count for this page table. */ @@ -1063,7 +630,7 @@ MmDeleteVirtualMapping(PEPROCESS Process, PVOID Address, BOOLEAN FreePage, ULONG Idx; Ptrc = ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable; - Idx = Ke386Pae ? PAE_ADDR_TO_PAGE_TABLE(Address) : ADDR_TO_PAGE_TABLE(Address); + Idx = ADDR_TO_PAGE_TABLE(Address); Ptrc[Idx]--; if (Ptrc[Idx] == 0) @@ -1081,52 +648,6 @@ MmDeletePageFileMapping(PEPROCESS Process, PVOID Address, * FUNCTION: Delete a virtual mapping */ { - if (Ke386Pae) - { - ULONGLONG Pte; - PULONGLONG Pt; - - Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE); - if (Pt == NULL) - { - *SwapEntry = 0; - return; - } - - /* - * Atomically set the entry to zero and get the old value. - */ - Pte = 0LL; - Pte = ExfpInterlockedExchange64UL(Pt, &Pte); - - MiFlushTlb((PULONG)Pt, Address); - - /* - * Decrement the reference count for this page table. - */ - if (Process != NULL && Pte && - ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable != NULL && - Address < MmSystemRangeStart) - { - PUSHORT Ptrc; - - Ptrc = ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable; - - Ptrc[PAE_ADDR_TO_PAGE_TABLE(Address)]--; - if (Ptrc[PAE_ADDR_TO_PAGE_TABLE(Address)] == 0) - { - MmFreePageTable(Process, Address); - } - } - - - /* - * Return some information to the caller - */ - *SwapEntry = Pte >> 1; - } - else - { ULONG Pte; PULONG Pt; @@ -1168,47 +689,21 @@ MmDeletePageFileMapping(PEPROCESS Process, PVOID Address, * Return some information to the caller */ *SwapEntry = Pte >> 1; - } } BOOLEAN Mmi386MakeKernelPageTableGlobal(PVOID PAddress) { - if (Ke386Pae) - { - PULONGLONG Pt; - PULONGLONG Pde; - Pde = PAE_ADDR_TO_PDE(PAddress); - if (*Pde == 0LL) - { - Pt = MmGetPageTableForProcessForPAE(NULL, PAddress, FALSE); -#if 0 - /* Non existing mappings are not cached within the tlb. We must not invalidate this entry */ - FLASH_TLB_ONE(PAddress); -#endif - if (Pt != NULL) - { - return TRUE; - } - } - } - else - { PULONG Pt, Pde; Pde = ADDR_TO_PDE(PAddress); if (*Pde == 0) { Pt = MmGetPageTableForProcess(NULL, PAddress, FALSE); -#if 0 - /* Non existing mappings are not cached within the tlb. We must not invalidate this entry */ - FLASH_TLB_ONE(PAddress); -#endif if (Pt != NULL) { return TRUE; } } - } return(FALSE); } @@ -1216,14 +711,7 @@ BOOLEAN NTAPI MmIsDirtyPage(PEPROCESS Process, PVOID Address) { - if (Ke386Pae) - { - return MmGetPageEntryForProcessForPAE(Process, Address) & PA_DIRTY ? TRUE : FALSE; - } - else - { return MmGetPageEntryForProcess(Process, Address) & PA_DIRTY ? TRUE : FALSE; - } } BOOLEAN @@ -1235,37 +723,7 @@ MmIsAccessedAndResetAccessPage(PEPROCESS Process, PVOID Address) DPRINT1("MmIsAccessedAndResetAccessPage is called for user space without a process.\n"); KEBUGCHECK(0); } - if (Ke386Pae) - { - PULONGLONG Pt; - ULONGLONG Pte; - ULONGLONG tmpPte; - Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE); - if (Pt == NULL) - { - KEBUGCHECK(0); - } - - do - { - Pte = *Pt; - tmpPte = Pte & ~PA_ACCESSED; - } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte)); - - if (Pte & PA_ACCESSED) - { - MiFlushTlb((PULONG)Pt, Address); - return TRUE; - } - else - { - MmUnmapPageTable((PULONG)Pt); - return FALSE; - } - } - else - { PULONG Pt; ULONG Pte; @@ -1290,7 +748,6 @@ MmIsAccessedAndResetAccessPage(PEPROCESS Process, PVOID Address) MmUnmapPageTable(Pt); return FALSE; } - } } VOID @@ -1302,36 +759,7 @@ MmSetCleanPage(PEPROCESS Process, PVOID Address) DPRINT1("MmSetCleanPage is called for user space without a process.\n"); KEBUGCHECK(0); } - if (Ke386Pae) - { - PULONGLONG Pt; - ULONGLONG Pte; - ULONGLONG tmpPte; - Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE); - - if (Pt == NULL) - { - KEBUGCHECK(0); - } - - do - { - Pte = *Pt; - tmpPte = Pte & ~PA_DIRTY; - } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte)); - - if (Pte & PA_DIRTY) - { - MiFlushTlb((PULONG)Pt, Address); - } - else - { - MmUnmapPageTable((PULONG)Pt); - } - } - else - { PULONG Pt; ULONG Pte; @@ -1355,7 +783,6 @@ MmSetCleanPage(PEPROCESS Process, PVOID Address) { MmUnmapPageTable(Pt); } - } } VOID @@ -1367,34 +794,6 @@ MmSetDirtyPage(PEPROCESS Process, PVOID Address) DPRINT1("MmSetDirtyPage is called for user space without a process.\n"); KEBUGCHECK(0); } - if (Ke386Pae) - { - PULONGLONG Pt; - ULONGLONG Pte; - ULONGLONG tmpPte; - - Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE); - if (Pt == NULL) - { - KEBUGCHECK(0); - } - - do - { - Pte = *Pt; - tmpPte = Pte | PA_DIRTY; - } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte)); - if (!(Pte & PA_DIRTY)) - { - MiFlushTlb((PULONG)Pt, Address); - } - else - { - MmUnmapPageTable((PULONG)Pt); - } - } - else - { PULONG Pt; ULONG Pte; @@ -1416,41 +815,12 @@ MmSetDirtyPage(PEPROCESS Process, PVOID Address) { MmUnmapPageTable(Pt); } - } } VOID NTAPI MmEnableVirtualMapping(PEPROCESS Process, PVOID Address) { - if (Ke386Pae) - { - PULONGLONG Pt; - ULONGLONG Pte; - ULONGLONG tmpPte; - - Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE); - if (Pt == NULL) - { - KEBUGCHECK(0); - } - - do - { - Pte = *Pt; - tmpPte = Pte | PA_PRESENT; - } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte)); - if (!(Pte & PA_PRESENT)) - { - MiFlushTlb((PULONG)Pt, Address); - } - else - { - MmUnmapPageTable((PULONG)Pt); - } - } - else - { PULONG Pt; ULONG Pte; @@ -1472,39 +842,22 @@ MmEnableVirtualMapping(PEPROCESS Process, PVOID Address) { MmUnmapPageTable(Pt); } - } } BOOLEAN NTAPI MmIsPagePresent(PEPROCESS Process, PVOID Address) { - if (Ke386Pae) - { - return MmGetPageEntryForProcessForPAE(Process, Address) & PA_PRESENT ? TRUE : FALSE; - } - else - { return MmGetPageEntryForProcess(Process, Address) & PA_PRESENT ? TRUE : FALSE; - } } BOOLEAN NTAPI MmIsPageSwapEntry(PEPROCESS Process, PVOID Address) { - if (Ke386Pae) - { - ULONGLONG Entry; - Entry = MmGetPageEntryForProcessForPAE(Process, Address); - return !(Entry & PA_PRESENT) && Entry != 0 ? TRUE : FALSE; - } - else - { ULONG Entry; Entry = MmGetPageEntryForProcess(Process, Address); return !(Entry & PA_PRESENT) && Entry != 0 ? TRUE : FALSE; - } } NTSTATUS @@ -1542,51 +895,6 @@ MmCreateVirtualMappingForKernel(PVOID Address, Addr = Address; - if (Ke386Pae) - { - PULONGLONG Pt = NULL; - ULONGLONG Pte; - - oldPdeOffset = PAE_ADDR_TO_PDE_OFFSET(Addr) + 1; - for (i = 0; i < PageCount; i++, Addr = (PVOID)((ULONG_PTR)Addr + PAGE_SIZE)) - { - if (!(Attributes & PA_PRESENT) && Pages[i] != 0) - { - DPRINT1("Setting physical address but not allowing access at address " - "0x%.8X with attributes %x/%x.\n", - Addr, Attributes, flProtect); - KEBUGCHECK(0); - } - - PdeOffset = PAE_ADDR_TO_PDE_OFFSET(Addr); - if (oldPdeOffset != PdeOffset) - { - Pt = MmGetPageTableForProcessForPAE(NULL, Addr, TRUE); - if (Pt == NULL) - { - KEBUGCHECK(0); - } - } - else - { - Pt++; - } - oldPdeOffset = PdeOffset; - - Pte = PFN_TO_PTE(Pages[i]) | Attributes; - if (NoExecute) - { - Pte |= 0x8000000000000000LL; - } - Pte = ExfpInterlockedExchange64UL(Pt, &Pte); - if (Pte != 0LL) - { - KEBUGCHECK(0); - } - } - } - else - { PULONG Pt; ULONG Pte; @@ -1630,7 +938,6 @@ MmCreateVirtualMappingForKernel(PVOID Address, } (void)InterlockedExchangeUL(Pt, PFN_TO_PTE(Pages[i]) | Attributes); } - } return(STATUS_SUCCESS); } @@ -1656,35 +963,6 @@ MmCreatePageFileMapping(PEPROCESS Process, KEBUGCHECK(0); } - if (Ke386Pae) - { - PULONGLONG Pt; - ULONGLONG Pte; - ULONGLONG tmpPte; - - Pt = MmGetPageTableForProcessForPAE(Process, Address, TRUE); - if (Pt == NULL) - { - KEBUGCHECK(0); - } - tmpPte = SwapEntry << 1; - Pte = ExfpInterlockedExchange64UL(Pt, &tmpPte); - if (PAE_PAGE_MASK((Pte)) != 0) - { - MmMarkPageUnmapped(PAE_PTE_TO_PFN((Pte))); - } - - if (Pte != 0) - { - MiFlushTlb((PULONG)Pt, Address); - } - else - { - MmUnmapPageTable((PULONG)Pt); - } - } - else - { PULONG Pt; ULONG Pte; @@ -1707,7 +985,7 @@ MmCreatePageFileMapping(PEPROCESS Process, { MmUnmapPageTable(Pt); } - } + if (Process != NULL && ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable != NULL && Address < MmSystemRangeStart) @@ -1716,7 +994,7 @@ MmCreatePageFileMapping(PEPROCESS Process, ULONG Idx; Ptrc = ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable; - Idx = Ke386Pae ? PAE_ADDR_TO_PAGE_TABLE(Address) : ADDR_TO_PAGE_TABLE(Address); + Idx = ADDR_TO_PAGE_TABLE(Address); Ptrc[Idx]++; } return(STATUS_SUCCESS); @@ -1791,79 +1069,7 @@ MmCreateVirtualMappingUnsafe(PEPROCESS Process, Addr = Address; - if (Ke386Pae) - { - ULONGLONG Pte, tmpPte; - PULONGLONG Pt = NULL; - - oldPdeOffset = PAE_ADDR_TO_PDE_OFFSET(Addr) + 1; - for (i = 0; i < PageCount; i++, Addr = (PVOID)((ULONG_PTR)Addr + PAGE_SIZE)) - { - if (!(Attributes & PA_PRESENT) && Pages[i] != 0) - { - DPRINT1("Setting physical address but not allowing access at address " - "0x%.8X with attributes %x/%x.\n", - Addr, Attributes, flProtect); - KEBUGCHECK(0); - } - PdeOffset = PAE_ADDR_TO_PDE_OFFSET(Addr); - if (oldPdeOffset != PdeOffset) - { - MmUnmapPageTable((PULONG)Pt); - Pt = MmGetPageTableForProcessForPAE(Process, Addr, TRUE); - if (Pt == NULL) - { - KEBUGCHECK(0); - } - } - else - { - Pt++; - } - oldPdeOffset = PdeOffset; - - MmMarkPageMapped(Pages[i]); - tmpPte = PAE_PFN_TO_PTE(Pages[i]) | Attributes; - if (NoExecute) - { - tmpPte |= 0x8000000000000000LL; - } - Pte = ExfpInterlockedExchange64UL(Pt, &tmpPte); - if (PAE_PAGE_MASK((Pte)) != 0LL && !((Pte) & PA_PRESENT)) - { - KEBUGCHECK(0); - } - if (PAE_PAGE_MASK((Pte)) != 0LL) - { - MmMarkPageUnmapped(PAE_PTE_TO_PFN((Pte))); - } - if (Address < MmSystemRangeStart && - ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable != NULL && - Attributes & PA_PRESENT) - { - PUSHORT Ptrc; - - Ptrc = ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable; - - Ptrc[PAE_ADDR_TO_PAGE_TABLE(Addr)]++; - } - if (Pte != 0LL) - { - if (Address > MmSystemRangeStart || - (Pt >= (PULONGLONG)PAGETABLE_MAP && Pt < (PULONGLONG)PAGETABLE_MAP + 4*512*512)) - { - MiFlushTlb((PULONG)Pt, Address); - } - } - } - if (Addr > Address) - { - MmUnmapPageTable((PULONG)Pt); - } - } - else - { - PULONG Pt = NULL; + PULONG Pt = NULL; ULONG Pte; oldPdeOffset = ADDR_TO_PDE_OFFSET(Addr) + 1; for (i = 0; i < PageCount; i++, Addr = (PVOID)((ULONG_PTR)Addr + PAGE_SIZE)) @@ -1925,7 +1131,7 @@ MmCreateVirtualMappingUnsafe(PEPROCESS Process, { MmUnmapPageTable(Pt); } - } + return(STATUS_SUCCESS); } @@ -1961,14 +1167,9 @@ MmGetPageProtect(PEPROCESS Process, PVOID Address) { ULONG Entry; ULONG Protect; - if (Ke386Pae) - { - Entry = MmGetPageEntryForProcessForPAE(Process, Address); - } - else - { + Entry = MmGetPageEntryForProcess(Process, Address); - } + if (!(Entry & PA_PRESENT)) { @@ -2029,35 +1230,6 @@ MmSetPageProtect(PEPROCESS Process, PVOID Address, ULONG flProtect) { Attributes |= PA_USER; } - if (Ke386Pae) - { - PULONGLONG Pt; - ULONGLONG tmpPte, Pte; - - Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE); - if (Pt == NULL) - { - DPRINT1("Address %x\n", Address); - KEBUGCHECK(0); - } - do - { - Pte = *Pt; - tmpPte = PAE_PAGE_MASK(Pte) | Attributes | (Pte & (PA_ACCESSED|PA_DIRTY)); - if (NoExecute) - { - tmpPte |= 0x8000000000000000LL; - } - else - { - tmpPte &= ~0x8000000000000000LL; - } - } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte)); - - MiFlushTlb((PULONG)Pt, Address); - } - else - { PULONG Pt; Pt = MmGetPageTableForProcess(Process, Address, FALSE); @@ -2067,7 +1239,6 @@ MmSetPageProtect(PEPROCESS Process, PVOID Address, ULONG flProtect) } InterlockedExchange((PLONG)Pt, PAGE_MASK(*Pt) | Attributes | (*Pt & (PA_ACCESSED|PA_DIRTY))); MiFlushTlb(Pt, Address); - } } /* @@ -2082,22 +1253,6 @@ MmGetPhysicalAddress(PVOID vaddr) PHYSICAL_ADDRESS p; DPRINT("MmGetPhysicalAddress(vaddr %x)\n", vaddr); - if (Ke386Pae) - { - ULONGLONG Pte; - Pte = MmGetPageEntryForProcessForPAE(NULL, vaddr); - if (Pte != 0 && Pte & PA_PRESENT) - { - p.QuadPart = PAE_PAGE_MASK(Pte); - p.u.LowPart |= (ULONG_PTR)vaddr & (PAGE_SIZE - 1); - } - else - { - p.QuadPart = 0; - } - } - else - { ULONG Pte; Pte = MmGetPageEntryForProcess(NULL, vaddr); if (Pte != 0 && Pte & PA_PRESENT) @@ -2109,7 +1264,6 @@ MmGetPhysicalAddress(PVOID vaddr) { p.QuadPart = 0; } - } return p; } @@ -2120,68 +1274,6 @@ MmCreateHyperspaceMapping(PFN_TYPE Page) PVOID Address; ULONG i; - if (Ke386Pae) - { - ULONGLONG Entry; - ULONGLONG ZeroEntry = 0LL; - PULONGLONG Pte; - - Entry = PFN_TO_PTE(Page) | PA_PRESENT | PA_READWRITE; - Pte = PAE_ADDR_TO_PTE(HYPERSPACE) + Page % 1024; - - if (Page & 1024) - { - for (i = Page %1024; i < 1024; i++, Pte++) - { - if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry)) - { - break; - } - } - if (i >= 1024) - { - Pte = PAE_ADDR_TO_PTE(HYPERSPACE); - for (i = 0; i < Page % 1024; i++, Pte++) - { - if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry)) - { - break; - } - } - if (i >= Page % 1024) - { - KEBUGCHECK(0); - } - } - } - else - { - for (i = Page %1024; (LONG)i >= 0; i--, Pte--) - { - if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry)) - { - break; - } - } - if ((LONG)i < 0) - { - Pte = PAE_ADDR_TO_PTE(HYPERSPACE) + 1023; - for (i = 1023; i > Page % 1024; i--, Pte--) - { - if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry)) - { - break; - } - } - if (i <= Page % 1024) - { - KEBUGCHECK(0); - } - } - } - } - else - { ULONG Entry; PULONG Pte; Entry = PFN_TO_PTE(Page) | PA_PRESENT | PA_READWRITE; @@ -2236,7 +1328,6 @@ MmCreateHyperspaceMapping(PFN_TYPE Page) } } } - } Address = (PVOID)((ULONG_PTR)HYPERSPACE + i * PAGE_SIZE); __invlpg(Address); return Address; @@ -2248,18 +1339,11 @@ MmChangeHyperspaceMapping(PVOID Address, PFN_TYPE NewPage) { PFN_TYPE Pfn; ASSERT (IS_HYPERSPACE(Address)); - if (Ke386Pae) - { - ULONGLONG Entry = PAE_PFN_TO_PTE(NewPage) | PA_PRESENT | PA_READWRITE; - Entry = (ULONG)ExfpInterlockedExchange64UL(PAE_ADDR_TO_PTE(Address), &Entry); - Pfn = PAE_PTE_TO_PFN(Entry); - } - else - { + ULONG Entry; Entry = InterlockedExchange((PLONG)ADDR_TO_PTE(Address), PFN_TO_PTE(NewPage) | PA_PRESENT | PA_READWRITE); Pfn = PTE_TO_PFN(Entry); - } + __invlpg(Address); return Pfn; } @@ -2270,18 +1354,11 @@ MmDeleteHyperspaceMapping(PVOID Address) { PFN_TYPE Pfn; ASSERT (IS_HYPERSPACE(Address)); - if (Ke386Pae) - { - ULONGLONG Entry = 0LL; - Entry = (ULONG)ExfpInterlockedExchange64UL(PAE_ADDR_TO_PTE(Address), &Entry); - Pfn = PAE_PTE_TO_PFN(Entry); - } - else - { + ULONG Entry; Entry = InterlockedExchange((PLONG)ADDR_TO_PTE(Address), 0); Pfn = PTE_TO_PFN(Entry); - } + __invlpg(Address); return Pfn; } @@ -2296,55 +1373,7 @@ MmUpdatePageDir(PEPROCESS Process, PVOID Address, ULONG Size) { KEBUGCHECK(0); } - if (Ke386Pae) - { - PULONGLONG PageDirTable; - PULONGLONG Pde; - ULONGLONG ZeroPde = 0LL; - ULONG i; - for (i = PAE_ADDR_TO_PDTE_OFFSET(Address); i <= PAE_ADDR_TO_PDTE_OFFSET((PVOID)((ULONG_PTR)Address + Size)); i++) - { - if (i == PAE_ADDR_TO_PDTE_OFFSET(Address)) - { - StartOffset = PAE_ADDR_TO_PDE_PAGE_OFFSET(Address); - } - else - { - StartOffset = 0; - } - if (i == PAE_ADDR_TO_PDTE_OFFSET((PVOID)((ULONG_PTR)Address + Size))) - { - EndOffset = PAE_ADDR_TO_PDE_PAGE_OFFSET((PVOID)((ULONG_PTR)Address + Size)); - } - else - { - EndOffset = 511; - } - - if (Process != NULL && Process != PsGetCurrentProcess()) - { - PageDirTable = MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart)); - Pde = (PULONGLONG)MmCreateHyperspaceMapping(PTE_TO_PFN(PageDirTable[i])); - MmDeleteHyperspaceMapping(PageDirTable); - } - else - { - Pde = (PULONGLONG)PAE_PAGEDIRECTORY_MAP + i*512; - } - - for (Offset = StartOffset; Offset <= EndOffset; Offset++) - { - if (i * 512 + Offset < PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) || i * 512 + Offset >= PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP)+4) - { - (void)ExfInterlockedCompareExchange64UL(&Pde[Offset], &MmGlobalKernelPageDirectoryForPAE[i*512 + Offset], &ZeroPde); - } - } - MmUnmapPageTable((PULONG)Pde); - } - } - else - { PULONG Pde; StartOffset = ADDR_TO_PDE_OFFSET(Address); EndOffset = ADDR_TO_PDE_OFFSET((PVOID)((ULONG_PTR)Address + Size)); @@ -2368,7 +1397,6 @@ MmUpdatePageDir(PEPROCESS Process, PVOID Address, ULONG Size) { MmDeleteHyperspaceMapping(Pde); } - } } VOID @@ -2380,26 +1408,6 @@ MmInitGlobalKernelPageDirectory(VOID) DPRINT("MmInitGlobalKernelPageDirectory()\n"); - if (Ke386Pae) - { - PULONGLONG CurrentPageDirectory = (PULONGLONG)PAE_PAGEDIRECTORY_MAP; - for (i = PAE_ADDR_TO_PDE_OFFSET(MmSystemRangeStart); i < 4 * 512; i++) - { - if (!(i >= PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) && i < PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) + 4) && - !(i >= PAE_ADDR_TO_PDE_OFFSET(HYPERSPACE) && i < PAE_ADDR_TO_PDE_OFFSET(HYPERSPACE) + 2) && - 0LL == MmGlobalKernelPageDirectoryForPAE[i] && 0LL != CurrentPageDirectory[i]) - { - (void)ExfpInterlockedExchange64UL(&MmGlobalKernelPageDirectoryForPAE[i], &CurrentPageDirectory[i]); - if (Ke386GlobalPagesEnabled) - { - MmGlobalKernelPageDirectoryForPAE[i] |= PA_GLOBAL; - CurrentPageDirectory[i] |= PA_GLOBAL; - } - } - } - } - else - { PULONG CurrentPageDirectory = (PULONG)PAGEDIRECTORY_MAP; for (i = ADDR_TO_PDE_OFFSET(MmSystemRangeStart); i < 1024; i++) { @@ -2415,14 +1423,13 @@ MmInitGlobalKernelPageDirectory(VOID) } } } - } } ULONG NTAPI MiGetUserPageDirectoryCount(VOID) { - return Ke386Pae ? PAE_ADDR_TO_PDE_OFFSET(MmSystemRangeStart) : ADDR_TO_PDE_OFFSET(MmSystemRangeStart); + return ADDR_TO_PDE_OFFSET(MmSystemRangeStart); } VOID @@ -2443,7 +1450,7 @@ MiInitPageDirectoryMap(VOID) Status = MmCreateMemoryArea(MmGetKernelAddressSpace(), MEMORY_AREA_SYSTEM, &BaseAddress, - Ke386Pae ? 0x800000 : 0x400000, + 0x400000, PAGE_READWRITE, &kernel_map_desc, TRUE, diff --git a/reactos/ntoskrnl/mm/i386/pagepae.c b/reactos/ntoskrnl/mm/i386/pagepae.c new file mode 100644 index 00000000000..5e13e7830aa --- /dev/null +++ b/reactos/ntoskrnl/mm/i386/pagepae.c @@ -0,0 +1,2472 @@ +/* $Id: page.c 32364 2008-02-14 20:30:31Z ros-arm-bringup $ + * + * COPYRIGHT: See COPYING in the top level directory + * PROJECT: ReactOS kernel + * FILE: ntoskrnl/mm/i386/page.c + * PURPOSE: Low level memory managment manipulation + * + * PROGRAMMERS: David Welch (welch@cwcom.net) + */ + +/* INCLUDES ***************************************************************/ + +#include +#define NDEBUG +#include + +#if defined (ALLOC_PRAGMA) +#pragma alloc_text(INIT, MmInitGlobalKernelPageDirectory) +#pragma alloc_text(INIT, MiInitPageDirectoryMap) +#endif + + +/* GLOBALS *****************************************************************/ + +#define PA_BIT_PRESENT (0) +#define PA_BIT_READWRITE (1) +#define PA_BIT_USER (2) +#define PA_BIT_WT (3) +#define PA_BIT_CD (4) +#define PA_BIT_ACCESSED (5) +#define PA_BIT_DIRTY (6) +#define PA_BIT_GLOBAL (8) + +#define PA_PRESENT (1 << PA_BIT_PRESENT) +#define PA_READWRITE (1 << PA_BIT_READWRITE) +#define PA_USER (1 << PA_BIT_USER) +#define PA_DIRTY (1 << PA_BIT_DIRTY) +#define PA_WT (1 << PA_BIT_WT) +#define PA_CD (1 << PA_BIT_CD) +#define PA_ACCESSED (1 << PA_BIT_ACCESSED) +#define PA_GLOBAL (1 << PA_BIT_GLOBAL) + +#define PAGETABLE_MAP (0xc0000000) +#define PAGEDIRECTORY_MAP (0xc0000000 + (PAGETABLE_MAP / (1024))) + +#define PAE_PAGEDIRECTORY_MAP (0xc0000000 + (PAGETABLE_MAP / (512))) + +#define HYPERSPACE (Ke386Pae ? 0xc0800000 : 0xc0400000) +#define IS_HYPERSPACE(v) (((ULONG)(v) >= HYPERSPACE && (ULONG)(v) < HYPERSPACE + 0x400000)) + +ULONG MmGlobalKernelPageDirectory[1024]; +ULONGLONG MmGlobalKernelPageDirectoryForPAE[2048]; + +#define PTE_TO_PFN(X) ((X) >> PAGE_SHIFT) +#define PFN_TO_PTE(X) ((X) << PAGE_SHIFT) + +#define PAE_PTE_TO_PFN(X) (PAE_PAGE_MASK(X) >> PAGE_SHIFT) +#define PAE_PFN_TO_PTE(X) ((X) << PAGE_SHIFT) + +#if defined(__GNUC__) +#define PTE_TO_PAGE(X) ((LARGE_INTEGER)(LONGLONG)(PAGE_MASK(X))) +#else +__inline LARGE_INTEGER PTE_TO_PAGE(ULONG npage) +{ + LARGE_INTEGER dummy; + dummy.QuadPart = (LONGLONG)(PAGE_MASK(npage)); + return dummy; +} +#endif + +extern BOOLEAN Ke386Pae; +extern BOOLEAN Ke386NoExecute; + +/* FUNCTIONS ***************************************************************/ + +BOOLEAN MmUnmapPageTable(PULONG Pt); + +ULONG_PTR +NTAPI +MiFlushTlbIpiRoutine(ULONG_PTR Address) +{ + if (Address == (ULONG_PTR)-1) + { + KeFlushCurrentTb(); + } + else if (Address == (ULONG_PTR)-2) + { + KeFlushCurrentTb(); + } + else + { + __invlpg((PVOID)Address); + } + return 0; +} + +VOID +MiFlushTlb(PULONG Pt, PVOID Address) +{ +#ifdef CONFIG_SMP + if (Pt) + { + MmUnmapPageTable(Pt); + } + if (KeNumberProcessors > 1) + { + KeIpiGenericCall(MiFlushTlbIpiRoutine, (ULONG_PTR)Address); + } + else + { + MiFlushTlbIpiRoutine((ULONG_PTR)Address); + } +#else + if ((Pt && MmUnmapPageTable(Pt)) || Address >= MmSystemRangeStart) + { + __invlpg(Address); + } +#endif +} + + + +PULONG +MmGetPageDirectory(VOID) +{ + return (PULONG)__readcr3(); +} + +static ULONG +ProtectToPTE(ULONG flProtect) +{ + ULONG Attributes = 0; + + if (flProtect & (PAGE_NOACCESS|PAGE_GUARD)) + { + Attributes = 0; + } + else if (flProtect & PAGE_IS_WRITABLE) + { + Attributes = PA_PRESENT | PA_READWRITE; + } + else if (flProtect & (PAGE_IS_READABLE | PAGE_IS_EXECUTABLE)) + { + Attributes = PA_PRESENT; + } + else + { + DPRINT1("Unknown main protection type.\n"); + KEBUGCHECK(0); + } + if (Ke386NoExecute && + !(flProtect & PAGE_IS_EXECUTABLE)) + { + Attributes = Attributes | 0x80000000; + } + + if (flProtect & PAGE_SYSTEM) + { + } + else + { + Attributes = Attributes | PA_USER; + } + if (flProtect & PAGE_NOCACHE) + { + Attributes = Attributes | PA_CD; + } + if (flProtect & PAGE_WRITETHROUGH) + { + Attributes = Attributes | PA_WT; + } + return(Attributes); +} + +#define ADDR_TO_PAGE_TABLE(v) (((ULONG)(v)) / (1024 * PAGE_SIZE)) + +#define ADDR_TO_PDE(v) (PULONG)(PAGEDIRECTORY_MAP + \ + ((((ULONG)(v)) / (1024 * 1024))&(~0x3))) +#define ADDR_TO_PTE(v) (PULONG)(PAGETABLE_MAP + ((((ULONG)(v) / 1024))&(~0x3))) + +#define ADDR_TO_PDE_OFFSET(v) ((((ULONG)(v)) / (1024 * PAGE_SIZE))) + +#define ADDR_TO_PTE_OFFSET(v) ((((ULONG)(v)) % (1024 * PAGE_SIZE)) / PAGE_SIZE) + + +#define PAE_ADDR_TO_PAGE_TABLE(v) (((ULONG)(v)) / (512 * PAGE_SIZE)) + +#define PAE_ADDR_TO_PDE(v) (PULONGLONG) (PAE_PAGEDIRECTORY_MAP + \ + ((((ULONG_PTR)(v)) / (512 * 512))&(~0x7))) +#define PAE_ADDR_TO_PTE(v) (PULONGLONG) (PAGETABLE_MAP + ((((ULONG_PTR)(v) / 512))&(~0x7))) + + +#define PAE_ADDR_TO_PDTE_OFFSET(v) (((ULONG_PTR)(v)) / (512 * 512 * PAGE_SIZE)) + +#define PAE_ADDR_TO_PDE_PAGE_OFFSET(v) ((((ULONG_PTR)(v)) % (512 * 512 * PAGE_SIZE)) / (512 * PAGE_SIZE)) + +#define PAE_ADDR_TO_PDE_OFFSET(v) (((ULONG_PTR)(v))/ (512 * PAGE_SIZE)) + +#define PAE_ADDR_TO_PTE_OFFSET(v) ((((ULONG_PTR)(v)) % (512 * PAGE_SIZE)) / PAGE_SIZE) + + +NTSTATUS +NTAPI +Mmi386ReleaseMmInfo(PEPROCESS Process) +{ + PUSHORT LdtDescriptor; + ULONG LdtBase; + ULONG i, j; + + DPRINT("Mmi386ReleaseMmInfo(Process %x)\n",Process); + + LdtDescriptor = (PUSHORT) &Process->Pcb.LdtDescriptor; + LdtBase = LdtDescriptor[1] | + ((LdtDescriptor[2] & 0xff) << 16) | + ((LdtDescriptor[3] & ~0xff) << 16); + + DPRINT("LdtBase: %x\n", LdtBase); + + if (LdtBase) + { + ExFreePool((PVOID) LdtBase); + } + + if (Ke386Pae) + { + PULONGLONG PageDirTable; + PULONGLONG PageDir; + PULONGLONG Pde; + ULONG k; + + PageDirTable = (PULONGLONG)MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart)); + for (i = 0; i < 4; i++) + { + PageDir = (PULONGLONG)MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(PageDirTable[i])); + if (i < PAE_ADDR_TO_PDTE_OFFSET(MmSystemRangeStart)) + { + for (j = 0; j < 512; j++) + { + if (PageDir[j] != 0LL) + { + DPRINT1("ProcessId %d, Pde for %08x - %08x is not freed, RefCount %d\n", + Process->UniqueProcessId, + (i * 512 + j) * 512 * PAGE_SIZE, (i * 512 + j + 1) * 512 * PAGE_SIZE - 1, + ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable[i*512 + j]); + Pde = MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(PageDir[j])); + for (k = 0; k < 512; k++) + { + if(Pde[k] != 0) + { + if (Pde[k] & PA_PRESENT) + { + DPRINT1("Page at %08x is not freed\n", + (i * 512 + j) * 512 * PAGE_SIZE + k * PAGE_SIZE); + } + else + { + DPRINT1("Swapentry %x at %x is not freed\n", + (i * 512 + j) * 512 * PAGE_SIZE + k * PAGE_SIZE); + } + } + } + MmDeleteHyperspaceMapping(Pde); + MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(PageDir[j])); + } + } + } + if (i == PAE_ADDR_TO_PDTE_OFFSET(HYPERSPACE)) + { + MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE)])); + MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE)+1])); + } + MmDeleteHyperspaceMapping(PageDir); + MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(PageDirTable[i])); + } + MmDeleteHyperspaceMapping((PVOID)PageDirTable); + MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart)); + } + else + { + PULONG Pde; + PULONG PageDir; + PageDir = MmCreateHyperspaceMapping(PTE_TO_PFN(Process->Pcb.DirectoryTableBase.u.LowPart)); + for (i = 0; i < ADDR_TO_PDE_OFFSET(MmSystemRangeStart); i++) + { + if (PageDir[i] != 0) + { + DPRINT1("Pde for %08x - %08x is not freed, RefCount %d\n", + i * 4 * 1024 * 1024, (i + 1) * 4 * 1024 * 1024 - 1, + ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable[i]); + Pde = MmCreateHyperspaceMapping(PTE_TO_PFN(PageDir[i])); + for (j = 0; j < 1024; j++) + { + if(Pde[j] != 0) + { + if (Pde[j] & PA_PRESENT) + { + DPRINT1("Page at %08x is not freed\n", + i * 4 * 1024 * 1024 + j * PAGE_SIZE); + } + else + { + DPRINT1("Swapentry %x at %x is not freed\n", + Pde[j], i * 4 * 1024 * 1024 + j * PAGE_SIZE); + } + } + } + MmDeleteHyperspaceMapping(Pde); + MmReleasePageMemoryConsumer(MC_NPPOOL, PTE_TO_PFN(PageDir[i])); + } + } + MmReleasePageMemoryConsumer(MC_NPPOOL, PTE_TO_PFN(PageDir[ADDR_TO_PDE_OFFSET(HYPERSPACE)])); + MmDeleteHyperspaceMapping(PageDir); + MmReleasePageMemoryConsumer(MC_NPPOOL, PTE_TO_PFN(Process->Pcb.DirectoryTableBase.u.LowPart)); + } + +#if defined(__GNUC__) + + Process->Pcb.DirectoryTableBase.QuadPart = 0LL; +#else + + Process->Pcb.DirectoryTableBase.QuadPart = 0; +#endif + + DPRINT("Finished Mmi386ReleaseMmInfo()\n"); + return(STATUS_SUCCESS); +} + +NTSTATUS +NTAPI +MmInitializeHandBuiltProcess(IN PEPROCESS Process, + IN PLARGE_INTEGER DirectoryTableBase) +{ + /* Share the directory base with the idle process */ + *DirectoryTableBase = PsGetCurrentProcess()->Pcb.DirectoryTableBase; + + /* Initialize the Addresss Space */ + MmInitializeAddressSpace(Process, (PMADDRESS_SPACE)&Process->VadRoot); + + /* The process now has an address space */ + Process->HasAddressSpace = TRUE; + return STATUS_SUCCESS; +} + +BOOLEAN +STDCALL +MmCreateProcessAddressSpace(IN ULONG MinWs, + IN PEPROCESS Process, + IN PLARGE_INTEGER DirectoryTableBase) +{ + NTSTATUS Status; + ULONG i, j; + PFN_TYPE Pfn[7]; + ULONG Count; + + DPRINT("MmCopyMmInfo(Src %x, Dest %x)\n", MinWs, Process); + + Count = Ke386Pae ? 7 : 2; + + for (i = 0; i < Count; i++) + { + Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn[i]); + if (!NT_SUCCESS(Status)) + { + for (j = 0; j < i; j++) + { + MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn[j]); + } + + return FALSE; + } + } + + if (Ke386Pae) + { + PULONGLONG PageDirTable; + PULONGLONG PageDir; + + PageDirTable = MmCreateHyperspaceMapping(Pfn[0]); + for (i = 0; i < 4; i++) + { + PageDirTable[i] = PAE_PFN_TO_PTE(Pfn[1+i]) | PA_PRESENT; + } + MmDeleteHyperspaceMapping(PageDirTable); + for (i = PAE_ADDR_TO_PDTE_OFFSET(MmSystemRangeStart); i < 4; i++) + { + PageDir = (PULONGLONG)MmCreateHyperspaceMapping(Pfn[i+1]); + memcpy(PageDir, &MmGlobalKernelPageDirectoryForPAE[i * 512], 512 * sizeof(ULONGLONG)); + if (PAE_ADDR_TO_PDTE_OFFSET(PAGETABLE_MAP) == i) + { + for (j = 0; j < 4; j++) + { + PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(PAGETABLE_MAP) + j] = PAE_PFN_TO_PTE(Pfn[1+j]) | PA_PRESENT | PA_READWRITE; + } + } + if (PAE_ADDR_TO_PDTE_OFFSET(HYPERSPACE) == i) + { + PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE)] = PAE_PFN_TO_PTE(Pfn[5]) | PA_PRESENT | PA_READWRITE; + PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE)+1] = PAE_PFN_TO_PTE(Pfn[6]) | PA_PRESENT | PA_READWRITE; + } + MmDeleteHyperspaceMapping(PageDir); + } + } + else + { + PULONG PageDirectory; + PageDirectory = MmCreateHyperspaceMapping(Pfn[0]); + + memcpy(PageDirectory + ADDR_TO_PDE_OFFSET(MmSystemRangeStart), + MmGlobalKernelPageDirectory + ADDR_TO_PDE_OFFSET(MmSystemRangeStart), + (1024 - ADDR_TO_PDE_OFFSET(MmSystemRangeStart)) * sizeof(ULONG)); + + DPRINT("Addr %x\n",ADDR_TO_PDE_OFFSET(PAGETABLE_MAP)); + PageDirectory[ADDR_TO_PDE_OFFSET(PAGETABLE_MAP)] = PFN_TO_PTE(Pfn[0]) | PA_PRESENT | PA_READWRITE; + PageDirectory[ADDR_TO_PDE_OFFSET(HYPERSPACE)] = PFN_TO_PTE(Pfn[1]) | PA_PRESENT | PA_READWRITE; + + MmDeleteHyperspaceMapping(PageDirectory); + } + + DirectoryTableBase->QuadPart = PFN_TO_PTE(Pfn[0]); + DPRINT("Finished MmCopyMmInfo(): %I64x\n", DirectoryTableBase->QuadPart); + return TRUE; +} + +VOID +NTAPI +MmDeletePageTable(PEPROCESS Process, PVOID Address) +{ + PEPROCESS CurrentProcess = PsGetCurrentProcess(); + + if (Process != NULL && Process != CurrentProcess) + { + KeAttachProcess(&Process->Pcb); + } + + if (Ke386Pae) + { + ULONGLONG ZeroPde = 0LL; + (void)ExfpInterlockedExchange64UL(PAE_ADDR_TO_PDE(Address), &ZeroPde); + MiFlushTlb((PULONG)PAE_ADDR_TO_PDE(Address), PAE_ADDR_TO_PTE(Address)); + } + else + { + *(ADDR_TO_PDE(Address)) = 0; + MiFlushTlb(ADDR_TO_PDE(Address), ADDR_TO_PTE(Address)); + } + if (Address >= MmSystemRangeStart) + { + KEBUGCHECK(0); + // MmGlobalKernelPageDirectory[ADDR_TO_PDE_OFFSET(Address)] = 0; + } + if (Process != NULL && Process != CurrentProcess) + { + KeDetachProcess(); + } +} + +VOID +NTAPI +MmFreePageTable(PEPROCESS Process, PVOID Address) +{ + PEPROCESS CurrentProcess = PsGetCurrentProcess(); + ULONG i; + PFN_TYPE Pfn; + + DPRINT("ProcessId %d, Address %x\n", Process->UniqueProcessId, Address); + if (Process != NULL && Process != CurrentProcess) + { + KeAttachProcess(&Process->Pcb); + } + if (Ke386Pae) + { + PULONGLONG PageTable; + ULONGLONG ZeroPte = 0LL; + PageTable = (PULONGLONG)PAGE_ROUND_DOWN((PVOID)PAE_ADDR_TO_PTE(Address)); + for (i = 0; i < 512; i++) + { + if (PageTable[i] != 0LL) + { + DbgPrint("Page table entry not clear at %x/%x (is %I64x)\n", + ((ULONG)Address / (4*1024*1024)), i, PageTable[i]); + KEBUGCHECK(0); + } + } + Pfn = PAE_PTE_TO_PFN(*(PAE_ADDR_TO_PDE(Address))); + (void)ExfpInterlockedExchange64UL(PAE_ADDR_TO_PDE(Address), &ZeroPte); + MiFlushTlb((PULONG)PAE_ADDR_TO_PDE(Address), PAE_ADDR_TO_PTE(Address)); + } + else + { + PULONG PageTable; + PageTable = (PULONG)PAGE_ROUND_DOWN((PVOID)ADDR_TO_PTE(Address)); + for (i = 0; i < 1024; i++) + { + if (PageTable[i] != 0) + { + DbgPrint("Page table entry not clear at %x/%x (is %x)\n", + ((ULONG)Address / (4*1024*1024)), i, PageTable[i]); + KEBUGCHECK(0); + } + } + Pfn = PTE_TO_PFN(*(ADDR_TO_PDE(Address))); + *(ADDR_TO_PDE(Address)) = 0; + MiFlushTlb(ADDR_TO_PDE(Address), ADDR_TO_PTE(Address)); + } + + if (Address >= MmSystemRangeStart) + { + // MmGlobalKernelPageDirectory[ADDR_TO_PDE_OFFSET(Address)] = 0; + KEBUGCHECK(0); + } + else + { + MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn); + } + if (Process != NULL && Process != CurrentProcess) + { + KeDetachProcess(); + } +} + +static PULONGLONG +MmGetPageTableForProcessForPAE(PEPROCESS Process, PVOID Address, BOOLEAN Create) +{ + NTSTATUS Status; + PFN_TYPE Pfn; + ULONGLONG Entry; + ULONGLONG ZeroEntry = 0LL; + PULONGLONG Pt; + PULONGLONG PageDir; + PULONGLONG PageDirTable; + + DPRINT("MmGetPageTableForProcessForPAE(%x %x %d)\n", + Process, Address, Create); + if (Address >= (PVOID)PAGETABLE_MAP && Address < (PVOID)((ULONG_PTR)PAGETABLE_MAP + 0x800000)) + { + KEBUGCHECK(0); + } + if (Address < MmSystemRangeStart && Process && Process != PsGetCurrentProcess()) + { + PageDirTable = MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart)); + if (PageDirTable == NULL) + { + KEBUGCHECK(0); + } + PageDir = MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(PageDirTable[PAE_ADDR_TO_PDTE_OFFSET(Address)])); + MmDeleteHyperspaceMapping(PageDirTable); + if (PageDir == NULL) + { + KEBUGCHECK(0); + } + PageDir += PAE_ADDR_TO_PDE_PAGE_OFFSET(Address); + Entry = ExfInterlockedCompareExchange64UL(PageDir, &ZeroEntry, &ZeroEntry); + if (Entry == 0LL) + { + if (Create == FALSE) + { + MmDeleteHyperspaceMapping(PageDir); + return NULL; + } + Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn); + if (!NT_SUCCESS(Status)) + { + KEBUGCHECK(0); + } + Entry = PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE | PA_USER; + Entry = ExfInterlockedCompareExchange64UL(PageDir, &Entry, &ZeroEntry); + if (Entry != 0LL) + { + MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn); + Pfn = PAE_PTE_TO_PFN(Entry); + } + } + else + { + Pfn = PAE_PTE_TO_PFN(Entry); + } + MmDeleteHyperspaceMapping(PageDir); + Pt = MmCreateHyperspaceMapping(Pfn); + if (Pt == NULL) + { + KEBUGCHECK(0); + } + return Pt + PAE_ADDR_TO_PTE_OFFSET(Address); + } + PageDir = PAE_ADDR_TO_PDE(Address); + if (0LL == ExfInterlockedCompareExchange64UL(PageDir, &ZeroEntry, &ZeroEntry)) + { + if (Address >= MmSystemRangeStart) + { + if (MmGlobalKernelPageDirectoryForPAE[PAE_ADDR_TO_PDE_OFFSET(Address)] == 0LL) + { + if (Create == FALSE) + { + return NULL; + } + Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn); + if (!NT_SUCCESS(Status)) + { + KEBUGCHECK(0); + } + Entry = PAE_PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE; + if (Ke386GlobalPagesEnabled) + { + Entry |= PA_GLOBAL; + } + if (0LL != ExfInterlockedCompareExchange64UL(&MmGlobalKernelPageDirectoryForPAE[PAE_ADDR_TO_PDE_OFFSET(Address)], &Entry, &ZeroEntry)) + { + MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn); + } + } + (void)ExfInterlockedCompareExchange64UL(PageDir, &MmGlobalKernelPageDirectoryForPAE[PAE_ADDR_TO_PDE_OFFSET(Address)], &ZeroEntry); + } + else + { + if (Create == FALSE) + { + return NULL; + } + Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn); + if (!NT_SUCCESS(Status)) + { + KEBUGCHECK(0); + } + Entry = PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE | PA_USER; + Entry = ExfInterlockedCompareExchange64UL(PageDir, &Entry, &ZeroEntry); + if (Entry != 0LL) + { + MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn); + } + } + } + return (PULONGLONG)PAE_ADDR_TO_PTE(Address); +} + +static PULONG +MmGetPageTableForProcess(PEPROCESS Process, PVOID Address, BOOLEAN Create) +{ + ULONG PdeOffset = ADDR_TO_PDE_OFFSET(Address); + NTSTATUS Status; + PFN_TYPE Pfn; + ULONG Entry; + PULONG Pt, PageDir; + + if (Address < MmSystemRangeStart && Process && Process != PsGetCurrentProcess()) + { + PageDir = MmCreateHyperspaceMapping(PTE_TO_PFN(Process->Pcb.DirectoryTableBase.LowPart)); + if (PageDir == NULL) + { + KEBUGCHECK(0); + } + if (0 == InterlockedCompareExchangeUL(&PageDir[PdeOffset], 0, 0)) + { + if (Create == FALSE) + { + MmDeleteHyperspaceMapping(PageDir); + return NULL; + } + Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn); + if (!NT_SUCCESS(Status) || Pfn == 0) + { + KEBUGCHECK(0); + } + Entry = InterlockedCompareExchangeUL(&PageDir[PdeOffset], PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE | PA_USER, 0); + if (Entry != 0) + { + MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn); + Pfn = PTE_TO_PFN(Entry); + } + } + else + { + Pfn = PTE_TO_PFN(PageDir[PdeOffset]); + } + MmDeleteHyperspaceMapping(PageDir); + Pt = MmCreateHyperspaceMapping(Pfn); + if (Pt == NULL) + { + KEBUGCHECK(0); + } + return Pt + ADDR_TO_PTE_OFFSET(Address); + } + PageDir = ADDR_TO_PDE(Address); + if (0 == InterlockedCompareExchangeUL(PageDir, 0, 0)) + { + if (Address >= MmSystemRangeStart) + { + if (0 == InterlockedCompareExchangeUL(&MmGlobalKernelPageDirectory[PdeOffset], 0, 0)) + { + if (Create == FALSE) + { + return NULL; + } + Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn); + if (!NT_SUCCESS(Status) || Pfn == 0) + { + KEBUGCHECK(0); + } + Entry = PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE; + if (Ke386GlobalPagesEnabled) + { + Entry |= PA_GLOBAL; + } + if(0 != InterlockedCompareExchangeUL(&MmGlobalKernelPageDirectory[PdeOffset], Entry, 0)) + { + MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn); + } + } + (void)InterlockedExchangeUL(PageDir, MmGlobalKernelPageDirectory[PdeOffset]); + } + else + { + if (Create == FALSE) + { + return NULL; + } + Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn); + if (!NT_SUCCESS(Status) || Pfn == 0) + { + KEBUGCHECK(0); + } + Entry = InterlockedCompareExchangeUL(PageDir, PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE | PA_USER, 0); + if (Entry != 0) + { + MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn); + } + } + } + return (PULONG)ADDR_TO_PTE(Address); +} + +BOOLEAN MmUnmapPageTable(PULONG Pt) +{ + if (Ke386Pae) + { + if ((PULONGLONG)Pt >= (PULONGLONG)PAGETABLE_MAP && (PULONGLONG)Pt < (PULONGLONG)PAGETABLE_MAP + 4*512*512) + { + return TRUE; + } + } + else + { + if (Pt >= (PULONG)PAGETABLE_MAP && Pt < (PULONG)PAGETABLE_MAP + 1024*1024) + { + return TRUE; + } + } + if (Pt) + { + MmDeleteHyperspaceMapping((PVOID)PAGE_ROUND_DOWN(Pt)); + } + return FALSE; +} + +static ULONGLONG MmGetPageEntryForProcessForPAE(PEPROCESS Process, PVOID Address) +{ + ULONGLONG Pte; + PULONGLONG Pt; + + Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE); + if (Pt) + { + Pte = *Pt; + MmUnmapPageTable((PULONG)Pt); + return Pte; + } + return 0; +} + +static ULONG MmGetPageEntryForProcess(PEPROCESS Process, PVOID Address) +{ + ULONG Pte; + PULONG Pt; + + Pt = MmGetPageTableForProcess(Process, Address, FALSE); + if (Pt) + { + Pte = *Pt; + MmUnmapPageTable(Pt); + return Pte; + } + return 0; +} + +PFN_TYPE +NTAPI +MmGetPfnForProcess(PEPROCESS Process, + PVOID Address) +{ + + if (Ke386Pae) + { + ULONGLONG Entry; + Entry = MmGetPageEntryForProcessForPAE(Process, Address); + if (!(Entry & PA_PRESENT)) + { + return 0; + } + return(PAE_PTE_TO_PFN(Entry)); + } + else + { + ULONG Entry; + Entry = MmGetPageEntryForProcess(Process, Address); + if (!(Entry & PA_PRESENT)) + { + return 0; + } + return(PTE_TO_PFN(Entry)); + } +} + +VOID +NTAPI +MmDisableVirtualMapping(PEPROCESS Process, PVOID Address, BOOLEAN* WasDirty, PPFN_TYPE Page) +/* + * FUNCTION: Delete a virtual mapping + */ +{ + BOOLEAN WasValid; + if (Ke386Pae) + { + ULONGLONG Pte; + ULONGLONG tmpPte; + PULONGLONG Pt; + + Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE); + if (Pt == NULL) + { + KEBUGCHECK(0); + } + /* + * Atomically disable the present bit and get the old value. + */ + do + { + Pte = *Pt; + tmpPte = Pte & ~PA_PRESENT; + } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte)); + + MiFlushTlb((PULONG)Pt, Address); + WasValid = PAE_PAGE_MASK(Pte) != 0LL ? TRUE : FALSE; + if (!WasValid) + { + KEBUGCHECK(0); + } + + /* + * Return some information to the caller + */ + if (WasDirty != NULL) + { + *WasDirty = Pte & PA_DIRTY ? TRUE : FALSE; + } + if (Page != NULL) + { + *Page = PAE_PTE_TO_PFN(Pte); + } + } + else + { + ULONG Pte; + PULONG Pt; + + Pt = MmGetPageTableForProcess(Process, Address, FALSE); + if (Pt == NULL) + { + KEBUGCHECK(0); + } + /* + * Atomically disable the present bit and get the old value. + */ + do + { + Pte = *Pt; + } while (Pte != InterlockedCompareExchangeUL(Pt, Pte & ~PA_PRESENT, Pte)); + + MiFlushTlb(Pt, Address); + WasValid = (PAGE_MASK(Pte) != 0); + if (!WasValid) + { + KEBUGCHECK(0); + } + + /* + * Return some information to the caller + */ + if (WasDirty != NULL) + { + *WasDirty = Pte & PA_DIRTY; + } + if (Page != NULL) + { + *Page = PTE_TO_PFN(Pte); + } + } +} + +VOID +NTAPI +MmRawDeleteVirtualMapping(PVOID Address) +{ + if (Ke386Pae) + { + PULONGLONG Pt; + ULONGLONG ZeroPte = 0LL; + Pt = MmGetPageTableForProcessForPAE(NULL, Address, FALSE); + if (Pt) + { + /* + * Set the entry to zero + */ + (void)ExfpInterlockedExchange64UL(Pt, &ZeroPte); + MiFlushTlb((PULONG)Pt, Address); + } + } + else + { + PULONG Pt; + + Pt = MmGetPageTableForProcess(NULL, Address, FALSE); + if (Pt && *Pt) + { + /* + * Set the entry to zero + */ + (void)InterlockedExchangeUL(Pt, 0); + MiFlushTlb(Pt, Address); + } + } +} + +VOID +NTAPI +MmDeleteVirtualMapping(PEPROCESS Process, PVOID Address, BOOLEAN FreePage, + BOOLEAN* WasDirty, PPFN_TYPE Page) +/* + * FUNCTION: Delete a virtual mapping + */ +{ + BOOLEAN WasValid = FALSE; + PFN_TYPE Pfn; + + DPRINT("MmDeleteVirtualMapping(%x, %x, %d, %x, %x)\n", + Process, Address, FreePage, WasDirty, Page); + if (Ke386Pae) + { + ULONGLONG Pte; + PULONGLONG Pt; + + Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE); + if (Pt == NULL) + { + if (WasDirty != NULL) + { + *WasDirty = FALSE; + } + if (Page != NULL) + { + *Page = 0; + } + return; + } + + /* + * Atomically set the entry to zero and get the old value. + */ + Pte = 0LL; + Pte = ExfpInterlockedExchange64UL(Pt, &Pte); + + MiFlushTlb((PULONG)Pt, Address); + + WasValid = PAE_PAGE_MASK(Pte) != 0 ? TRUE : FALSE; + if (WasValid) + { + Pfn = PAE_PTE_TO_PFN(Pte); + MmMarkPageUnmapped(Pfn); + } + else + { + Pfn = 0; + } + + if (FreePage && WasValid) + { + MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn); + } + + /* + * Return some information to the caller + */ + if (WasDirty != NULL) + { + *WasDirty = Pte & PA_DIRTY ? TRUE : FALSE; + } + if (Page != NULL) + { + *Page = Pfn; + } + } + else + { + ULONG Pte; + PULONG Pt; + + Pt = MmGetPageTableForProcess(Process, Address, FALSE); + + if (Pt == NULL) + { + if (WasDirty != NULL) + { + *WasDirty = FALSE; + } + if (Page != NULL) + { + *Page = 0; + } + return; + } + + /* + * Atomically set the entry to zero and get the old value. + */ + Pte = InterlockedExchangeUL(Pt, 0); + + MiFlushTlb(Pt, Address); + + WasValid = (PAGE_MASK(Pte) != 0); + if (WasValid) + { + Pfn = PTE_TO_PFN(Pte); + MmMarkPageUnmapped(Pfn); + } + else + { + Pfn = 0; + } + + if (FreePage && WasValid) + { + MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn); + } + + /* + * Return some information to the caller + */ + if (WasDirty != NULL) + { + *WasDirty = Pte & PA_DIRTY ? TRUE : FALSE; + } + if (Page != NULL) + { + *Page = Pfn; + } + } + /* + * Decrement the reference count for this page table. + */ + if (Process != NULL && WasValid && + ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable != NULL && + Address < MmSystemRangeStart) + { + PUSHORT Ptrc; + ULONG Idx; + + Ptrc = ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable; + Idx = Ke386Pae ? PAE_ADDR_TO_PAGE_TABLE(Address) : ADDR_TO_PAGE_TABLE(Address); + + Ptrc[Idx]--; + if (Ptrc[Idx] == 0) + { + MmFreePageTable(Process, Address); + } + } +} + +VOID +NTAPI +MmDeletePageFileMapping(PEPROCESS Process, PVOID Address, + SWAPENTRY* SwapEntry) +/* + * FUNCTION: Delete a virtual mapping + */ +{ + if (Ke386Pae) + { + ULONGLONG Pte; + PULONGLONG Pt; + + Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE); + if (Pt == NULL) + { + *SwapEntry = 0; + return; + } + + /* + * Atomically set the entry to zero and get the old value. + */ + Pte = 0LL; + Pte = ExfpInterlockedExchange64UL(Pt, &Pte); + + MiFlushTlb((PULONG)Pt, Address); + + /* + * Decrement the reference count for this page table. + */ + if (Process != NULL && Pte && + ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable != NULL && + Address < MmSystemRangeStart) + { + PUSHORT Ptrc; + + Ptrc = ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable; + + Ptrc[PAE_ADDR_TO_PAGE_TABLE(Address)]--; + if (Ptrc[PAE_ADDR_TO_PAGE_TABLE(Address)] == 0) + { + MmFreePageTable(Process, Address); + } + } + + + /* + * Return some information to the caller + */ + *SwapEntry = Pte >> 1; + } + else + { + ULONG Pte; + PULONG Pt; + + Pt = MmGetPageTableForProcess(Process, Address, FALSE); + + if (Pt == NULL) + { + *SwapEntry = 0; + return; + } + + /* + * Atomically set the entry to zero and get the old value. + */ + Pte = InterlockedExchangeUL(Pt, 0); + + MiFlushTlb(Pt, Address); + + /* + * Decrement the reference count for this page table. + */ + if (Process != NULL && Pte && + ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable != NULL && + Address < MmSystemRangeStart) + { + PUSHORT Ptrc; + + Ptrc = ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable; + + Ptrc[ADDR_TO_PAGE_TABLE(Address)]--; + if (Ptrc[ADDR_TO_PAGE_TABLE(Address)] == 0) + { + MmFreePageTable(Process, Address); + } + } + + + /* + * Return some information to the caller + */ + *SwapEntry = Pte >> 1; + } +} + +BOOLEAN +Mmi386MakeKernelPageTableGlobal(PVOID PAddress) +{ + if (Ke386Pae) + { + PULONGLONG Pt; + PULONGLONG Pde; + Pde = PAE_ADDR_TO_PDE(PAddress); + if (*Pde == 0LL) + { + Pt = MmGetPageTableForProcessForPAE(NULL, PAddress, FALSE); +#if 0 + /* Non existing mappings are not cached within the tlb. We must not invalidate this entry */ + FLASH_TLB_ONE(PAddress); +#endif + if (Pt != NULL) + { + return TRUE; + } + } + } + else + { + PULONG Pt, Pde; + Pde = ADDR_TO_PDE(PAddress); + if (*Pde == 0) + { + Pt = MmGetPageTableForProcess(NULL, PAddress, FALSE); +#if 0 + /* Non existing mappings are not cached within the tlb. We must not invalidate this entry */ + FLASH_TLB_ONE(PAddress); +#endif + if (Pt != NULL) + { + return TRUE; + } + } + } + return(FALSE); +} + +BOOLEAN +NTAPI +MmIsDirtyPage(PEPROCESS Process, PVOID Address) +{ + if (Ke386Pae) + { + return MmGetPageEntryForProcessForPAE(Process, Address) & PA_DIRTY ? TRUE : FALSE; + } + else + { + return MmGetPageEntryForProcess(Process, Address) & PA_DIRTY ? TRUE : FALSE; + } +} + +BOOLEAN +NTAPI +MmIsAccessedAndResetAccessPage(PEPROCESS Process, PVOID Address) +{ + if (Address < MmSystemRangeStart && Process == NULL) + { + DPRINT1("MmIsAccessedAndResetAccessPage is called for user space without a process.\n"); + KEBUGCHECK(0); + } + if (Ke386Pae) + { + PULONGLONG Pt; + ULONGLONG Pte; + ULONGLONG tmpPte; + + Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE); + if (Pt == NULL) + { + KEBUGCHECK(0); + } + + do + { + Pte = *Pt; + tmpPte = Pte & ~PA_ACCESSED; + } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte)); + + if (Pte & PA_ACCESSED) + { + MiFlushTlb((PULONG)Pt, Address); + return TRUE; + } + else + { + MmUnmapPageTable((PULONG)Pt); + return FALSE; + } + } + else + { + PULONG Pt; + ULONG Pte; + + Pt = MmGetPageTableForProcess(Process, Address, FALSE); + if (Pt == NULL) + { + KEBUGCHECK(0); + } + + do + { + Pte = *Pt; + } while (Pte != InterlockedCompareExchangeUL(Pt, Pte & ~PA_ACCESSED, Pte)); + + if (Pte & PA_ACCESSED) + { + MiFlushTlb(Pt, Address); + return TRUE; + } + else + { + MmUnmapPageTable(Pt); + return FALSE; + } + } +} + +VOID +NTAPI +MmSetCleanPage(PEPROCESS Process, PVOID Address) +{ + if (Address < MmSystemRangeStart && Process == NULL) + { + DPRINT1("MmSetCleanPage is called for user space without a process.\n"); + KEBUGCHECK(0); + } + if (Ke386Pae) + { + PULONGLONG Pt; + ULONGLONG Pte; + ULONGLONG tmpPte; + + Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE); + + if (Pt == NULL) + { + KEBUGCHECK(0); + } + + do + { + Pte = *Pt; + tmpPte = Pte & ~PA_DIRTY; + } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte)); + + if (Pte & PA_DIRTY) + { + MiFlushTlb((PULONG)Pt, Address); + } + else + { + MmUnmapPageTable((PULONG)Pt); + } + } + else + { + PULONG Pt; + ULONG Pte; + + Pt = MmGetPageTableForProcess(Process, Address, FALSE); + + if (Pt == NULL) + { + KEBUGCHECK(0); + } + + do + { + Pte = *Pt; + } while (Pte != InterlockedCompareExchangeUL(Pt, Pte & ~PA_DIRTY, Pte)); + + if (Pte & PA_DIRTY) + { + MiFlushTlb(Pt, Address); + } + else + { + MmUnmapPageTable(Pt); + } + } +} + +VOID +NTAPI +MmSetDirtyPage(PEPROCESS Process, PVOID Address) +{ + if (Address < MmSystemRangeStart && Process == NULL) + { + DPRINT1("MmSetDirtyPage is called for user space without a process.\n"); + KEBUGCHECK(0); + } + if (Ke386Pae) + { + PULONGLONG Pt; + ULONGLONG Pte; + ULONGLONG tmpPte; + + Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE); + if (Pt == NULL) + { + KEBUGCHECK(0); + } + + do + { + Pte = *Pt; + tmpPte = Pte | PA_DIRTY; + } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte)); + if (!(Pte & PA_DIRTY)) + { + MiFlushTlb((PULONG)Pt, Address); + } + else + { + MmUnmapPageTable((PULONG)Pt); + } + } + else + { + PULONG Pt; + ULONG Pte; + + Pt = MmGetPageTableForProcess(Process, Address, FALSE); + if (Pt == NULL) + { + KEBUGCHECK(0); + } + + do + { + Pte = *Pt; + } while (Pte != InterlockedCompareExchangeUL(Pt, Pte | PA_DIRTY, Pte)); + if (!(Pte & PA_DIRTY)) + { + MiFlushTlb(Pt, Address); + } + else + { + MmUnmapPageTable(Pt); + } + } +} + +VOID +NTAPI +MmEnableVirtualMapping(PEPROCESS Process, PVOID Address) +{ + if (Ke386Pae) + { + PULONGLONG Pt; + ULONGLONG Pte; + ULONGLONG tmpPte; + + Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE); + if (Pt == NULL) + { + KEBUGCHECK(0); + } + + do + { + Pte = *Pt; + tmpPte = Pte | PA_PRESENT; + } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte)); + if (!(Pte & PA_PRESENT)) + { + MiFlushTlb((PULONG)Pt, Address); + } + else + { + MmUnmapPageTable((PULONG)Pt); + } + } + else + { + PULONG Pt; + ULONG Pte; + + Pt = MmGetPageTableForProcess(Process, Address, FALSE); + if (Pt == NULL) + { + KEBUGCHECK(0); + } + + do + { + Pte = *Pt; + } while (Pte != InterlockedCompareExchangeUL(Pt, Pte | PA_PRESENT, Pte)); + if (!(Pte & PA_PRESENT)) + { + MiFlushTlb(Pt, Address); + } + else + { + MmUnmapPageTable(Pt); + } + } +} + +BOOLEAN +NTAPI +MmIsPagePresent(PEPROCESS Process, PVOID Address) +{ + if (Ke386Pae) + { + return MmGetPageEntryForProcessForPAE(Process, Address) & PA_PRESENT ? TRUE : FALSE; + } + else + { + return MmGetPageEntryForProcess(Process, Address) & PA_PRESENT ? TRUE : FALSE; + } +} + +BOOLEAN +NTAPI +MmIsPageSwapEntry(PEPROCESS Process, PVOID Address) +{ + if (Ke386Pae) + { + ULONGLONG Entry; + Entry = MmGetPageEntryForProcessForPAE(Process, Address); + return !(Entry & PA_PRESENT) && Entry != 0 ? TRUE : FALSE; + } + else + { + ULONG Entry; + Entry = MmGetPageEntryForProcess(Process, Address); + return !(Entry & PA_PRESENT) && Entry != 0 ? TRUE : FALSE; + } +} + +NTSTATUS +NTAPI +MmCreateVirtualMappingForKernel(PVOID Address, + ULONG flProtect, + PPFN_TYPE Pages, + ULONG PageCount) +{ + ULONG Attributes; + ULONG i; + PVOID Addr; + ULONG PdeOffset, oldPdeOffset; + BOOLEAN NoExecute = FALSE; + + DPRINT("MmCreateVirtualMappingForKernel(%x, %x, %x, %d)\n", + Address, flProtect, Pages, PageCount); + + if (Address < MmSystemRangeStart) + { + DPRINT1("MmCreateVirtualMappingForKernel is called for user space\n"); + KEBUGCHECK(0); + } + + Attributes = ProtectToPTE(flProtect); + if (Attributes & 0x80000000) + { + NoExecute = TRUE; + } + Attributes &= 0xfff; + if (Ke386GlobalPagesEnabled) + { + Attributes |= PA_GLOBAL; + } + + Addr = Address; + + if (Ke386Pae) + { + PULONGLONG Pt = NULL; + ULONGLONG Pte; + + oldPdeOffset = PAE_ADDR_TO_PDE_OFFSET(Addr) + 1; + for (i = 0; i < PageCount; i++, Addr = (PVOID)((ULONG_PTR)Addr + PAGE_SIZE)) + { + if (!(Attributes & PA_PRESENT) && Pages[i] != 0) + { + DPRINT1("Setting physical address but not allowing access at address " + "0x%.8X with attributes %x/%x.\n", + Addr, Attributes, flProtect); + KEBUGCHECK(0); + } + + PdeOffset = PAE_ADDR_TO_PDE_OFFSET(Addr); + if (oldPdeOffset != PdeOffset) + { + Pt = MmGetPageTableForProcessForPAE(NULL, Addr, TRUE); + if (Pt == NULL) + { + KEBUGCHECK(0); + } + } + else + { + Pt++; + } + oldPdeOffset = PdeOffset; + + Pte = PFN_TO_PTE(Pages[i]) | Attributes; + if (NoExecute) + { + Pte |= 0x8000000000000000LL; + } + Pte = ExfpInterlockedExchange64UL(Pt, &Pte); + if (Pte != 0LL) + { + KEBUGCHECK(0); + } + } + } + else + { + PULONG Pt; + ULONG Pte; + + oldPdeOffset = ADDR_TO_PDE_OFFSET(Addr); + Pt = MmGetPageTableForProcess(NULL, Addr, TRUE); + if (Pt == NULL) + { + KEBUGCHECK(0); + } + Pt--; + + for (i = 0; i < PageCount; i++, Addr = (PVOID)((ULONG_PTR)Addr + PAGE_SIZE)) + { + if (!(Attributes & PA_PRESENT) && Pages[i] != 0) + { + DPRINT1("Setting physical address but not allowing access at address " + "0x%.8X with attributes %x/%x.\n", + Addr, Attributes, flProtect); + KEBUGCHECK(0); + } + + PdeOffset = ADDR_TO_PDE_OFFSET(Addr); + if (oldPdeOffset != PdeOffset) + { + Pt = MmGetPageTableForProcess(NULL, Addr, TRUE); + if (Pt == NULL) + { + KEBUGCHECK(0); + } + } + else + { + Pt++; + } + oldPdeOffset = PdeOffset; + + Pte = *Pt; + if (Pte != 0) + { + KEBUGCHECK(0); + } + (void)InterlockedExchangeUL(Pt, PFN_TO_PTE(Pages[i]) | Attributes); + } + } + + return(STATUS_SUCCESS); +} + +NTSTATUS +NTAPI +MmCreatePageFileMapping(PEPROCESS Process, + PVOID Address, + SWAPENTRY SwapEntry) +{ + if (Process == NULL && Address < MmSystemRangeStart) + { + DPRINT1("No process\n"); + KEBUGCHECK(0); + } + if (Process != NULL && Address >= MmSystemRangeStart) + { + DPRINT1("Setting kernel address with process context\n"); + KEBUGCHECK(0); + } + if (SwapEntry & (1 << 31)) + { + KEBUGCHECK(0); + } + + if (Ke386Pae) + { + PULONGLONG Pt; + ULONGLONG Pte; + ULONGLONG tmpPte; + + Pt = MmGetPageTableForProcessForPAE(Process, Address, TRUE); + if (Pt == NULL) + { + KEBUGCHECK(0); + } + tmpPte = SwapEntry << 1; + Pte = ExfpInterlockedExchange64UL(Pt, &tmpPte); + if (PAE_PAGE_MASK((Pte)) != 0) + { + MmMarkPageUnmapped(PAE_PTE_TO_PFN((Pte))); + } + + if (Pte != 0) + { + MiFlushTlb((PULONG)Pt, Address); + } + else + { + MmUnmapPageTable((PULONG)Pt); + } + } + else + { + PULONG Pt; + ULONG Pte; + + Pt = MmGetPageTableForProcess(Process, Address, TRUE); + if (Pt == NULL) + { + KEBUGCHECK(0); + } + Pte = *Pt; + if (PAGE_MASK((Pte)) != 0) + { + MmMarkPageUnmapped(PTE_TO_PFN((Pte))); + } + (void)InterlockedExchangeUL(Pt, SwapEntry << 1); + if (Pte != 0) + { + MiFlushTlb(Pt, Address); + } + else + { + MmUnmapPageTable(Pt); + } + } + if (Process != NULL && + ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable != NULL && + Address < MmSystemRangeStart) + { + PUSHORT Ptrc; + ULONG Idx; + + Ptrc = ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable; + Idx = Ke386Pae ? PAE_ADDR_TO_PAGE_TABLE(Address) : ADDR_TO_PAGE_TABLE(Address); + Ptrc[Idx]++; + } + return(STATUS_SUCCESS); +} + + +NTSTATUS +NTAPI +MmCreateVirtualMappingUnsafe(PEPROCESS Process, + PVOID Address, + ULONG flProtect, + PPFN_TYPE Pages, + ULONG PageCount) +{ + ULONG Attributes; + PVOID Addr; + ULONG i; + ULONG oldPdeOffset, PdeOffset; + BOOLEAN NoExecute = FALSE; + + DPRINT("MmCreateVirtualMappingUnsafe(%x, %x, %x, %x (%x), %d)\n", + Process, Address, flProtect, Pages, *Pages, PageCount); + + if (Process == NULL) + { + if (Address < MmSystemRangeStart) + { + DPRINT1("No process\n"); + KEBUGCHECK(0); + } + if (PageCount > 0x10000 || + (ULONG_PTR) Address / PAGE_SIZE + PageCount > 0x100000) + { + DPRINT1("Page count to large\n"); + KEBUGCHECK(0); + } + } + else + { + if (Address >= MmSystemRangeStart) + { + DPRINT1("Setting kernel address with process context\n"); + KEBUGCHECK(0); + } + if (PageCount > (ULONG_PTR)MmSystemRangeStart / PAGE_SIZE || + (ULONG_PTR) Address / PAGE_SIZE + PageCount > + (ULONG_PTR)MmSystemRangeStart / PAGE_SIZE) + { + DPRINT1("Page Count to large\n"); + KEBUGCHECK(0); + } + } + + Attributes = ProtectToPTE(flProtect); + if (Attributes & 0x80000000) + { + NoExecute = TRUE; + } + Attributes &= 0xfff; + if (Address >= MmSystemRangeStart) + { + Attributes &= ~PA_USER; + if (Ke386GlobalPagesEnabled) + { + Attributes |= PA_GLOBAL; + } + } + else + { + Attributes |= PA_USER; + } + + Addr = Address; + + if (Ke386Pae) + { + ULONGLONG Pte, tmpPte; + PULONGLONG Pt = NULL; + + oldPdeOffset = PAE_ADDR_TO_PDE_OFFSET(Addr) + 1; + for (i = 0; i < PageCount; i++, Addr = (PVOID)((ULONG_PTR)Addr + PAGE_SIZE)) + { + if (!(Attributes & PA_PRESENT) && Pages[i] != 0) + { + DPRINT1("Setting physical address but not allowing access at address " + "0x%.8X with attributes %x/%x.\n", + Addr, Attributes, flProtect); + KEBUGCHECK(0); + } + PdeOffset = PAE_ADDR_TO_PDE_OFFSET(Addr); + if (oldPdeOffset != PdeOffset) + { + MmUnmapPageTable((PULONG)Pt); + Pt = MmGetPageTableForProcessForPAE(Process, Addr, TRUE); + if (Pt == NULL) + { + KEBUGCHECK(0); + } + } + else + { + Pt++; + } + oldPdeOffset = PdeOffset; + + MmMarkPageMapped(Pages[i]); + tmpPte = PAE_PFN_TO_PTE(Pages[i]) | Attributes; + if (NoExecute) + { + tmpPte |= 0x8000000000000000LL; + } + Pte = ExfpInterlockedExchange64UL(Pt, &tmpPte); + if (PAE_PAGE_MASK((Pte)) != 0LL && !((Pte) & PA_PRESENT)) + { + KEBUGCHECK(0); + } + if (PAE_PAGE_MASK((Pte)) != 0LL) + { + MmMarkPageUnmapped(PAE_PTE_TO_PFN((Pte))); + } + if (Address < MmSystemRangeStart && + ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable != NULL && + Attributes & PA_PRESENT) + { + PUSHORT Ptrc; + + Ptrc = ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable; + + Ptrc[PAE_ADDR_TO_PAGE_TABLE(Addr)]++; + } + if (Pte != 0LL) + { + if (Address > MmSystemRangeStart || + (Pt >= (PULONGLONG)PAGETABLE_MAP && Pt < (PULONGLONG)PAGETABLE_MAP + 4*512*512)) + { + MiFlushTlb((PULONG)Pt, Address); + } + } + } + if (Addr > Address) + { + MmUnmapPageTable((PULONG)Pt); + } + } + else + { + PULONG Pt = NULL; + ULONG Pte; + oldPdeOffset = ADDR_TO_PDE_OFFSET(Addr) + 1; + for (i = 0; i < PageCount; i++, Addr = (PVOID)((ULONG_PTR)Addr + PAGE_SIZE)) + { + if (!(Attributes & PA_PRESENT) && Pages[i] != 0) + { + DPRINT1("Setting physical address but not allowing access at address " + "0x%.8X with attributes %x/%x.\n", + Addr, Attributes, flProtect); + KEBUGCHECK(0); + } + PdeOffset = ADDR_TO_PDE_OFFSET(Addr); + if (oldPdeOffset != PdeOffset) + { + MmUnmapPageTable(Pt); + Pt = MmGetPageTableForProcess(Process, Addr, TRUE); + if (Pt == NULL) + { + KEBUGCHECK(0); + } + } + else + { + Pt++; + } + oldPdeOffset = PdeOffset; + + Pte = *Pt; + MmMarkPageMapped(Pages[i]); + if (PAGE_MASK((Pte)) != 0 && !((Pte) & PA_PRESENT)) + { + KEBUGCHECK(0); + } + if (PAGE_MASK((Pte)) != 0) + { + MmMarkPageUnmapped(PTE_TO_PFN((Pte))); + } + (void)InterlockedExchangeUL(Pt, PFN_TO_PTE(Pages[i]) | Attributes); + if (Address < MmSystemRangeStart && + ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable != NULL && + Attributes & PA_PRESENT) + { + PUSHORT Ptrc; + + Ptrc = ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable; + + Ptrc[ADDR_TO_PAGE_TABLE(Addr)]++; + } + if (Pte != 0) + { + if (Address > MmSystemRangeStart || + (Pt >= (PULONG)PAGETABLE_MAP && Pt < (PULONG)PAGETABLE_MAP + 1024*1024)) + { + MiFlushTlb(Pt, Address); + } + } + } + if (Addr > Address) + { + MmUnmapPageTable(Pt); + } + } + return(STATUS_SUCCESS); +} + +NTSTATUS +NTAPI +MmCreateVirtualMapping(PEPROCESS Process, + PVOID Address, + ULONG flProtect, + PPFN_TYPE Pages, + ULONG PageCount) +{ + ULONG i; + + for (i = 0; i < PageCount; i++) + { + if (!MmIsPageInUse(Pages[i])) + { + DPRINT1("Page at address %x not in use\n", PFN_TO_PTE(Pages[i])); + KEBUGCHECK(0); + } + } + + return(MmCreateVirtualMappingUnsafe(Process, + Address, + flProtect, + Pages, + PageCount)); +} + +ULONG +NTAPI +MmGetPageProtect(PEPROCESS Process, PVOID Address) +{ + ULONG Entry; + ULONG Protect; + if (Ke386Pae) + { + Entry = MmGetPageEntryForProcessForPAE(Process, Address); + } + else + { + Entry = MmGetPageEntryForProcess(Process, Address); + } + + if (!(Entry & PA_PRESENT)) + { + Protect = PAGE_NOACCESS; + } + else + { + if (Entry & PA_READWRITE) + { + Protect = PAGE_READWRITE; + } + else + { + Protect = PAGE_EXECUTE_READ; + } + if (Entry & PA_CD) + { + Protect |= PAGE_NOCACHE; + } + if (Entry & PA_WT) + { + Protect |= PAGE_WRITETHROUGH; + } + if (!(Entry & PA_USER)) + { + Protect |= PAGE_SYSTEM; + } + + } + return(Protect); +} + +VOID +NTAPI +MmSetPageProtect(PEPROCESS Process, PVOID Address, ULONG flProtect) +{ + ULONG Attributes = 0; + BOOLEAN NoExecute = FALSE; + + DPRINT("MmSetPageProtect(Process %x Address %x flProtect %x)\n", + Process, Address, flProtect); + + Attributes = ProtectToPTE(flProtect); + if (Attributes & 0x80000000) + { + NoExecute = TRUE; + } + Attributes &= 0xfff; + if (Address >= MmSystemRangeStart) + { + Attributes &= ~PA_USER; + if (Ke386GlobalPagesEnabled) + { + Attributes |= PA_GLOBAL; + } + } + else + { + Attributes |= PA_USER; + } + if (Ke386Pae) + { + PULONGLONG Pt; + ULONGLONG tmpPte, Pte; + + Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE); + if (Pt == NULL) + { + DPRINT1("Address %x\n", Address); + KEBUGCHECK(0); + } + do + { + Pte = *Pt; + tmpPte = PAE_PAGE_MASK(Pte) | Attributes | (Pte & (PA_ACCESSED|PA_DIRTY)); + if (NoExecute) + { + tmpPte |= 0x8000000000000000LL; + } + else + { + tmpPte &= ~0x8000000000000000LL; + } + } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte)); + + MiFlushTlb((PULONG)Pt, Address); + } + else + { + PULONG Pt; + + Pt = MmGetPageTableForProcess(Process, Address, FALSE); + if (Pt == NULL) + { + KEBUGCHECK(0); + } + InterlockedExchange((PLONG)Pt, PAGE_MASK(*Pt) | Attributes | (*Pt & (PA_ACCESSED|PA_DIRTY))); + MiFlushTlb(Pt, Address); + } +} + +/* + * @implemented + */ +PHYSICAL_ADDRESS STDCALL +MmGetPhysicalAddress(PVOID vaddr) +/* + * FUNCTION: Returns the physical address corresponding to a virtual address + */ +{ + PHYSICAL_ADDRESS p; + + DPRINT("MmGetPhysicalAddress(vaddr %x)\n", vaddr); + if (Ke386Pae) + { + ULONGLONG Pte; + Pte = MmGetPageEntryForProcessForPAE(NULL, vaddr); + if (Pte != 0 && Pte & PA_PRESENT) + { + p.QuadPart = PAE_PAGE_MASK(Pte); + p.u.LowPart |= (ULONG_PTR)vaddr & (PAGE_SIZE - 1); + } + else + { + p.QuadPart = 0; + } + } + else + { + ULONG Pte; + Pte = MmGetPageEntryForProcess(NULL, vaddr); + if (Pte != 0 && Pte & PA_PRESENT) + { + p.QuadPart = PAGE_MASK(Pte); + p.u.LowPart |= (ULONG_PTR)vaddr & (PAGE_SIZE - 1); + } + else + { + p.QuadPart = 0; + } + } + return p; +} + +PVOID +NTAPI +MmCreateHyperspaceMapping(PFN_TYPE Page) +{ + PVOID Address; + ULONG i; + + if (Ke386Pae) + { + ULONGLONG Entry; + ULONGLONG ZeroEntry = 0LL; + PULONGLONG Pte; + + Entry = PFN_TO_PTE(Page) | PA_PRESENT | PA_READWRITE; + Pte = PAE_ADDR_TO_PTE(HYPERSPACE) + Page % 1024; + + if (Page & 1024) + { + for (i = Page %1024; i < 1024; i++, Pte++) + { + if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry)) + { + break; + } + } + if (i >= 1024) + { + Pte = PAE_ADDR_TO_PTE(HYPERSPACE); + for (i = 0; i < Page % 1024; i++, Pte++) + { + if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry)) + { + break; + } + } + if (i >= Page % 1024) + { + KEBUGCHECK(0); + } + } + } + else + { + for (i = Page %1024; (LONG)i >= 0; i--, Pte--) + { + if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry)) + { + break; + } + } + if ((LONG)i < 0) + { + Pte = PAE_ADDR_TO_PTE(HYPERSPACE) + 1023; + for (i = 1023; i > Page % 1024; i--, Pte--) + { + if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry)) + { + break; + } + } + if (i <= Page % 1024) + { + KEBUGCHECK(0); + } + } + } + } + else + { + ULONG Entry; + PULONG Pte; + Entry = PFN_TO_PTE(Page) | PA_PRESENT | PA_READWRITE; + Pte = ADDR_TO_PTE(HYPERSPACE) + Page % 1024; + if (Page & 1024) + { + for (i = Page % 1024; i < 1024; i++, Pte++) + { + if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0)) + { + break; + } + } + if (i >= 1024) + { + Pte = ADDR_TO_PTE(HYPERSPACE); + for (i = 0; i < Page % 1024; i++, Pte++) + { + if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0)) + { + break; + } + } + if (i >= Page % 1024) + { + KEBUGCHECK(0); + } + } + } + else + { + for (i = Page % 1024; (LONG)i >= 0; i--, Pte--) + { + if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0)) + { + break; + } + } + if ((LONG)i < 0) + { + Pte = ADDR_TO_PTE(HYPERSPACE) + 1023; + for (i = 1023; i > Page % 1024; i--, Pte--) + { + if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0)) + { + break; + } + } + if (i <= Page % 1024) + { + KEBUGCHECK(0); + } + } + } + } + Address = (PVOID)((ULONG_PTR)HYPERSPACE + i * PAGE_SIZE); + __invlpg(Address); + return Address; +} + +PFN_TYPE +NTAPI +MmChangeHyperspaceMapping(PVOID Address, PFN_TYPE NewPage) +{ + PFN_TYPE Pfn; + ASSERT (IS_HYPERSPACE(Address)); + if (Ke386Pae) + { + ULONGLONG Entry = PAE_PFN_TO_PTE(NewPage) | PA_PRESENT | PA_READWRITE; + Entry = (ULONG)ExfpInterlockedExchange64UL(PAE_ADDR_TO_PTE(Address), &Entry); + Pfn = PAE_PTE_TO_PFN(Entry); + } + else + { + ULONG Entry; + Entry = InterlockedExchange((PLONG)ADDR_TO_PTE(Address), PFN_TO_PTE(NewPage) | PA_PRESENT | PA_READWRITE); + Pfn = PTE_TO_PFN(Entry); + } + __invlpg(Address); + return Pfn; +} + +PFN_TYPE +NTAPI +MmDeleteHyperspaceMapping(PVOID Address) +{ + PFN_TYPE Pfn; + ASSERT (IS_HYPERSPACE(Address)); + if (Ke386Pae) + { + ULONGLONG Entry = 0LL; + Entry = (ULONG)ExfpInterlockedExchange64UL(PAE_ADDR_TO_PTE(Address), &Entry); + Pfn = PAE_PTE_TO_PFN(Entry); + } + else + { + ULONG Entry; + Entry = InterlockedExchange((PLONG)ADDR_TO_PTE(Address), 0); + Pfn = PTE_TO_PFN(Entry); + } + __invlpg(Address); + return Pfn; +} + +VOID +NTAPI +MmUpdatePageDir(PEPROCESS Process, PVOID Address, ULONG Size) +{ + ULONG StartOffset, EndOffset, Offset; + + if (Address < MmSystemRangeStart) + { + KEBUGCHECK(0); + } + if (Ke386Pae) + { + PULONGLONG PageDirTable; + PULONGLONG Pde; + ULONGLONG ZeroPde = 0LL; + ULONG i; + + for (i = PAE_ADDR_TO_PDTE_OFFSET(Address); i <= PAE_ADDR_TO_PDTE_OFFSET((PVOID)((ULONG_PTR)Address + Size)); i++) + { + if (i == PAE_ADDR_TO_PDTE_OFFSET(Address)) + { + StartOffset = PAE_ADDR_TO_PDE_PAGE_OFFSET(Address); + } + else + { + StartOffset = 0; + } + if (i == PAE_ADDR_TO_PDTE_OFFSET((PVOID)((ULONG_PTR)Address + Size))) + { + EndOffset = PAE_ADDR_TO_PDE_PAGE_OFFSET((PVOID)((ULONG_PTR)Address + Size)); + } + else + { + EndOffset = 511; + } + + if (Process != NULL && Process != PsGetCurrentProcess()) + { + PageDirTable = MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart)); + Pde = (PULONGLONG)MmCreateHyperspaceMapping(PTE_TO_PFN(PageDirTable[i])); + MmDeleteHyperspaceMapping(PageDirTable); + } + else + { + Pde = (PULONGLONG)PAE_PAGEDIRECTORY_MAP + i*512; + } + + for (Offset = StartOffset; Offset <= EndOffset; Offset++) + { + if (i * 512 + Offset < PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) || i * 512 + Offset >= PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP)+4) + { + (void)ExfInterlockedCompareExchange64UL(&Pde[Offset], &MmGlobalKernelPageDirectoryForPAE[i*512 + Offset], &ZeroPde); + } + } + MmUnmapPageTable((PULONG)Pde); + } + } + else + { + PULONG Pde; + StartOffset = ADDR_TO_PDE_OFFSET(Address); + EndOffset = ADDR_TO_PDE_OFFSET((PVOID)((ULONG_PTR)Address + Size)); + + if (Process != NULL && Process != PsGetCurrentProcess()) + { + Pde = MmCreateHyperspaceMapping(PTE_TO_PFN(Process->Pcb.DirectoryTableBase.u.LowPart)); + } + else + { + Pde = (PULONG)PAGEDIRECTORY_MAP; + } + for (Offset = StartOffset; Offset <= EndOffset; Offset++) + { + if (Offset != ADDR_TO_PDE_OFFSET(PAGETABLE_MAP)) + { + (void)InterlockedCompareExchangeUL(&Pde[Offset], MmGlobalKernelPageDirectory[Offset], 0); + } + } + if (Pde != (PULONG)PAGEDIRECTORY_MAP) + { + MmDeleteHyperspaceMapping(Pde); + } + } +} + +VOID +INIT_FUNCTION +NTAPI +MmInitGlobalKernelPageDirectory(VOID) +{ + ULONG i; + + DPRINT("MmInitGlobalKernelPageDirectory()\n"); + + if (Ke386Pae) + { + PULONGLONG CurrentPageDirectory = (PULONGLONG)PAE_PAGEDIRECTORY_MAP; + for (i = PAE_ADDR_TO_PDE_OFFSET(MmSystemRangeStart); i < 4 * 512; i++) + { + if (!(i >= PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) && i < PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) + 4) && + !(i >= PAE_ADDR_TO_PDE_OFFSET(HYPERSPACE) && i < PAE_ADDR_TO_PDE_OFFSET(HYPERSPACE) + 2) && + 0LL == MmGlobalKernelPageDirectoryForPAE[i] && 0LL != CurrentPageDirectory[i]) + { + (void)ExfpInterlockedExchange64UL(&MmGlobalKernelPageDirectoryForPAE[i], &CurrentPageDirectory[i]); + if (Ke386GlobalPagesEnabled) + { + MmGlobalKernelPageDirectoryForPAE[i] |= PA_GLOBAL; + CurrentPageDirectory[i] |= PA_GLOBAL; + } + } + } + } + else + { + PULONG CurrentPageDirectory = (PULONG)PAGEDIRECTORY_MAP; + for (i = ADDR_TO_PDE_OFFSET(MmSystemRangeStart); i < 1024; i++) + { + if (i != ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) && + i != ADDR_TO_PDE_OFFSET(HYPERSPACE) && + 0 == MmGlobalKernelPageDirectory[i] && 0 != CurrentPageDirectory[i]) + { + MmGlobalKernelPageDirectory[i] = CurrentPageDirectory[i]; + if (Ke386GlobalPagesEnabled) + { + MmGlobalKernelPageDirectory[i] |= PA_GLOBAL; + CurrentPageDirectory[i] |= PA_GLOBAL; + } + } + } + } +} + +ULONG +NTAPI +MiGetUserPageDirectoryCount(VOID) +{ + return Ke386Pae ? PAE_ADDR_TO_PDE_OFFSET(MmSystemRangeStart) : ADDR_TO_PDE_OFFSET(MmSystemRangeStart); +} + +VOID +INIT_FUNCTION +NTAPI +MiInitPageDirectoryMap(VOID) +{ + MEMORY_AREA* kernel_map_desc = NULL; + MEMORY_AREA* hyperspace_desc = NULL; + PHYSICAL_ADDRESS BoundaryAddressMultiple; + PVOID BaseAddress; + NTSTATUS Status; + + DPRINT("MiInitPageDirectoryMap()\n"); + + BoundaryAddressMultiple.QuadPart = 0; + BaseAddress = (PVOID)PAGETABLE_MAP; + Status = MmCreateMemoryArea(MmGetKernelAddressSpace(), + MEMORY_AREA_SYSTEM, + &BaseAddress, + Ke386Pae ? 0x800000 : 0x400000, + PAGE_READWRITE, + &kernel_map_desc, + TRUE, + 0, + BoundaryAddressMultiple); + if (!NT_SUCCESS(Status)) + { + KEBUGCHECK(0); + } + BaseAddress = (PVOID)HYPERSPACE; + Status = MmCreateMemoryArea(MmGetKernelAddressSpace(), + MEMORY_AREA_SYSTEM, + &BaseAddress, + 0x400000, + PAGE_READWRITE, + &hyperspace_desc, + TRUE, + 0, + BoundaryAddressMultiple); + if (!NT_SUCCESS(Status)) + { + KEBUGCHECK(0); + } +} + +/* EOF */