From 9b74e5ed4ff49d162b8416d5e365857754299538 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Fri, 23 Oct 2020 11:30:42 +0200 Subject: [PATCH 01/99] [NTOSKRNL] Do not compile unused code --- ntoskrnl/cache/section/data.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ntoskrnl/cache/section/data.c b/ntoskrnl/cache/section/data.c index 1fb0fc20c56..b7649dd1f62 100644 --- a/ntoskrnl/cache/section/data.c +++ b/ntoskrnl/cache/section/data.c @@ -303,6 +303,7 @@ MmFinalizeSegment(PMM_SECTION_SEGMENT Segment) ExFreePoolWithTag(Segment, TAG_MM_SECTION_SEGMENT); } +#ifdef NEWCC NTSTATUS NTAPI MmCreateCacheSection(PROS_SECTION_OBJECT *SectionObject, @@ -533,6 +534,7 @@ MmCreateCacheSection(PROS_SECTION_OBJECT *SectionObject, *SectionObject = Section; return STATUS_SUCCESS; } +#endif NTSTATUS NTAPI From 073ec9b0d4595b997d1464e59a803e539b348a3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Fri, 23 Oct 2020 11:38:20 +0200 Subject: [PATCH 02/99] [NTOSKRNL][NDK] Get rid of SECTION_OBJECT typedef We will use the more recent SECTION (already used in ARM3) and this will avoid confusions --- ntoskrnl/mm/ARM3/section.c | 4 ++-- ntoskrnl/ps/process.c | 2 +- sdk/include/ndk/mmtypes.h | 13 ------------- 3 files changed, 3 insertions(+), 16 deletions(-) diff --git a/ntoskrnl/mm/ARM3/section.c b/ntoskrnl/mm/ARM3/section.c index 02ead9a70a5..fe7e8b46145 100644 --- a/ntoskrnl/mm/ARM3/section.c +++ b/ntoskrnl/mm/ARM3/section.c @@ -1700,7 +1700,7 @@ PFILE_OBJECT NTAPI MmGetFileObjectForSection(IN PVOID SectionObject) { - PSECTION_OBJECT Section; + PSECTION Section; ASSERT(KeGetCurrentIrql() == PASSIVE_LEVEL); ASSERT(SectionObject != NULL); @@ -1773,7 +1773,7 @@ VOID NTAPI MmGetImageInformation (OUT PSECTION_IMAGE_INFORMATION ImageInformation) { - PSECTION_OBJECT SectionObject; + PSECTION SectionObject; /* Get the section object of this process*/ SectionObject = PsGetCurrentProcess()->SectionObject; diff --git a/ntoskrnl/ps/process.c b/ntoskrnl/ps/process.c index 179fd1d8fa4..0e6f112a53b 100644 --- a/ntoskrnl/ps/process.c +++ b/ntoskrnl/ps/process.c @@ -358,7 +358,7 @@ PspCreateProcess(OUT PHANDLE ProcessHandle, PEPROCESS Process, Parent; PVOID ExceptionPortObject; PDEBUG_OBJECT DebugObject; - PSECTION_OBJECT SectionObject; + PSECTION SectionObject; NTSTATUS Status, AccessStatus; ULONG_PTR DirectoryTableBase[2] = {0,0}; KAFFINITY Affinity; diff --git a/sdk/include/ndk/mmtypes.h b/sdk/include/ndk/mmtypes.h index ca62ebdf5ce..3e9fabf1c02 100644 --- a/sdk/include/ndk/mmtypes.h +++ b/sdk/include/ndk/mmtypes.h @@ -623,19 +623,6 @@ typedef struct _SEGMENT_OBJECT PMMSUBSECTION_FLAGS MmSubSectionFlags; } SEGMENT_OBJECT, *PSEGMENT_OBJECT; -// -// Section Object -// -typedef struct _SECTION_OBJECT -{ - PVOID StartingVa; - PVOID EndingVa; - PVOID Parent; - PVOID LeftChild; - PVOID RightChild; - PSEGMENT_OBJECT Segment; -} SECTION_OBJECT, *PSECTION_OBJECT; - // // Generic Address Range Structure // From 54c760574b2ac55006c0e233fd1ee7857a5a9a42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Fri, 23 Oct 2020 11:42:09 +0200 Subject: [PATCH 03/99] [NTOSKRNL] Rename MM_ROS_SECTION_OBJECT:MaximumSize to SizeOfSection For consistency with the SECTION struct --- ntoskrnl/include/internal/mm.h | 2 +- ntoskrnl/mm/section.c | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ntoskrnl/include/internal/mm.h b/ntoskrnl/include/internal/mm.h index d1754ec4f5c..e803142d651 100644 --- a/ntoskrnl/include/internal/mm.h +++ b/ntoskrnl/include/internal/mm.h @@ -199,7 +199,7 @@ typedef struct _ROS_SECTION_OBJECT { CSHORT Type; CSHORT Size; - LARGE_INTEGER MaximumSize; + LARGE_INTEGER SizeOfSection; ULONG SectionPageProtection; ULONG AllocationAttributes; PFILE_OBJECT FileObject; diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index f586ec37754..13835bb94ac 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -2881,7 +2881,7 @@ MmCreatePageFileSection(PROS_SECTION_OBJECT *SectionObject, Section->Size = 'TN'; Section->SectionPageProtection = SectionPageProtection; Section->AllocationAttributes = AllocationAttributes; - Section->MaximumSize = MaximumSize; + Section->SizeOfSection = MaximumSize; Segment = ExAllocatePoolWithTag(NonPagedPool, sizeof(MM_SECTION_SEGMENT), TAG_MM_SECTION_SEGMENT); if (Segment == NULL) @@ -3087,7 +3087,7 @@ MmCreateDataFileSection(PROS_SECTION_OBJECT *SectionObject, } MmUnlockSectionSegment(Segment); Section->FileObject = FileObject; - Section->MaximumSize = MaximumSize; + Section->SizeOfSection = MaximumSize; #ifndef NEWCC CcRosReferenceCache(FileObject); #endif @@ -4678,11 +4678,11 @@ MmMapViewOfSection(IN PVOID SectionObject, if ((*ViewSize) == 0) { - (*ViewSize) = Section->MaximumSize.u.LowPart - ViewOffset; + (*ViewSize) = Section->SizeOfSection.u.LowPart - ViewOffset; } - else if (((*ViewSize)+ViewOffset) > Section->MaximumSize.u.LowPart) + else if (((*ViewSize)+ViewOffset) > Section->SizeOfSection.u.LowPart) { - (*ViewSize) = Section->MaximumSize.u.LowPart - ViewOffset; + (*ViewSize) = Section->SizeOfSection.u.LowPart - ViewOffset; } *ViewSize = PAGE_ROUND_UP(*ViewSize); @@ -4869,11 +4869,11 @@ MmMapViewInSystemSpace (IN PVOID SectionObject, if ((*ViewSize) == 0) { - (*ViewSize) = Section->MaximumSize.u.LowPart; + (*ViewSize) = Section->SizeOfSection.u.LowPart; } - else if ((*ViewSize) > Section->MaximumSize.u.LowPart) + else if ((*ViewSize) > Section->SizeOfSection.u.LowPart) { - (*ViewSize) = Section->MaximumSize.u.LowPart; + (*ViewSize) = Section->SizeOfSection.u.LowPart; } MmLockSectionSegment(Section->Segment); @@ -5132,7 +5132,7 @@ MmCreateSection (OUT PVOID * Section, Status = MmCreateCacheSection(SectionObject, DesiredAccess, ObjectAttributes, - MaximumSize, + SizeOfSection, SectionPageProtection, AllocationAttributes, FileObject); From 7afecee5ebc39c1446be320f057fd47c63bdbf5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Fri, 23 Oct 2020 11:46:46 +0200 Subject: [PATCH 04/99] [NTOSKRNL] Rename MM_ROS_SECTION_OBJECT:SectionPageProtection to InitialPageProtection --- ntoskrnl/include/internal/mm.h | 2 +- ntoskrnl/mm/section.c | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ntoskrnl/include/internal/mm.h b/ntoskrnl/include/internal/mm.h index e803142d651..160c19a8a76 100644 --- a/ntoskrnl/include/internal/mm.h +++ b/ntoskrnl/include/internal/mm.h @@ -200,7 +200,7 @@ typedef struct _ROS_SECTION_OBJECT CSHORT Type; CSHORT Size; LARGE_INTEGER SizeOfSection; - ULONG SectionPageProtection; + ULONG InitialPageProtection; ULONG AllocationAttributes; PFILE_OBJECT FileObject; union diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index 13835bb94ac..9704ddba8f6 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -2879,7 +2879,7 @@ MmCreatePageFileSection(PROS_SECTION_OBJECT *SectionObject, RtlZeroMemory(Section, sizeof(ROS_SECTION_OBJECT)); Section->Type = 'SC'; Section->Size = 'TN'; - Section->SectionPageProtection = SectionPageProtection; + Section->InitialPageProtection = SectionPageProtection; Section->AllocationAttributes = AllocationAttributes; Section->SizeOfSection = MaximumSize; Segment = ExAllocatePoolWithTag(NonPagedPool, sizeof(MM_SECTION_SEGMENT), @@ -2949,7 +2949,7 @@ MmCreateDataFileSection(PROS_SECTION_OBJECT *SectionObject, RtlZeroMemory(Section, sizeof(ROS_SECTION_OBJECT)); Section->Type = 'SC'; Section->Size = 'TN'; - Section->SectionPageProtection = SectionPageProtection; + Section->InitialPageProtection = SectionPageProtection; Section->AllocationAttributes = AllocationAttributes; /* @@ -3774,7 +3774,7 @@ MmCreateImageSection(PROS_SECTION_OBJECT *SectionObject, RtlZeroMemory(Section, sizeof(ROS_SECTION_OBJECT)); Section->Type = 'SC'; Section->Size = 'TN'; - Section->SectionPageProtection = SectionPageProtection; + Section->InitialPageProtection = SectionPageProtection; Section->AllocationAttributes = AllocationAttributes; if (FileObject->SectionObjectPointer->ImageSectionObject == NULL) @@ -4641,21 +4641,21 @@ MmMapViewOfSection(IN PVOID SectionObject, { /* check for write access */ if ((Protect & (PAGE_READWRITE|PAGE_EXECUTE_READWRITE)) && - !(Section->SectionPageProtection & (PAGE_READWRITE|PAGE_EXECUTE_READWRITE))) + !(Section->InitialPageProtection & (PAGE_READWRITE|PAGE_EXECUTE_READWRITE))) { MmUnlockAddressSpace(AddressSpace); return STATUS_SECTION_PROTECTION; } /* check for read access */ if ((Protect & (PAGE_READONLY|PAGE_WRITECOPY|PAGE_EXECUTE_READ|PAGE_EXECUTE_WRITECOPY)) && - !(Section->SectionPageProtection & (PAGE_READONLY|PAGE_READWRITE|PAGE_WRITECOPY|PAGE_EXECUTE_READ|PAGE_EXECUTE_READWRITE|PAGE_EXECUTE_WRITECOPY))) + !(Section->InitialPageProtection & (PAGE_READONLY|PAGE_READWRITE|PAGE_WRITECOPY|PAGE_EXECUTE_READ|PAGE_EXECUTE_READWRITE|PAGE_EXECUTE_WRITECOPY))) { MmUnlockAddressSpace(AddressSpace); return STATUS_SECTION_PROTECTION; } /* check for execute access */ if ((Protect & (PAGE_EXECUTE|PAGE_EXECUTE_READ|PAGE_EXECUTE_READWRITE|PAGE_EXECUTE_WRITECOPY)) && - !(Section->SectionPageProtection & (PAGE_EXECUTE|PAGE_EXECUTE_READ|PAGE_EXECUTE_READWRITE|PAGE_EXECUTE_WRITECOPY))) + !(Section->InitialPageProtection & (PAGE_EXECUTE|PAGE_EXECUTE_READ|PAGE_EXECUTE_READWRITE|PAGE_EXECUTE_WRITECOPY))) { MmUnlockAddressSpace(AddressSpace); return STATUS_SECTION_PROTECTION; @@ -5133,7 +5133,7 @@ MmCreateSection (OUT PVOID * Section, DesiredAccess, ObjectAttributes, SizeOfSection, - SectionPageProtection, + InitialPageProtection, AllocationAttributes, FileObject); } From 819406ac975e8e770a3eec42de9d39608489ffab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Fri, 23 Oct 2020 12:56:08 +0200 Subject: [PATCH 05/99] [NTOSKRNL] Get rid of pagefile-backed sections support in legacy Mm --- ntoskrnl/include/internal/mm.h | 1 - ntoskrnl/mm/section.c | 218 +++++++++------------------------ 2 files changed, 58 insertions(+), 161 deletions(-) diff --git a/ntoskrnl/include/internal/mm.h b/ntoskrnl/include/internal/mm.h index 160c19a8a76..47bb43b2a30 100644 --- a/ntoskrnl/include/internal/mm.h +++ b/ntoskrnl/include/internal/mm.h @@ -87,7 +87,6 @@ typedef ULONG_PTR SWAPENTRY; #define SEC_PHYSICALMEMORY (0x80000000) -#define MM_PAGEFILE_SEGMENT (0x1) #define MM_DATAFILE_SEGMENT (0x2) #define MC_CACHE (0) diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index 9704ddba8f6..aa76377aa9a 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -947,9 +947,7 @@ MmUnsharePageEntrySectionSegment(PROS_SECTION_OBJECT Section, SavedSwapEntry = MmGetSavedSwapEntryPage(Page); if (SavedSwapEntry == 0) { - if (!PageOut && - ((Segment->Flags & MM_PAGEFILE_SEGMENT) || - (Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED))) + if (!PageOut && (Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) { /* * FIXME: @@ -975,8 +973,7 @@ MmUnsharePageEntrySectionSegment(PROS_SECTION_OBJECT Section, } else { - if ((Segment->Flags & MM_PAGEFILE_SEGMENT) || - (Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) + if (Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED) { if (!PageOut) { @@ -1497,11 +1494,6 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, /* * Sanity check */ - if (Segment->Flags & MM_PAGEFILE_SEGMENT) - { - DPRINT1("Found a swaped out private page in a pagefile section.\n"); - KeBugCheck(MEMORY_MANAGEMENT); - } MmDeletePageFileMapping(Process, Address, &SwapEntry); } @@ -1614,9 +1606,8 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, MmCreatePageFileMapping(Process, PAddress, MM_WAIT_ENTRY); MmUnlockAddressSpace(AddressSpace); - if ((Segment->Flags & MM_PAGEFILE_SEGMENT) || - ((Offset.QuadPart >= (LONGLONG)PAGE_ROUND_UP(Segment->RawLength.QuadPart) && - (Section->AllocationAttributes & SEC_IMAGE)))) + if ((Offset.QuadPart >= (LONGLONG)PAGE_ROUND_UP(Segment->RawLength.QuadPart)) && + (Section->AllocationAttributes & SEC_IMAGE)) { MI_SET_USAGE(MI_USAGE_SECTION); if (Process) MI_SET_PROCESS2(Process->ImageFileName); @@ -2102,8 +2093,7 @@ MmPageOutSectionView(PMMSUPPORT AddressSpace, */ if (!Context.Private && Entry != 0) { - if (!(Context.Segment->Flags & MM_PAGEFILE_SEGMENT) && - !(Context.Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) + if (!(Context.Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) { KeBugCheckEx(MEMORY_MANAGEMENT, Entry, (ULONG_PTR)Process, (ULONG_PTR)Address, 0); } @@ -2117,26 +2107,7 @@ MmPageOutSectionView(PMMSUPPORT AddressSpace, * we can't free the page at this point. */ SwapEntry = MmGetSavedSwapEntryPage(Page); - if (Context.Segment->Flags & MM_PAGEFILE_SEGMENT) - { - if (Context.Private) - { - DPRINT1("Found a %s private page (address %p) in a pagefile segment.\n", - Context.WasDirty ? "dirty" : "clean", Address); - KeBugCheckEx(MEMORY_MANAGEMENT, SwapEntry, (ULONG_PTR)Process, (ULONG_PTR)Address, 0); - } - if (!Context.WasDirty && SwapEntry != 0) - { - MmSetSavedSwapEntryPage(Page, 0); - MmLockSectionSegment(Context.Segment); - MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, MAKE_SWAP_SSE(SwapEntry)); - MmUnlockSectionSegment(Context.Segment); - MmReleasePageMemoryConsumer(MC_USER, Page); - MiSetPageEvent(NULL, NULL); - return(STATUS_SUCCESS); - } - } - else if (Context.Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED) + if (Context.Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED) { if (Context.Private) { @@ -2321,8 +2292,7 @@ MmPageOutSectionView(PMMSUPPORT AddressSpace, */ DPRINT("MM: Wrote section page 0x%.8X to swap!\n", Page << PAGE_SHIFT); MmSetSavedSwapEntryPage(Page, 0); - if (Context.Segment->Flags & MM_PAGEFILE_SEGMENT || - Context.Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED) + if (Context.Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED) { MmLockSectionSegment(Context.Segment); MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, MAKE_SWAP_SSE(SwapEntry)); @@ -2715,17 +2685,7 @@ MmpDeleteSection(PVOID ObjectBody) if (Section->Segment == NULL) return; - if (Section->Segment->Flags & MM_PAGEFILE_SEGMENT) - { - MmpFreePageFileSegment(Section->Segment); - MmFreePageTablesSectionSegment(Section->Segment, NULL); - ExFreePool(Section->Segment); - Section->Segment = NULL; - } - else - { - (void)InterlockedDecrementUL(&Section->Segment->ReferenceCount); - } + (void)InterlockedDecrementUL(&Section->Segment->ReferenceCount); } if (Section->FileObject != NULL) { @@ -2758,29 +2718,65 @@ MmCreatePhysicalMemorySection(VOID) UNICODE_STRING Name = RTL_CONSTANT_STRING(L"\\Device\\PhysicalMemory"); LARGE_INTEGER SectionSize; HANDLE Handle; + PMM_SECTION_SEGMENT Segment; /* * Create the section mapping physical memory */ - SectionSize.QuadPart = 0xFFFFFFFF; + SectionSize.QuadPart = ~((ULONG_PTR)0); InitializeObjectAttributes(&Obj, &Name, OBJ_PERMANENT | OBJ_KERNEL_EXCLUSIVE, NULL, NULL); - Status = MmCreateSection((PVOID)&PhysSection, - SECTION_ALL_ACCESS, - &Obj, - &SectionSize, - PAGE_EXECUTE_READWRITE, - SEC_PHYSICALMEMORY, - NULL, - NULL); + /* + * Create the Object + */ + Status = ObCreateObject(KernelMode, + MmSectionObjectType, + &Obj, + ExGetPreviousMode(), + NULL, + sizeof(ROS_SECTION_OBJECT), + 0, + 0, + (PVOID*)&PhysSection); if (!NT_SUCCESS(Status)) { - DPRINT1("Failed to create PhysicalMemory section\n"); - KeBugCheck(MEMORY_MANAGEMENT); + DPRINT1("MmCreatePhysicalMemorySection: failed to create object (0x%lx)\n", Status); + return(Status); } + + /* + * Initialize it + */ + RtlZeroMemory(PhysSection, sizeof(ROS_SECTION_OBJECT)); + PhysSection->Type = 'SC'; + PhysSection->Size = 'TN'; + PhysSection->InitialPageProtection = PAGE_EXECUTE_READWRITE; + PhysSection->AllocationAttributes = SEC_PHYSICALMEMORY; + PhysSection->SizeOfSection = SectionSize; + Segment = ExAllocatePoolWithTag(NonPagedPool, sizeof(MM_SECTION_SEGMENT), + TAG_MM_SECTION_SEGMENT); + if (Segment == NULL) + { + ObDereferenceObject(PhysSection); + return(STATUS_NO_MEMORY); + } + RtlZeroMemory(Segment, sizeof(MM_SECTION_SEGMENT)); + PhysSection->Segment = Segment; + Segment->ReferenceCount = 1; + ExInitializeFastMutex(&Segment->Lock); + Segment->Image.FileOffset = 0; + Segment->Protection = PAGE_EXECUTE_READWRITE; + Segment->RawLength = SectionSize; + Segment->Length = SectionSize; + Segment->Flags = 0; + Segment->WriteCopy = FALSE; + Segment->Image.VirtualAddress = 0; + Segment->Image.Characteristics = 0; + MiInitializeSectionPageTable(Segment); + Status = ObInsertObject(PhysSection, NULL, SECTION_ALL_ACCESS, @@ -2792,8 +2788,6 @@ MmCreatePhysicalMemorySection(VOID) ObDereferenceObject(PhysSection); } ObCloseHandle(Handle, KernelMode); - PhysSection->AllocationAttributes |= SEC_PHYSICALMEMORY; - PhysSection->Segment->Flags &= ~MM_PAGEFILE_SEGMENT; return(STATUS_SUCCESS); } @@ -2831,81 +2825,6 @@ MmInitSectionImplementation(VOID) return(STATUS_SUCCESS); } -NTSTATUS -NTAPI -MmCreatePageFileSection(PROS_SECTION_OBJECT *SectionObject, - ACCESS_MASK DesiredAccess, - POBJECT_ATTRIBUTES ObjectAttributes, - PLARGE_INTEGER UMaximumSize, - ULONG SectionPageProtection, - ULONG AllocationAttributes) -/* - * Create a section which is backed by the pagefile - */ -{ - LARGE_INTEGER MaximumSize; - PROS_SECTION_OBJECT Section; - PMM_SECTION_SEGMENT Segment; - NTSTATUS Status; - - if (UMaximumSize == NULL) - { - DPRINT1("MmCreatePageFileSection: (UMaximumSize == NULL)\n"); - return(STATUS_INVALID_PARAMETER); - } - MaximumSize = *UMaximumSize; - - /* - * Create the section - */ - Status = ObCreateObject(ExGetPreviousMode(), - MmSectionObjectType, - ObjectAttributes, - ExGetPreviousMode(), - NULL, - sizeof(ROS_SECTION_OBJECT), - 0, - 0, - (PVOID*)(PVOID)&Section); - if (!NT_SUCCESS(Status)) - { - DPRINT1("MmCreatePageFileSection: failed to create object (0x%lx)\n", Status); - return(Status); - } - - /* - * Initialize it - */ - RtlZeroMemory(Section, sizeof(ROS_SECTION_OBJECT)); - Section->Type = 'SC'; - Section->Size = 'TN'; - Section->InitialPageProtection = SectionPageProtection; - Section->AllocationAttributes = AllocationAttributes; - Section->SizeOfSection = MaximumSize; - Segment = ExAllocatePoolWithTag(NonPagedPool, sizeof(MM_SECTION_SEGMENT), - TAG_MM_SECTION_SEGMENT); - if (Segment == NULL) - { - ObDereferenceObject(Section); - return(STATUS_NO_MEMORY); - } - RtlZeroMemory(Segment, sizeof(MM_SECTION_SEGMENT)); - Section->Segment = Segment; - Segment->ReferenceCount = 1; - ExInitializeFastMutex(&Segment->Lock); - Segment->Image.FileOffset = 0; - Segment->Protection = SectionPageProtection; - Segment->RawLength.QuadPart = MaximumSize.u.LowPart; - Segment->Length.QuadPart = PAGE_ROUND_UP(MaximumSize.u.LowPart); - Segment->Flags = MM_PAGEFILE_SEGMENT; - Segment->WriteCopy = FALSE; - Segment->Image.VirtualAddress = 0; - Segment->Image.Characteristics = 0; - *SectionObject = Section; - MiInitializeSectionPageTable(Segment); - return(STATUS_SUCCESS); -} - NTSTATUS NTAPI MmCreateDataFileSection(PROS_SECTION_OBJECT *SectionObject, @@ -4026,11 +3945,6 @@ MmFreeSectionPage(PVOID Context, MEMORY_AREA* MemoryArea, PVOID Address, /* * Sanity check */ - if (Segment->Flags & MM_PAGEFILE_SEGMENT) - { - DPRINT1("Found a swap entry for a page in a pagefile section.\n"); - KeBugCheck(MEMORY_MANAGEMENT); - } MmFreeSwapPage(SwapEntry); } else if (Page != 0) @@ -4038,14 +3952,6 @@ MmFreeSectionPage(PVOID Context, MEMORY_AREA* MemoryArea, PVOID Address, if (IS_SWAP_FROM_SSE(Entry) || Page != PFN_FROM_SSE(Entry)) { - /* - * Sanity check - */ - if (Segment->Flags & MM_PAGEFILE_SEGMENT) - { - DPRINT1("Found a private page in a pagefile section.\n"); - KeBugCheck(MEMORY_MANAGEMENT); - } /* * Just dereference private pages */ @@ -5140,17 +5046,9 @@ MmCreateSection (OUT PVOID * Section, #endif else { - if ((AllocationAttributes & SEC_PHYSICALMEMORY) == 0) - { - DPRINT1("Invalid path: %lx %p %p\n", AllocationAttributes, FileObject, FileHandle); - } -// ASSERT(AllocationAttributes & SEC_PHYSICALMEMORY); - Status = MmCreatePageFileSection(SectionObject, - DesiredAccess, - ObjectAttributes, - MaximumSize, - SectionPageProtection, - AllocationAttributes); + /* All cases should be handled above, and the Physical Memorw section was created at initialization phase */ + ASSERT(FALSE); + Status = STATUS_INVALID_PARAMETER; if (FileObject) ObDereferenceObject(FileObject); } From 10ac04ff6eb2a6cdfb0e368fea9ec1b271c2452f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Fri, 23 Oct 2020 13:17:45 +0200 Subject: [PATCH 06/99] [NTOSKRNL] Get rid of ROS_SECTION_OBJECT::AllocationAttributes in favor of MM_SECTION_FLAGS --- ntoskrnl/include/internal/mm.h | 6 +++- ntoskrnl/mm/ARM3/section.c | 9 +++--- ntoskrnl/mm/section.c | 56 +++++++++++++++++++++------------- 3 files changed, 43 insertions(+), 28 deletions(-) diff --git a/ntoskrnl/include/internal/mm.h b/ntoskrnl/include/internal/mm.h index 47bb43b2a30..5700ad8ff65 100644 --- a/ntoskrnl/include/internal/mm.h +++ b/ntoskrnl/include/internal/mm.h @@ -200,7 +200,11 @@ typedef struct _ROS_SECTION_OBJECT CSHORT Size; LARGE_INTEGER SizeOfSection; ULONG InitialPageProtection; - ULONG AllocationAttributes; + union + { + ULONG LongFlags; + MMSECTION_FLAGS Flags; + } u; PFILE_OBJECT FileObject; union { diff --git a/ntoskrnl/mm/ARM3/section.c b/ntoskrnl/mm/ARM3/section.c index fe7e8b46145..d1f1e5bbaf2 100644 --- a/ntoskrnl/mm/ARM3/section.c +++ b/ntoskrnl/mm/ARM3/section.c @@ -1834,7 +1834,7 @@ MmGetFileNameForSection(IN PVOID Section, return STATUS_SECTION_NOT_IMAGE; } } - else if (!(((PROS_SECTION_OBJECT)Section)->AllocationAttributes & SEC_IMAGE)) + else if (!(((PROS_SECTION_OBJECT)Section)->u.Flags.Image)) { /* It's not, fail */ DPRINT1("Not an image section\n"); @@ -3715,8 +3715,7 @@ NtMapViewOfSection(IN HANDLE SectionHandle, return Status; } - if (MiIsRosSectionObject(Section) && - (Section->AllocationAttributes & SEC_PHYSICALMEMORY)) + if (MiIsRosSectionObject(Section) && Section->u.Flags.PhysicalMemory) { if (PreviousMode == UserMode && SafeSectionOffset.QuadPart + SafeViewSize > MmHighestPhysicalPage << PAGE_SHIFT) @@ -3765,7 +3764,7 @@ NtMapViewOfSection(IN HANDLE SectionHandle, { /* Check if this is an image for the current process */ if (MiIsRosSectionObject(Section) && - (Section->AllocationAttributes & SEC_IMAGE) && + (Section->u.Flags.Image) && (Process == PsGetCurrentProcess()) && (Status != STATUS_IMAGE_NOT_AT_BASE)) { @@ -3873,7 +3872,7 @@ NtExtendSection(IN HANDLE SectionHandle, if (!NT_SUCCESS(Status)) return Status; /* Really this should go in MmExtendSection */ - if (!(Section->AllocationAttributes & SEC_FILE)) + if (!Section->u.Flags.File || Section->u.Flags.Image) { DPRINT1("Not extending a file\n"); ObDereferenceObject(Section); diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index aa76377aa9a..9920f36ddcf 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -914,7 +914,7 @@ MmUnsharePageEntrySectionSegment(PROS_SECTION_OBJECT Section, LARGE_INTEGER FileOffset; FileOffset.QuadPart = Offset->QuadPart + Segment->Image.FileOffset; - IsImageSection = Section->AllocationAttributes & SEC_IMAGE ? TRUE : FALSE; + IsImageSection = Section->u.Flags.Image; #endif Page = PFN_FROM_SSE(Entry); @@ -1087,7 +1087,7 @@ MiReadPage(PMEMORY_AREA MemoryArea, SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; RawLength = MemoryArea->Data.SectionData.Segment->RawLength.QuadPart; FileOffset = SegOffset + MemoryArea->Data.SectionData.Segment->Image.FileOffset; - IsImageSection = MemoryArea->Data.SectionData.Section->AllocationAttributes & SEC_IMAGE ? TRUE : FALSE; + IsImageSection = MemoryArea->Data.SectionData.Section->u.Flags.Image; ASSERT(SharedCacheMap); @@ -1557,7 +1557,7 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, /* * Satisfying a page fault on a map of /Device/PhysicalMemory is easy */ - if (Section->AllocationAttributes & SEC_PHYSICALMEMORY) + if (Section->u.Flags.PhysicalMemory) { MmUnlockSectionSegment(Segment); /* @@ -1606,8 +1606,7 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, MmCreatePageFileMapping(Process, PAddress, MM_WAIT_ENTRY); MmUnlockAddressSpace(AddressSpace); - if ((Offset.QuadPart >= (LONGLONG)PAGE_ROUND_UP(Segment->RawLength.QuadPart)) && - (Section->AllocationAttributes & SEC_IMAGE)) + if ((Offset.QuadPart >= (LONGLONG)PAGE_ROUND_UP(Segment->RawLength.QuadPart)) && Section->u.Flags.Image) { MI_SET_USAGE(MI_USAGE_SECTION); if (Process) MI_SET_PROCESS2(Process->ImageFileName); @@ -1989,7 +1988,7 @@ MmPageOutSectionView(PMMSUPPORT AddressSpace, #ifndef NEWCC FileOffset = Context.Offset.QuadPart + Context.Segment->Image.FileOffset; - IsImageSection = Context.Section->AllocationAttributes & SEC_IMAGE ? TRUE : FALSE; + IsImageSection = Context.Section->u.Flags.Image; FileObject = Context.Section->FileObject; if (FileObject != NULL && @@ -2015,7 +2014,7 @@ MmPageOutSectionView(PMMSUPPORT AddressSpace, * This should never happen since mappings of physical memory are never * placed in the rmap lists. */ - if (Context.Section->AllocationAttributes & SEC_PHYSICALMEMORY) + if (Context.Section->u.Flags.PhysicalMemory) { DPRINT1("Trying to page out from physical memory section address 0x%p " "process %p\n", Address, @@ -2368,7 +2367,7 @@ MmWritePageSectionView(PMMSUPPORT AddressSpace, */ Segment = MemoryArea->Data.SectionData.Segment; Section = MemoryArea->Data.SectionData.Section; - IsImageSection = Section->AllocationAttributes & SEC_IMAGE ? TRUE : FALSE; + IsImageSection = Section->u.Flags.Image; FileObject = Section->FileObject; DirectMapped = FALSE; @@ -2395,7 +2394,7 @@ MmWritePageSectionView(PMMSUPPORT AddressSpace, * This should never happen since mappings of physical memory are never * placed in the rmap lists. */ - if (Section->AllocationAttributes & SEC_PHYSICALMEMORY) + if (Section->u.Flags.PhysicalMemory) { DPRINT1("Trying to write back page from physical memory mapped at %p " "process %p\n", Address, @@ -2545,7 +2544,7 @@ MmQuerySectionView(PMEMORY_AREA MemoryArea, } Section = MemoryArea->Data.SectionData.Section; - if (Section->AllocationAttributes & SEC_IMAGE) + if (Section->u.Flags.Image) { Segment = MemoryArea->Data.SectionData.Segment; Info->AllocationBase = (PUCHAR)MA_GetStartingAddress(MemoryArea) - Segment->Image.VirtualAddress; @@ -2621,7 +2620,7 @@ MmpDeleteSection(PVOID ObjectBody) } DPRINT("MmpDeleteSection(ObjectBody %p)\n", ObjectBody); - if (Section->AllocationAttributes & SEC_IMAGE) + if (Section->u.Flags.Image) { ULONG i; ULONG NrSegments; @@ -2754,7 +2753,7 @@ MmCreatePhysicalMemorySection(VOID) PhysSection->Type = 'SC'; PhysSection->Size = 'TN'; PhysSection->InitialPageProtection = PAGE_EXECUTE_READWRITE; - PhysSection->AllocationAttributes = SEC_PHYSICALMEMORY; + PhysSection->u.Flags.PhysicalMemory = 1; PhysSection->SizeOfSection = SectionSize; Segment = ExAllocatePoolWithTag(NonPagedPool, sizeof(MM_SECTION_SEGMENT), TAG_MM_SECTION_SEGMENT); @@ -2869,7 +2868,9 @@ MmCreateDataFileSection(PROS_SECTION_OBJECT *SectionObject, Section->Type = 'SC'; Section->Size = 'TN'; Section->InitialPageProtection = SectionPageProtection; - Section->AllocationAttributes = AllocationAttributes; + Section->u.Flags.File = 1; + if (AllocationAttributes & SEC_NO_CHANGE) + Section->u.Flags.NoChange = 1; /* * FIXME: This is propably not entirely correct. We can't look into @@ -3694,7 +3695,10 @@ MmCreateImageSection(PROS_SECTION_OBJECT *SectionObject, Section->Type = 'SC'; Section->Size = 'TN'; Section->InitialPageProtection = SectionPageProtection; - Section->AllocationAttributes = AllocationAttributes; + Section->u.Flags.File = 1; + Section->u.Flags.Image = 1; + if (AllocationAttributes & SEC_NO_CHANGE) + Section->u.Flags.NoChange = 1; if (FileObject->SectionObjectPointer->ImageSectionObject == NULL) { @@ -3871,7 +3875,7 @@ MmMapViewOfSegment(PMMSUPPORT AddressSpace, MArea->Data.SectionData.Segment = Segment; MArea->Data.SectionData.Section = Section; MArea->Data.SectionData.ViewOffset.QuadPart = ViewOffset; - if (Section->AllocationAttributes & SEC_IMAGE) + if (Section->u.Flags.Image) { MArea->VadNode.u.VadFlags.VadType = VadImageMap; } @@ -4017,7 +4021,7 @@ MmUnmapViewOfSegment(PMMSUPPORT AddressSpace, ExFreePoolWithTag(CurrentRegion, TAG_MM_REGION); } - if (Section->AllocationAttributes & SEC_PHYSICALMEMORY) + if (Section->u.Flags.PhysicalMemory) { Status = MmFreeMemoryArea(AddressSpace, MemoryArea, @@ -4070,7 +4074,7 @@ MiRosUnmapViewOfSection(IN PEPROCESS Process, Section = MemoryArea->Data.SectionData.Section; - if ((Section != NULL) && (Section->AllocationAttributes & SEC_IMAGE)) + if ((Section != NULL) && Section->u.Flags.Image) { ULONG i; ULONG NrSegments; @@ -4237,8 +4241,15 @@ NtQuerySection( _SEH2_TRY { - Sbi->Attributes = RosSection->AllocationAttributes; - if (RosSection->AllocationAttributes & SEC_IMAGE) + Sbi->Attributes = 0; + if (RosSection->u.Flags.Image) + Sbi->Attributes |= SEC_IMAGE; + if (RosSection->u.Flags.File) + Sbi->Attributes |= SEC_FILE; + if (RosSection->u.Flags.NoChange) + Sbi->Attributes |= SEC_NO_CHANGE; + + if (RosSection->u.Flags.Image) { Sbi->BaseAddress = 0; Sbi->Size.QuadPart = 0; @@ -4270,7 +4281,7 @@ NtQuerySection( _SEH2_TRY { - if (RosSection->AllocationAttributes & SEC_IMAGE) + if (RosSection->u.Flags.Image) { PMM_IMAGE_SECTION_OBJECT ImageSectionObject; ImageSectionObject = RosSection->ImageSection; @@ -4448,11 +4459,12 @@ MmMapViewOfSection(IN PVOID SectionObject, Section = (PROS_SECTION_OBJECT)SectionObject; AddressSpace = &Process->Vm; - AllocationType |= (Section->AllocationAttributes & SEC_NO_CHANGE); + if (Section->u.Flags.NoChange) + AllocationType |= SEC_NO_CHANGE; MmLockAddressSpace(AddressSpace); - if (Section->AllocationAttributes & SEC_IMAGE) + if (Section->u.Flags.Image) { ULONG i; ULONG NrSegments; From c3ec5d801c87f8d417e1703bdc19bdfd21959187 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Fri, 23 Oct 2020 14:42:02 +0200 Subject: [PATCH 07/99] [NTOSKRNL] Disable more unused code --- ntoskrnl/cache/section/data.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ntoskrnl/cache/section/data.c b/ntoskrnl/cache/section/data.c index b7649dd1f62..cd92979ac1c 100644 --- a/ntoskrnl/cache/section/data.c +++ b/ntoskrnl/cache/section/data.c @@ -755,6 +755,7 @@ MmUnmapViewOfCacheSegment(PMMSUPPORT AddressSpace, return STATUS_SUCCESS; } +#ifdef NEWCC NTSTATUS NTAPI MmExtendCacheSection(PROS_SECTION_OBJECT Section, @@ -792,6 +793,7 @@ MmExtendCacheSection(PROS_SECTION_OBJECT Section, MmUnlockSectionSegment(Segment); return STATUS_SUCCESS; } +#endif NTSTATUS NTAPI From 25722e647e0f105d77ee28fa6d12999afe21cab8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Fri, 23 Oct 2020 14:42:21 +0200 Subject: [PATCH 08/99] [NTOSKRNL] Embed a SECTION struct inside MM_ROS_SECTION_OBJECT --- ntoskrnl/include/internal/mm.h | 17 +++---------- ntoskrnl/mm/ARM3/miarm.h | 3 +-- ntoskrnl/mm/section.c | 45 ++++++++++++++++++++-------------- 3 files changed, 31 insertions(+), 34 deletions(-) diff --git a/ntoskrnl/include/internal/mm.h b/ntoskrnl/include/internal/mm.h index 5700ad8ff65..c118866d5b5 100644 --- a/ntoskrnl/include/internal/mm.h +++ b/ntoskrnl/include/internal/mm.h @@ -196,21 +196,10 @@ typedef struct _MM_IMAGE_SECTION_OBJECT typedef struct _ROS_SECTION_OBJECT { - CSHORT Type; - CSHORT Size; - LARGE_INTEGER SizeOfSection; - ULONG InitialPageProtection; - union - { - ULONG LongFlags; - MMSECTION_FLAGS Flags; - } u; + SECTION; PFILE_OBJECT FileObject; - union - { - PMM_IMAGE_SECTION_OBJECT ImageSection; - PMM_SECTION_SEGMENT Segment; - }; + + PMM_IMAGE_SECTION_OBJECT ImageSection; } ROS_SECTION_OBJECT, *PROS_SECTION_OBJECT; #define MA_GetStartingAddress(_MemoryArea) ((_MemoryArea)->VadNode.StartingVpn << PAGE_SHIFT) diff --git a/ntoskrnl/mm/ARM3/miarm.h b/ntoskrnl/mm/ARM3/miarm.h index b374e94ad75..ade2696cf1d 100644 --- a/ntoskrnl/mm/ARM3/miarm.h +++ b/ntoskrnl/mm/ARM3/miarm.h @@ -1072,8 +1072,7 @@ BOOLEAN MiIsRosSectionObject(IN PVOID Section) { PROS_SECTION_OBJECT RosSection = Section; - if ((RosSection->Type == 'SC') && (RosSection->Size == 'TN')) return TRUE; - return FALSE; + return RosSection->u.Flags.filler; } #define MI_IS_ROS_PFN(x) ((x)->u4.AweAllocation == TRUE) diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index 9920f36ddcf..fbf44868d90 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -2684,7 +2684,7 @@ MmpDeleteSection(PVOID ObjectBody) if (Section->Segment == NULL) return; - (void)InterlockedDecrementUL(&Section->Segment->ReferenceCount); + (void)InterlockedDecrementUL(&((PMM_SECTION_SEGMENT)Section->Segment)->ReferenceCount); } if (Section->FileObject != NULL) { @@ -2750,8 +2750,9 @@ MmCreatePhysicalMemorySection(VOID) * Initialize it */ RtlZeroMemory(PhysSection, sizeof(ROS_SECTION_OBJECT)); - PhysSection->Type = 'SC'; - PhysSection->Size = 'TN'; + + /* Mark this as a "ROS Section" */ + PhysSection->u.Flags.filler = 1; PhysSection->InitialPageProtection = PAGE_EXECUTE_READWRITE; PhysSection->u.Flags.PhysicalMemory = 1; PhysSection->SizeOfSection = SectionSize; @@ -2763,7 +2764,7 @@ MmCreatePhysicalMemorySection(VOID) return(STATUS_NO_MEMORY); } RtlZeroMemory(Segment, sizeof(MM_SECTION_SEGMENT)); - PhysSection->Segment = Segment; + PhysSection->Segment = (PSEGMENT)Segment; Segment->ReferenceCount = 1; ExInitializeFastMutex(&Segment->Lock); Segment->Image.FileOffset = 0; @@ -2865,8 +2866,9 @@ MmCreateDataFileSection(PROS_SECTION_OBJECT *SectionObject, * Initialize it */ RtlZeroMemory(Section, sizeof(ROS_SECTION_OBJECT)); - Section->Type = 'SC'; - Section->Size = 'TN'; + + /* Mark this as a "ROS" section */ + Section->u.Flags.filler = 1; Section->InitialPageProtection = SectionPageProtection; Section->u.Flags.File = 1; if (AllocationAttributes & SEC_NO_CHANGE) @@ -2958,7 +2960,7 @@ MmCreateDataFileSection(PROS_SECTION_OBJECT *SectionObject, ObDereferenceObject(FileObject); return(STATUS_NO_MEMORY); } - Section->Segment = Segment; + Section->Segment = (PSEGMENT)Segment; Segment->ReferenceCount = 1; ExInitializeFastMutex(&Segment->Lock); /* @@ -2994,7 +2996,7 @@ MmCreateDataFileSection(PROS_SECTION_OBJECT *SectionObject, Segment = (PMM_SECTION_SEGMENT)FileObject->SectionObjectPointer-> DataSectionObject; - Section->Segment = Segment; + Section->Segment = (PSEGMENT)Segment; (void)InterlockedIncrementUL(&Segment->ReferenceCount); MmLockSectionSegment(Segment); @@ -3692,8 +3694,10 @@ MmCreateImageSection(PROS_SECTION_OBJECT *SectionObject, * Initialize it */ RtlZeroMemory(Section, sizeof(ROS_SECTION_OBJECT)); - Section->Type = 'SC'; - Section->Size = 'TN'; + + /* Mark this as a "ROS" Section */ + Section->u.Flags.filler = 1; + Section->InitialPageProtection = SectionPageProtection; Section->u.Flags.File = 1; Section->u.Flags.Image = 1; @@ -4256,8 +4260,8 @@ NtQuerySection( } else { - Sbi->BaseAddress = (PVOID)RosSection->Segment->Image.VirtualAddress; - Sbi->Size.QuadPart = RosSection->Segment->Length.QuadPart; + Sbi->BaseAddress = (PVOID)((PMM_SECTION_SEGMENT)RosSection->Segment)->Image.VirtualAddress; + Sbi->Size.QuadPart = ((PMM_SECTION_SEGMENT)RosSection->Segment)->Length.QuadPart; } if (ResultLength != NULL) @@ -4557,6 +4561,8 @@ MmMapViewOfSection(IN PVOID SectionObject, } else { + PMM_SECTION_SEGMENT Segment = (PMM_SECTION_SEGMENT)Section->Segment; + /* check for write access */ if ((Protect & (PAGE_READWRITE|PAGE_EXECUTE_READWRITE)) && !(Section->InitialPageProtection & (PAGE_READWRITE|PAGE_EXECUTE_READWRITE))) @@ -4605,16 +4611,16 @@ MmMapViewOfSection(IN PVOID SectionObject, *ViewSize = PAGE_ROUND_UP(*ViewSize); - MmLockSectionSegment(Section->Segment); + MmLockSectionSegment(Segment); Status = MmMapViewOfSegment(AddressSpace, Section, - Section->Segment, + Segment, BaseAddress, *ViewSize, Protect, ViewOffset, AllocationType & (MEM_TOP_DOWN|SEC_NO_CHANGE)); - MmUnlockSectionSegment(Section->Segment); + MmUnlockSectionSegment(Segment); if (!NT_SUCCESS(Status)) { MmUnlockAddressSpace(AddressSpace); @@ -4765,6 +4771,7 @@ MmMapViewInSystemSpace (IN PVOID SectionObject, IN OUT PSIZE_T ViewSize) { PROS_SECTION_OBJECT Section; + PMM_SECTION_SEGMENT Segment; PMMSUPPORT AddressSpace; NTSTATUS Status; PAGED_CODE(); @@ -4780,6 +4787,8 @@ MmMapViewInSystemSpace (IN PVOID SectionObject, DPRINT("MmMapViewInSystemSpace() called\n"); Section = (PROS_SECTION_OBJECT)SectionObject; + Segment = (PMM_SECTION_SEGMENT)Section->Segment; + AddressSpace = MmGetKernelAddressSpace(); MmLockAddressSpace(AddressSpace); @@ -4794,19 +4803,19 @@ MmMapViewInSystemSpace (IN PVOID SectionObject, (*ViewSize) = Section->SizeOfSection.u.LowPart; } - MmLockSectionSegment(Section->Segment); + MmLockSectionSegment(Segment); Status = MmMapViewOfSegment(AddressSpace, Section, - Section->Segment, + Segment, MappedBase, *ViewSize, PAGE_READWRITE, 0, 0); - MmUnlockSectionSegment(Section->Segment); + MmUnlockSectionSegment(Segment); MmUnlockAddressSpace(AddressSpace); return Status; From 462d9a097483f6fdabeca18b7aa93b6b4bc9c993 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Fri, 23 Oct 2020 16:16:51 +0200 Subject: [PATCH 09/99] [NTOSKRNL] Unload the cargo: We only use tiny bits of "NEWCC" --- ntoskrnl/cache/section/data.c | 13 ++++++++----- ntoskrnl/cache/section/fault.c | 6 ++++++ ntoskrnl/cache/section/io.c | 2 ++ ntoskrnl/include/internal/mm.h | 2 ++ ntoskrnl/mm/ARM3/section.c | 9 +++++++++ ntoskrnl/mm/marea.c | 10 ++++++++++ ntoskrnl/mm/mmfault.c | 8 ++++---- ntoskrnl/mm/mminit.c | 5 +++++ ntoskrnl/mm/rmap.c | 2 ++ ntoskrnl/mm/section.c | 31 +++++++++++++++++++++++++++++-- ntoskrnl/ntos.cmake | 10 +++++----- 11 files changed, 82 insertions(+), 16 deletions(-) diff --git a/ntoskrnl/cache/section/data.c b/ntoskrnl/cache/section/data.c index cd92979ac1c..1ec6a3326d9 100644 --- a/ntoskrnl/cache/section/data.c +++ b/ntoskrnl/cache/section/data.c @@ -112,6 +112,7 @@ _MmUnlockSectionSegment(PMM_SECTION_SEGMENT Segment, const char *file, int line) //DPRINT("MmUnlockSectionSegment(%p,%s:%d)\n", Segment, file, line); } +#ifdef NEWCC /* MiFlushMappedSection @@ -265,7 +266,6 @@ This deletes a segment entirely including its page map. It must have been unmapped in every address space. */ - VOID NTAPI MmFinalizeSegment(PMM_SECTION_SEGMENT Segment) @@ -274,6 +274,8 @@ MmFinalizeSegment(PMM_SECTION_SEGMENT Segment) DPRINT("Finalize segment %p\n", Segment); + __debugbreak(); + MmLockSectionSegment(Segment); RemoveEntryList(&Segment->ListOfSegments); if (Segment->Flags & MM_DATAFILE_SEGMENT) { @@ -303,7 +305,6 @@ MmFinalizeSegment(PMM_SECTION_SEGMENT Segment) ExFreePoolWithTag(Segment, TAG_MM_SECTION_SEGMENT); } -#ifdef NEWCC NTSTATUS NTAPI MmCreateCacheSection(PROS_SECTION_OBJECT *SectionObject, @@ -534,7 +535,6 @@ MmCreateCacheSection(PROS_SECTION_OBJECT *SectionObject, *SectionObject = Section; return STATUS_SUCCESS; } -#endif NTSTATUS NTAPI @@ -605,6 +605,7 @@ _MiMapViewOfSegment(PMMSUPPORT AddressSpace, return STATUS_SUCCESS; } +#endif /* @@ -621,6 +622,8 @@ MiFreeSegmentPage(PMM_SECTION_SEGMENT Segment, ULONG_PTR Entry; PFILE_OBJECT FileObject = Segment->FileObject; + __debugbreak(); + Entry = MmGetPageEntrySectionSegment(Segment, FileOffset); DPRINTC("MiFreeSegmentPage(%p:%I64x -> Entry %Ix\n", Segment, @@ -710,6 +713,7 @@ MmFreeCacheSectionPage(PVOID Context, } } +#ifdef NEWCC NTSTATUS NTAPI MmUnmapViewOfCacheSegment(PMMSUPPORT AddressSpace, @@ -755,7 +759,6 @@ MmUnmapViewOfCacheSegment(PMMSUPPORT AddressSpace, return STATUS_SUCCESS; } -#ifdef NEWCC NTSTATUS NTAPI MmExtendCacheSection(PROS_SECTION_OBJECT Section, @@ -793,7 +796,6 @@ MmExtendCacheSection(PROS_SECTION_OBJECT Section, MmUnlockSectionSegment(Segment); return STATUS_SUCCESS; } -#endif NTSTATUS NTAPI @@ -844,5 +846,6 @@ MmUnmapCacheViewInSystemSpace (IN PVOID MappedBase) return Status; } +#endif /* NEWCC */ /* EOF */ diff --git a/ntoskrnl/cache/section/fault.c b/ntoskrnl/cache/section/fault.c index c70805a71ae..e869dac88fb 100644 --- a/ntoskrnl/cache/section/fault.c +++ b/ntoskrnl/cache/section/fault.c @@ -82,6 +82,8 @@ rmaps, so each mapping should be immediately followed by an rmap addition. #define DPRINTC DPRINT extern KEVENT MmWaitPageEvent; + +#ifdef NEWCC extern PMMWSL MmWorkingSetList; /* @@ -150,6 +152,7 @@ MmNotPresentFaultCachePage ( if (Segment->FileObject) { + __debugbreak(); DPRINT("FileName %wZ\n", &Segment->FileObject->FileName); } @@ -453,9 +456,11 @@ MiCowCacheSectionPage ( DPRINT("Address 0x%p\n", Address); return STATUS_SUCCESS; } +#endif KEVENT MmWaitPageEvent; +#ifdef NEWCC typedef struct _WORK_QUEUE_WITH_CONTEXT { WORK_QUEUE_ITEM WorkItem; @@ -960,3 +965,4 @@ MmNotPresentFaultCacheSection(KPROCESSOR_MODE Mode, return Status; } +#endif diff --git a/ntoskrnl/cache/section/io.c b/ntoskrnl/cache/section/io.c index 20454abb905..8ac93b06134 100644 --- a/ntoskrnl/cache/section/io.c +++ b/ntoskrnl/cache/section/io.c @@ -188,6 +188,7 @@ MiSimpleRead(PFILE_OBJECT FileObject, return Status; } +#ifdef NEWCC /* Convenience function for writing from kernel space. This issues a paging @@ -334,3 +335,4 @@ _MiWriteBackPage(PFILE_OBJECT FileObject, return Status; } +#endif diff --git a/ntoskrnl/include/internal/mm.h b/ntoskrnl/include/internal/mm.h index c118866d5b5..1fbe513524f 100644 --- a/ntoskrnl/include/internal/mm.h +++ b/ntoskrnl/include/internal/mm.h @@ -69,7 +69,9 @@ typedef ULONG_PTR SWAPENTRY; #endif #define MEMORY_AREA_SECTION_VIEW (1) +#ifdef NEWCC #define MEMORY_AREA_CACHE (2) +#endif #define MEMORY_AREA_OWNED_BY_ARM3 (15) #define MEMORY_AREA_STATIC (0x80000000) diff --git a/ntoskrnl/mm/ARM3/section.c b/ntoskrnl/mm/ARM3/section.c index d1f1e5bbaf2..bff71dfe81c 100644 --- a/ntoskrnl/mm/ARM3/section.c +++ b/ntoskrnl/mm/ARM3/section.c @@ -1666,9 +1666,14 @@ MiGetFileObjectForSectionAddress( } else { +#ifdef NEWCC ASSERT(MemoryArea->Type == MEMORY_AREA_CACHE); DPRINT1("Address is a cache section!\n"); return STATUS_SECTION_NOT_IMAGE; +#else + ASSERT(FALSE); + return STATUS_SECTION_NOT_IMAGE; +#endif } } else @@ -1739,8 +1744,12 @@ MiGetFileObjectForVad( } else { +#ifdef NEWCC ASSERT(MemoryArea->Type == MEMORY_AREA_CACHE); DPRINT1("VAD is a cache section!\n"); +#else + ASSERT(FALSE); +#endif return NULL; } } diff --git a/ntoskrnl/mm/marea.c b/ntoskrnl/mm/marea.c index d953f422f96..3736a5a36c8 100644 --- a/ntoskrnl/mm/marea.c +++ b/ntoskrnl/mm/marea.c @@ -178,7 +178,11 @@ MmInsertMemoryArea( ASSERT(Process != NULL); if (marea->Type != MEMORY_AREA_OWNED_BY_ARM3) { +#ifdef NEWCC ASSERT(marea->Type == MEMORY_AREA_SECTION_VIEW || marea->Type == MEMORY_AREA_CACHE); +#else + ASSERT(marea->Type == MEMORY_AREA_SECTION_VIEW); +#endif /* Insert the VAD */ MiLockProcessWorkingSetUnsafe(PsGetCurrentProcess(), PsGetCurrentThread()); @@ -353,7 +357,11 @@ MmFreeMemoryArea( if (MemoryArea->Vad) { ASSERT(MemoryArea->VadNode.EndingVpn + 1 < (ULONG_PTR)MmSystemRangeStart >> PAGE_SHIFT); +#ifdef NEWCC ASSERT(MemoryArea->Type == MEMORY_AREA_SECTION_VIEW || MemoryArea->Type == MEMORY_AREA_CACHE); +#else + ASSERT(MemoryArea->Type == MEMORY_AREA_SECTION_VIEW); +#endif /* MmCleanProcessAddressSpace might have removed it (and this would be MmDeleteProcessAdressSpace) */ ASSERT(MemoryArea->VadNode.u.VadFlags.Spare != 0); @@ -545,10 +553,12 @@ MiRosCleanupMemoryArea( { Status = MiRosUnmapViewOfSection(Process, BaseAddress, Process->ProcessExiting); } +#ifdef NEWCC else if (MemoryArea->Type == MEMORY_AREA_CACHE) { Status = MmUnmapViewOfCacheSegment(&Process->Vm, BaseAddress); } +#endif else { /* There shouldn't be anything else! */ diff --git a/ntoskrnl/mm/mmfault.c b/ntoskrnl/mm/mmfault.c index 68910e7a2d9..75f7b584a22 100644 --- a/ntoskrnl/mm/mmfault.c +++ b/ntoskrnl/mm/mmfault.c @@ -79,7 +79,7 @@ MmpAccessFault(KPROCESSOR_MODE Mode, MemoryArea, (PVOID)Address); break; - +#ifdef NEWCC case MEMORY_AREA_CACHE: // This code locks for itself to keep from having to break a lock // passed in. @@ -89,7 +89,7 @@ MmpAccessFault(KPROCESSOR_MODE Mode, if (!FromMdl) MmLockAddressSpace(AddressSpace); break; - +#endif default: Status = STATUS_ACCESS_VIOLATION; break; @@ -171,7 +171,7 @@ MmNotPresentFault(KPROCESSOR_MODE Mode, (PVOID)Address, FromMdl); break; - +#ifdef NEWCC case MEMORY_AREA_CACHE: // This code locks for itself to keep from having to break a lock // passed in. @@ -181,7 +181,7 @@ MmNotPresentFault(KPROCESSOR_MODE Mode, if (!FromMdl) MmLockAddressSpace(AddressSpace); break; - +#endif default: Status = STATUS_ACCESS_VIOLATION; break; diff --git a/ntoskrnl/mm/mminit.c b/ntoskrnl/mm/mminit.c index 422ba568bb9..6c1a5f274d6 100644 --- a/ntoskrnl/mm/mminit.c +++ b/ntoskrnl/mm/mminit.c @@ -214,12 +214,17 @@ MmInitSystem(IN ULONG Phase, /* Initialize the kernel address space */ ASSERT(Phase == 1); +#ifdef NEWCC InitializeListHead(&MiSegmentList); ExInitializeFastMutex(&MiGlobalPageOperation); KeInitializeEvent(&MmWaitPageEvent, SynchronizationEvent, FALSE); // Until we're fully demand paged, we can do things the old way through // the balance manager + // CcInitView will override this... MmInitializeMemoryConsumer(MC_CACHE, MiRosTrimCache); +#else + KeInitializeEvent(&MmWaitPageEvent, SynchronizationEvent, FALSE); +#endif MmKernelAddressSpace = &PsIdleProcess->Vm; diff --git a/ntoskrnl/mm/rmap.c b/ntoskrnl/mm/rmap.c index aa6a7b0d6f2..9fea631dd09 100644 --- a/ntoskrnl/mm/rmap.c +++ b/ntoskrnl/mm/rmap.c @@ -167,12 +167,14 @@ MmPageOutPhysicalAddress(PFN_NUMBER Page) */ Status = MmPageOutSectionView(AddressSpace, MemoryArea, Address, Entry); } +#ifdef NEWCC else if (Type == MEMORY_AREA_CACHE) { /* NEWCC does locking itself */ MmUnlockAddressSpace(AddressSpace); Status = MmpPageOutPhysicalAddress(Page); } +#endif else { KeBugCheck(MEMORY_MANAGEMENT); diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index fbf44868d90..32ee1f48786 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -60,6 +60,29 @@ extern MMSESSION MmSession; +#ifndef NEWCC +KEVENT MmWaitPageEvent; + +VOID +NTAPI +_MmLockSectionSegment(PMM_SECTION_SEGMENT Segment, const char *file, int line) +{ + //DPRINT("MmLockSectionSegment(%p,%s:%d)\n", Segment, file, line); + ExAcquireFastMutex(&Segment->Lock); + Segment->Locked = TRUE; +} + +VOID +NTAPI +_MmUnlockSectionSegment(PMM_SECTION_SEGMENT Segment, const char *file, int line) +{ + ASSERT(Segment->Locked); + Segment->Locked = FALSE; + ExReleaseFastMutex(&Segment->Lock); + //DPRINT("MmUnlockSectionSegment(%p,%s:%d)\n", Segment, file, line); +} +#endif + NTSTATUS NTAPI MiMapViewInSystemSpace(IN PVOID Section, @@ -4067,9 +4090,13 @@ MiRosUnmapViewOfSection(IN PEPROCESS Process, MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace, BaseAddress); if (MemoryArea == NULL || - ((MemoryArea->Type != MEMORY_AREA_SECTION_VIEW) && - (MemoryArea->Type != MEMORY_AREA_CACHE)) || +#ifdef NEWCC + ((MemoryArea->Type != MEMORY_AREA_SECTION_VIEW) && (MemoryArea->Type != MEMORY_AREA_CACHE)) || +#else + (MemoryArea->Type != MEMORY_AREA_SECTION_VIEW) || +#endif MemoryArea->DeleteInProgress) + { if (MemoryArea) ASSERT(MemoryArea->Type != MEMORY_AREA_OWNED_BY_ARM3); MmUnlockAddressSpace(AddressSpace); diff --git a/ntoskrnl/ntos.cmake b/ntoskrnl/ntos.cmake index 79317e880db..2a098336288 100644 --- a/ntoskrnl/ntos.cmake +++ b/ntoskrnl/ntos.cmake @@ -28,7 +28,11 @@ if(NEWCC) ${REACTOS_SOURCE_DIR}/ntoskrnl/cache/lazyrite.c ${REACTOS_SOURCE_DIR}/ntoskrnl/cache/logsup.c ${REACTOS_SOURCE_DIR}/ntoskrnl/cache/mdlsup.c - ${REACTOS_SOURCE_DIR}/ntoskrnl/cache/pinsup.c) + ${REACTOS_SOURCE_DIR}/ntoskrnl/cache/pinsup.c + ${REACTOS_SOURCE_DIR}/ntoskrnl/cache/section/fault.c + ${REACTOS_SOURCE_DIR}/ntoskrnl/cache/section/swapout.c + ${REACTOS_SOURCE_DIR}/ntoskrnl/cache/section/data.c + ${REACTOS_SOURCE_DIR}/ntoskrnl/cache/section/reqtools.c) else() list(APPEND SOURCE ${REACTOS_SOURCE_DIR}/ntoskrnl/cc/cacheman.c @@ -42,11 +46,7 @@ endif() list(APPEND SOURCE ${REACTOS_SOURCE_DIR}/ntoskrnl/cache/section/io.c - ${REACTOS_SOURCE_DIR}/ntoskrnl/cache/section/data.c - ${REACTOS_SOURCE_DIR}/ntoskrnl/cache/section/fault.c - ${REACTOS_SOURCE_DIR}/ntoskrnl/cache/section/reqtools.c ${REACTOS_SOURCE_DIR}/ntoskrnl/cache/section/sptab.c - ${REACTOS_SOURCE_DIR}/ntoskrnl/cache/section/swapout.c ${REACTOS_SOURCE_DIR}/ntoskrnl/config/cmalloc.c ${REACTOS_SOURCE_DIR}/ntoskrnl/config/cmapi.c ${REACTOS_SOURCE_DIR}/ntoskrnl/config/cmboot.c From defbf63416fc622bdfbbbe8cb0283d559cbf84a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Fri, 23 Oct 2020 16:39:15 +0200 Subject: [PATCH 10/99] [NTOSKRNL] Use PSECTION struct where possible --- ntoskrnl/include/internal/mm.h | 2 +- ntoskrnl/mm/ARM3/miarm.h | 2 +- ntoskrnl/mm/ARM3/procsup.c | 10 ++++------ ntoskrnl/mm/ARM3/section.c | 15 +++------------ 4 files changed, 9 insertions(+), 20 deletions(-) diff --git a/ntoskrnl/include/internal/mm.h b/ntoskrnl/include/internal/mm.h index 1fbe513524f..4ca7e6717aa 100644 --- a/ntoskrnl/include/internal/mm.h +++ b/ntoskrnl/include/internal/mm.h @@ -221,7 +221,7 @@ typedef struct _MEMORY_AREA { struct { - ROS_SECTION_OBJECT* Section; + PROS_SECTION_OBJECT Section; LARGE_INTEGER ViewOffset; PMM_SECTION_SEGMENT Segment; LIST_ENTRY RegionListHead; diff --git a/ntoskrnl/mm/ARM3/miarm.h b/ntoskrnl/mm/ARM3/miarm.h index ade2696cf1d..025cd6f6561 100644 --- a/ntoskrnl/mm/ARM3/miarm.h +++ b/ntoskrnl/mm/ARM3/miarm.h @@ -1071,7 +1071,7 @@ FORCEINLINE BOOLEAN MiIsRosSectionObject(IN PVOID Section) { - PROS_SECTION_OBJECT RosSection = Section; + PSECTION RosSection = Section; return RosSection->u.Flags.filler; } diff --git a/ntoskrnl/mm/ARM3/procsup.c b/ntoskrnl/mm/ARM3/procsup.c index a183f8a099f..eb8e68180d4 100644 --- a/ntoskrnl/mm/ARM3/procsup.c +++ b/ntoskrnl/mm/ARM3/procsup.c @@ -941,7 +941,6 @@ MmInitializeProcessAddressSpace(IN PEPROCESS Process, NTSTATUS Status = STATUS_SUCCESS; SIZE_T ViewSize = 0; PVOID ImageBase = 0; - PROS_SECTION_OBJECT SectionObject = Section; PMMPTE PointerPte; KIRQL OldIrql; PMMPDE PointerPde; @@ -1048,10 +1047,11 @@ MmInitializeProcessAddressSpace(IN PEPROCESS Process, #endif /* Check if there's a Section Object */ - if (SectionObject) + if (Section) { /* Determine the image file name and save it to EPROCESS */ - FileName = SectionObject->FileObject->FileName; + PFILE_OBJECT FileObject = MmGetFileObjectForSection(Section); + FileName = FileObject->FileName; Source = (PWCHAR)((PCHAR)FileName.Buffer + FileName.Length); if (FileName.Buffer) { @@ -1083,9 +1083,7 @@ MmInitializeProcessAddressSpace(IN PEPROCESS Process, if (AuditName) { /* Setup the audit name */ - Status = SeInitializeProcessAuditName(SectionObject->FileObject, - FALSE, - AuditName); + Status = SeInitializeProcessAuditName(FileObject, FALSE, AuditName); if (!NT_SUCCESS(Status)) { /* Fail */ diff --git a/ntoskrnl/mm/ARM3/section.c b/ntoskrnl/mm/ARM3/section.c index bff71dfe81c..1090a8dfada 100644 --- a/ntoskrnl/mm/ARM3/section.c +++ b/ntoskrnl/mm/ARM3/section.c @@ -1831,19 +1831,10 @@ MmGetFileNameForSection(IN PVOID Section, OUT POBJECT_NAME_INFORMATION *ModuleName) { PFILE_OBJECT FileObject; + PSECTION SectionObject = Section; /* Make sure it's an image section */ - if (MiIsRosSectionObject(Section) == FALSE) - { - /* Check ARM3 Section flag */ - if (((PSECTION)Section)->u.Flags.Image == 0) - { - /* It's not, fail */ - DPRINT1("Not an image section\n"); - return STATUS_SECTION_NOT_IMAGE; - } - } - else if (!(((PROS_SECTION_OBJECT)Section)->u.Flags.Image)) + if (SectionObject->u.Flags.Image == 0) { /* It's not, fail */ DPRINT1("Not an image section\n"); @@ -3594,7 +3585,7 @@ NtMapViewOfSection(IN HANDLE SectionHandle, PVOID SafeBaseAddress; LARGE_INTEGER SafeSectionOffset; SIZE_T SafeViewSize; - PROS_SECTION_OBJECT Section; + PSECTION Section; PEPROCESS Process; NTSTATUS Status; ACCESS_MASK DesiredAccess; From c9f924940a954b0f558319e1df95c1c2d71e2dfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Fri, 23 Oct 2020 16:44:24 +0200 Subject: [PATCH 11/99] [NTOSKRNL/MM] Get rid of useless members of MEMORY_AREA struct --- ntoskrnl/include/internal/mm.h | 20 +++----- ntoskrnl/mm/ARM3/section.c | 4 +- ntoskrnl/mm/rmap.c | 12 ++--- ntoskrnl/mm/section.c | 90 +++++++++++++++++----------------- 4 files changed, 60 insertions(+), 66 deletions(-) diff --git a/ntoskrnl/include/internal/mm.h b/ntoskrnl/include/internal/mm.h index 4ca7e6717aa..4bb6d1a60ca 100644 --- a/ntoskrnl/include/internal/mm.h +++ b/ntoskrnl/include/internal/mm.h @@ -217,20 +217,14 @@ typedef struct _MEMORY_AREA BOOLEAN DeleteInProgress; ULONG Magic; PVOID Vad; - union + + struct { - struct - { - PROS_SECTION_OBJECT Section; - LARGE_INTEGER ViewOffset; - PMM_SECTION_SEGMENT Segment; - LIST_ENTRY RegionListHead; - } SectionData; - struct - { - LIST_ENTRY RegionListHead; - } VirtualMemoryData; - } Data; + PROS_SECTION_OBJECT Section; + LARGE_INTEGER ViewOffset; + PMM_SECTION_SEGMENT Segment; + LIST_ENTRY RegionListHead; + } SectionData; } MEMORY_AREA, *PMEMORY_AREA; typedef struct _MM_RMAP_ENTRY diff --git a/ntoskrnl/mm/ARM3/section.c b/ntoskrnl/mm/ARM3/section.c index 1090a8dfada..58d261e5476 100644 --- a/ntoskrnl/mm/ARM3/section.c +++ b/ntoskrnl/mm/ARM3/section.c @@ -1661,7 +1661,7 @@ MiGetFileObjectForSectionAddress( if (MemoryArea->Type == MEMORY_AREA_SECTION_VIEW) { /* Get the section pointer to the SECTION_OBJECT */ - Section = MemoryArea->Data.SectionData.Section; + Section = MemoryArea->SectionData.Section; *FileObject = Section->FileObject; } else @@ -1739,7 +1739,7 @@ MiGetFileObjectForVad( if (MemoryArea->Type == MEMORY_AREA_SECTION_VIEW) { /* Get the section pointer to the SECTION_OBJECT */ - Section = MemoryArea->Data.SectionData.Section; + Section = MemoryArea->SectionData.Section; FileObject = Section->FileObject; } else diff --git a/ntoskrnl/mm/rmap.c b/ntoskrnl/mm/rmap.c index 9fea631dd09..9f5938d7496 100644 --- a/ntoskrnl/mm/rmap.c +++ b/ntoskrnl/mm/rmap.c @@ -132,19 +132,19 @@ MmPageOutPhysicalAddress(PFN_NUMBER Page) if (Type == MEMORY_AREA_SECTION_VIEW) { ULONG_PTR Entry; - Offset = MemoryArea->Data.SectionData.ViewOffset.QuadPart + + Offset = MemoryArea->SectionData.ViewOffset.QuadPart + ((ULONG_PTR)Address - MA_GetStartingAddress(MemoryArea)); - MmLockSectionSegment(MemoryArea->Data.SectionData.Segment); + MmLockSectionSegment(MemoryArea->SectionData.Segment); /* * Get or create a pageop */ - Entry = MmGetPageEntrySectionSegment(MemoryArea->Data.SectionData.Segment, + Entry = MmGetPageEntrySectionSegment(MemoryArea->SectionData.Segment, (PLARGE_INTEGER)&Offset); if (Entry && MM_IS_WAIT_PTE(Entry)) { - MmUnlockSectionSegment(MemoryArea->Data.SectionData.Segment); + MmUnlockSectionSegment(MemoryArea->SectionData.Segment); MmUnlockAddressSpace(AddressSpace); if (Address < MmSystemRangeStart) { @@ -154,12 +154,12 @@ MmPageOutPhysicalAddress(PFN_NUMBER Page) return(STATUS_UNSUCCESSFUL); } - MmSetPageEntrySectionSegment(MemoryArea->Data.SectionData.Segment, (PLARGE_INTEGER)&Offset, MAKE_SWAP_SSE(MM_WAIT_ENTRY)); + MmSetPageEntrySectionSegment(MemoryArea->SectionData.Segment, (PLARGE_INTEGER)&Offset, MAKE_SWAP_SSE(MM_WAIT_ENTRY)); /* * Release locks now we have a page op. */ - MmUnlockSectionSegment(MemoryArea->Data.SectionData.Segment); + MmUnlockSectionSegment(MemoryArea->SectionData.Segment); MmUnlockAddressSpace(AddressSpace); /* diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index 32ee1f48786..91ef5133849 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -1043,12 +1043,12 @@ BOOLEAN MiIsPageFromCache(PMEMORY_AREA MemoryArea, LONGLONG SegOffset) { #ifndef NEWCC - if (!(MemoryArea->Data.SectionData.Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) + if (!(MemoryArea->SectionData.Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) { PROS_SHARED_CACHE_MAP SharedCacheMap; PROS_VACB Vacb; - SharedCacheMap = MemoryArea->Data.SectionData.Section->FileObject->SectionObjectPointer->SharedCacheMap; - Vacb = CcRosLookupVacb(SharedCacheMap, SegOffset + MemoryArea->Data.SectionData.Segment->Image.FileOffset); + SharedCacheMap = MemoryArea->SectionData.Section->FileObject->SectionObjectPointer->SharedCacheMap; + Vacb = CcRosLookupVacb(SharedCacheMap, SegOffset + MemoryArea->SectionData.Segment->Image.FileOffset); if (Vacb) { CcRosReleaseVacb(SharedCacheMap, Vacb, Vacb->Valid, FALSE, TRUE); @@ -1106,11 +1106,11 @@ MiReadPage(PMEMORY_AREA MemoryArea, BOOLEAN IsImageSection; LONGLONG Length; - FileObject = MemoryArea->Data.SectionData.Section->FileObject; + FileObject = MemoryArea->SectionData.Section->FileObject; SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; - RawLength = MemoryArea->Data.SectionData.Segment->RawLength.QuadPart; - FileOffset = SegOffset + MemoryArea->Data.SectionData.Segment->Image.FileOffset; - IsImageSection = MemoryArea->Data.SectionData.Section->u.Flags.Image; + RawLength = MemoryArea->SectionData.Segment->RawLength.QuadPart; + FileOffset = SegOffset + MemoryArea->SectionData.Segment->Image.FileOffset; + IsImageSection = MemoryArea->SectionData.Section->u.Flags.Image; ASSERT(SharedCacheMap); @@ -1123,7 +1123,7 @@ MiReadPage(PMEMORY_AREA MemoryArea, */ if (((FileOffset % PAGE_SIZE) == 0) && ((SegOffset + PAGE_SIZE <= RawLength) || !IsImageSection) && - !(MemoryArea->Data.SectionData.Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) + !(MemoryArea->SectionData.Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) { /* @@ -1282,9 +1282,9 @@ MiReadPage(PMEMORY_AREA MemoryArea, RtlZeroMemory(&Resources, sizeof(MM_REQUIRED_RESOURCES)); - Resources.Context = MemoryArea->Data.SectionData.Section->FileObject; + Resources.Context = MemoryArea->SectionData.Section->FileObject; Resources.FileOffset.QuadPart = SegOffset + - MemoryArea->Data.SectionData.Segment->Image.FileOffset; + MemoryArea->SectionData.Segment->Image.FileOffset; Resources.Consumer = MC_USER; Resources.Amount = PAGE_SIZE; @@ -1313,7 +1313,7 @@ MmAlterViewAttributes(PMMSUPPORT AddressSpace, MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace, BaseAddress); ASSERT(MemoryArea != NULL); - Segment = MemoryArea->Data.SectionData.Segment; + Segment = MemoryArea->SectionData.Segment; MmLockSectionSegment(Segment); if ((Segment->WriteCopy) && @@ -1351,7 +1351,7 @@ MmAlterViewAttributes(PMMSUPPORT AddressSpace, PFN_NUMBER Page; Offset.QuadPart = (ULONG_PTR)Address - MA_GetStartingAddress(MemoryArea) - + MemoryArea->Data.SectionData.ViewOffset.QuadPart; + + MemoryArea->SectionData.ViewOffset.QuadPart; Entry = MmGetPageEntrySectionSegment(Segment, &Offset); /* * An MM_WAIT_ENTRY is ok in this case... It'll just count as @@ -1423,12 +1423,12 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, PAddress = MM_ROUND_DOWN(Address, PAGE_SIZE); Offset.QuadPart = (ULONG_PTR)PAddress - MA_GetStartingAddress(MemoryArea) - + MemoryArea->Data.SectionData.ViewOffset.QuadPart; + + MemoryArea->SectionData.ViewOffset.QuadPart; - Segment = MemoryArea->Data.SectionData.Segment; - Section = MemoryArea->Data.SectionData.Section; + Segment = MemoryArea->SectionData.Segment; + Section = MemoryArea->SectionData.Section; Region = MmFindRegion((PVOID)MA_GetStartingAddress(MemoryArea), - &MemoryArea->Data.SectionData.RegionListHead, + &MemoryArea->SectionData.RegionListHead, Address, NULL); ASSERT(Region != NULL); @@ -1442,7 +1442,7 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, { /* Remove it */ Status = MmAlterRegion(AddressSpace, (PVOID)MA_GetStartingAddress(MemoryArea), - &MemoryArea->Data.SectionData.RegionListHead, + &MemoryArea->SectionData.RegionListHead, Address, PAGE_SIZE, Region->Type, Region->Protect & ~PAGE_GUARD, MmAlterViewAttributes); @@ -1844,12 +1844,12 @@ MmAccessFaultSectionView(PMMSUPPORT AddressSpace, */ PAddress = MM_ROUND_DOWN(Address, PAGE_SIZE); Offset.QuadPart = (ULONG_PTR)PAddress - MA_GetStartingAddress(MemoryArea) - + MemoryArea->Data.SectionData.ViewOffset.QuadPart; + + MemoryArea->SectionData.ViewOffset.QuadPart; - Segment = MemoryArea->Data.SectionData.Segment; - Section = MemoryArea->Data.SectionData.Section; + Segment = MemoryArea->SectionData.Segment; + Section = MemoryArea->SectionData.Section; Region = MmFindRegion((PVOID)MA_GetStartingAddress(MemoryArea), - &MemoryArea->Data.SectionData.RegionListHead, + &MemoryArea->SectionData.RegionListHead, Address, NULL); ASSERT(Region != NULL); @@ -1997,13 +1997,13 @@ MmPageOutSectionView(PMMSUPPORT AddressSpace, /* * Get the segment and section. */ - Context.Segment = MemoryArea->Data.SectionData.Segment; - Context.Section = MemoryArea->Data.SectionData.Section; + Context.Segment = MemoryArea->SectionData.Segment; + Context.Section = MemoryArea->SectionData.Section; Context.SectionEntry = Entry; Context.CallingProcess = Process; Context.Offset.QuadPart = (ULONG_PTR)Address - MA_GetStartingAddress(MemoryArea) - + MemoryArea->Data.SectionData.ViewOffset.QuadPart; + + MemoryArea->SectionData.ViewOffset.QuadPart; DirectMapped = FALSE; @@ -2383,13 +2383,13 @@ MmWritePageSectionView(PMMSUPPORT AddressSpace, Address = (PVOID)PAGE_ROUND_DOWN(Address); Offset.QuadPart = (ULONG_PTR)Address - MA_GetStartingAddress(MemoryArea) - + MemoryArea->Data.SectionData.ViewOffset.QuadPart; + + MemoryArea->SectionData.ViewOffset.QuadPart; /* * Get the segment and section. */ - Segment = MemoryArea->Data.SectionData.Segment; - Section = MemoryArea->Data.SectionData.Section; + Segment = MemoryArea->SectionData.Segment; + Section = MemoryArea->SectionData.Section; IsImageSection = Section->u.Flags.Image; FileObject = Section->FileObject; @@ -2528,7 +2528,7 @@ MmProtectSectionView(PMMSUPPORT AddressSpace, Length = (ULONG)MaxLength; Region = MmFindRegion((PVOID)MA_GetStartingAddress(MemoryArea), - &MemoryArea->Data.SectionData.RegionListHead, + &MemoryArea->SectionData.RegionListHead, BaseAddress, NULL); ASSERT(Region != NULL); @@ -2540,7 +2540,7 @@ MmProtectSectionView(PMMSUPPORT AddressSpace, *OldProtect = Region->Protect; Status = MmAlterRegion(AddressSpace, (PVOID)MA_GetStartingAddress(MemoryArea), - &MemoryArea->Data.SectionData.RegionListHead, + &MemoryArea->SectionData.RegionListHead, BaseAddress, Length, Region->Type, Protect, MmAlterViewAttributes); @@ -2559,17 +2559,17 @@ MmQuerySectionView(PMEMORY_AREA MemoryArea, PMM_SECTION_SEGMENT Segment; Region = MmFindRegion((PVOID)MA_GetStartingAddress(MemoryArea), - &MemoryArea->Data.SectionData.RegionListHead, + &MemoryArea->SectionData.RegionListHead, Address, &RegionBaseAddress); if (Region == NULL) { return STATUS_UNSUCCESSFUL; } - Section = MemoryArea->Data.SectionData.Section; + Section = MemoryArea->SectionData.Section; if (Section->u.Flags.Image) { - Segment = MemoryArea->Data.SectionData.Segment; + Segment = MemoryArea->SectionData.Segment; Info->AllocationBase = (PUCHAR)MA_GetStartingAddress(MemoryArea) - Segment->Image.VirtualAddress; Info->Type = MEM_IMAGE; } @@ -3899,15 +3899,15 @@ MmMapViewOfSegment(PMMSUPPORT AddressSpace, ObReferenceObject((PVOID)Section); - MArea->Data.SectionData.Segment = Segment; - MArea->Data.SectionData.Section = Section; - MArea->Data.SectionData.ViewOffset.QuadPart = ViewOffset; + MArea->SectionData.Segment = Segment; + MArea->SectionData.Section = Section; + MArea->SectionData.ViewOffset.QuadPart = ViewOffset; if (Section->u.Flags.Image) { MArea->VadNode.u.VadFlags.VadType = VadImageMap; } - MmInitializeRegion(&MArea->Data.SectionData.RegionListHead, + MmInitializeRegion(&MArea->SectionData.RegionListHead, ViewSize, 0, Protect); return(STATUS_SUCCESS); @@ -3936,10 +3936,10 @@ MmFreeSectionPage(PVOID Context, MEMORY_AREA* MemoryArea, PVOID Address, Address = (PVOID)PAGE_ROUND_DOWN(Address); Offset.QuadPart = ((ULONG_PTR)Address - MA_GetStartingAddress(MemoryArea)) + - MemoryArea->Data.SectionData.ViewOffset.QuadPart; + MemoryArea->SectionData.ViewOffset.QuadPart; - Section = MemoryArea->Data.SectionData.Section; - Segment = MemoryArea->Data.SectionData.Segment; + Section = MemoryArea->SectionData.Section; + Segment = MemoryArea->SectionData.Segment; Entry = MmGetPageEntrySectionSegment(Segment, &Offset); while (Entry && MM_IS_WAIT_PTE(Entry)) @@ -3963,7 +3963,7 @@ MmFreeSectionPage(PVOID Context, MEMORY_AREA* MemoryArea, PVOID Address, if (Page == PFN_FROM_SSE(Entry) && Dirty) { #ifndef NEWCC - FileObject = MemoryArea->Data.SectionData.Section->FileObject; + FileObject = MemoryArea->SectionData.Section->FileObject; SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; CcRosMarkDirtyFile(SharedCacheMap, Offset.QuadPart + Segment->Image.FileOffset); #endif @@ -4022,8 +4022,8 @@ MmUnmapViewOfSegment(PMMSUPPORT AddressSpace, return(STATUS_UNSUCCESSFUL); } - Section = MemoryArea->Data.SectionData.Section; - Segment = MemoryArea->Data.SectionData.Segment; + Section = MemoryArea->SectionData.Section; + Segment = MemoryArea->SectionData.Segment; #ifdef NEWCC if (Segment->Flags & MM_DATAFILE_SEGMENT) @@ -4040,7 +4040,7 @@ MmUnmapViewOfSegment(PMMSUPPORT AddressSpace, MmLockSectionSegment(Segment); - RegionListHead = &MemoryArea->Data.SectionData.RegionListHead; + RegionListHead = &MemoryArea->SectionData.RegionListHead; while (!IsListEmpty(RegionListHead)) { CurrentEntry = RemoveHeadList(RegionListHead); @@ -4103,7 +4103,7 @@ MiRosUnmapViewOfSection(IN PEPROCESS Process, return STATUS_NOT_MAPPED_VIEW; } - Section = MemoryArea->Data.SectionData.Section; + Section = MemoryArea->SectionData.Section; if ((Section != NULL) && Section->u.Flags.Image) { @@ -4113,7 +4113,7 @@ MiRosUnmapViewOfSection(IN PEPROCESS Process, PMM_SECTION_SEGMENT SectionSegments; PMM_SECTION_SEGMENT Segment; - Segment = MemoryArea->Data.SectionData.Segment; + Segment = MemoryArea->SectionData.Segment; ImageSectionObject = Section->ImageSection; SectionSegments = ImageSectionObject->Segments; NrSegments = ImageSectionObject->NrSegments; From c6a87f28bf4a0a3270adf6114ed2589e200c5b86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Fri, 23 Oct 2020 16:55:00 +0200 Subject: [PATCH 12/99] [NTOSKRNL] Get rid of MEMORY_AREA::Protect Use the VAD node instead --- ntoskrnl/include/internal/mm.h | 1 - ntoskrnl/mm/marea.c | 10 +++++----- ntoskrnl/mm/section.c | 10 +++++----- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/ntoskrnl/include/internal/mm.h b/ntoskrnl/include/internal/mm.h index 4bb6d1a60ca..a7affb0a49c 100644 --- a/ntoskrnl/include/internal/mm.h +++ b/ntoskrnl/include/internal/mm.h @@ -212,7 +212,6 @@ typedef struct _MEMORY_AREA MMVAD VadNode; ULONG Type; - ULONG Protect; ULONG Flags; BOOLEAN DeleteInProgress; ULONG Magic; diff --git a/ntoskrnl/mm/marea.c b/ntoskrnl/mm/marea.c index 3736a5a36c8..cd7b9594f1d 100644 --- a/ntoskrnl/mm/marea.c +++ b/ntoskrnl/mm/marea.c @@ -165,12 +165,13 @@ MiMakeProtectionMask( static VOID MmInsertMemoryArea( PMMSUPPORT AddressSpace, - PMEMORY_AREA marea) + PMEMORY_AREA marea, + ULONG Protect) { PEPROCESS Process = MmGetAddressSpaceOwner(AddressSpace); marea->VadNode.u.VadFlags.Spare = 1; - marea->VadNode.u.VadFlags.Protection = MiMakeProtectionMask(marea->Protect); + marea->VadNode.u.VadFlags.Protection = MiMakeProtectionMask(Protect); /* Build a lame VAD if this is a user-space allocation */ if (marea->VadNode.EndingVpn + 1 < (ULONG_PTR)MmSystemRangeStart >> PAGE_SHIFT) @@ -457,7 +458,6 @@ MmCreateMemoryArea(PMMSUPPORT AddressSpace, RtlZeroMemory(MemoryArea, sizeof(MEMORY_AREA)); MemoryArea->Type = Type & ~MEMORY_AREA_STATIC; - MemoryArea->Protect = Protect; MemoryArea->Flags = AllocationFlags; MemoryArea->Magic = 'erAM'; MemoryArea->DeleteInProgress = FALSE; @@ -478,7 +478,7 @@ MmCreateMemoryArea(PMMSUPPORT AddressSpace, MemoryArea->VadNode.StartingVpn = (ULONG_PTR)*BaseAddress >> PAGE_SHIFT; MemoryArea->VadNode.EndingVpn = ((ULONG_PTR)*BaseAddress + tmpLength - 1) >> PAGE_SHIFT; - MmInsertMemoryArea(AddressSpace, MemoryArea); + MmInsertMemoryArea(AddressSpace, MemoryArea, Protect); } else { @@ -516,7 +516,7 @@ MmCreateMemoryArea(PMMSUPPORT AddressSpace, MemoryArea->VadNode.StartingVpn = (ULONG_PTR)*BaseAddress >> PAGE_SHIFT; MemoryArea->VadNode.EndingVpn = ((ULONG_PTR)*BaseAddress + tmpLength - 1) >> PAGE_SHIFT; - MmInsertMemoryArea(AddressSpace, MemoryArea); + MmInsertMemoryArea(AddressSpace, MemoryArea, Protect); } *Result = MemoryArea; diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index 91ef5133849..59b6a22ec36 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -2222,7 +2222,7 @@ MmPageOutSectionView(PMMSUPPORT AddressSpace, { Status = MmCreateVirtualMapping(Process, Address, - MemoryArea->Protect, + MmProtectToValue[MemoryArea->VadNode.u.VadFlags.Protection], &Page, 1); MmSetDirtyPage(Process, Address); @@ -2243,7 +2243,7 @@ MmPageOutSectionView(PMMSUPPORT AddressSpace, */ Status = MmCreateVirtualMapping(Process, Address, - MemoryArea->Protect, + MmProtectToValue[MemoryArea->VadNode.u.VadFlags.Protection], &Page, 1); MmSetDirtyPage(Process, Address); @@ -2280,7 +2280,7 @@ MmPageOutSectionView(PMMSUPPORT AddressSpace, { Status = MmCreateVirtualMapping(Process, Address, - MemoryArea->Protect, + MmProtectToValue[MemoryArea->VadNode.u.VadFlags.Protection], &Page, 1); MmSetDirtyPage(Process, Address); @@ -2293,7 +2293,7 @@ MmPageOutSectionView(PMMSUPPORT AddressSpace, MmLockSectionSegment(Context.Segment); Status = MmCreateVirtualMapping(Process, Address, - MemoryArea->Protect, + MmProtectToValue[MemoryArea->VadNode.u.VadFlags.Protection], &Page, 1); MmSetDirtyPage(Process, Address); @@ -2579,7 +2579,7 @@ MmQuerySectionView(PMEMORY_AREA MemoryArea, Info->Type = MEM_MAPPED; } Info->BaseAddress = RegionBaseAddress; - Info->AllocationProtect = MemoryArea->Protect; + Info->AllocationProtect = MmProtectToValue[MemoryArea->VadNode.u.VadFlags.Protection]; Info->RegionSize = Region->Length; Info->State = MEM_COMMIT; Info->Protect = Region->Protect; From e980efebd4efe53a596cab313e825160c6bf76fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Fri, 23 Oct 2020 17:27:47 +0200 Subject: [PATCH 13/99] [NTOSKRNL] Store the MM_IMAGE_SECTION_OBJECT pointer in SECTION::Segment --- ntoskrnl/include/internal/mm.h | 2 -- ntoskrnl/mm/ARM3/section.c | 8 +++++++- ntoskrnl/mm/ARM3/sysldr.c | 20 ++++++++++---------- ntoskrnl/mm/section.c | 21 ++++++++++----------- 4 files changed, 27 insertions(+), 24 deletions(-) diff --git a/ntoskrnl/include/internal/mm.h b/ntoskrnl/include/internal/mm.h index a7affb0a49c..760b514de4a 100644 --- a/ntoskrnl/include/internal/mm.h +++ b/ntoskrnl/include/internal/mm.h @@ -200,8 +200,6 @@ typedef struct _ROS_SECTION_OBJECT { SECTION; PFILE_OBJECT FileObject; - - PMM_IMAGE_SECTION_OBJECT ImageSection; } ROS_SECTION_OBJECT, *PROS_SECTION_OBJECT; #define MA_GetStartingAddress(_MemoryArea) ((_MemoryArea)->VadNode.StartingVpn << PAGE_SHIFT) diff --git a/ntoskrnl/mm/ARM3/section.c b/ntoskrnl/mm/ARM3/section.c index 58d261e5476..81ecdc9f8dc 100644 --- a/ntoskrnl/mm/ARM3/section.c +++ b/ntoskrnl/mm/ARM3/section.c @@ -1789,8 +1789,14 @@ MmGetImageInformation (OUT PSECTION_IMAGE_INFORMATION ImageInformation) ASSERT(SectionObject != NULL); ASSERT(MiIsRosSectionObject(SectionObject) == TRUE); + if (SectionObject->u.Flags.Image == 0) + { + RtlZeroMemory(ImageInformation, sizeof(*ImageInformation)); + return; + } + /* Return the image information */ - *ImageInformation = ((PROS_SECTION_OBJECT)SectionObject)->ImageSection->ImageInformation; + *ImageInformation = ((PMM_IMAGE_SECTION_OBJECT)SectionObject->Segment)->ImageInformation; } NTSTATUS diff --git a/ntoskrnl/mm/ARM3/sysldr.c b/ntoskrnl/mm/ARM3/sysldr.c index 811defcde8b..0ebcae75e29 100644 --- a/ntoskrnl/mm/ARM3/sysldr.c +++ b/ntoskrnl/mm/ARM3/sysldr.c @@ -81,13 +81,13 @@ MiCacheImageSymbols(IN PVOID BaseAddress) NTSTATUS NTAPI -MiLoadImageSection(IN OUT PVOID *SectionPtr, - OUT PVOID *ImageBase, - IN PUNICODE_STRING FileName, - IN BOOLEAN SessionLoad, - IN PLDR_DATA_TABLE_ENTRY LdrEntry) +MiLoadImageSection(_Inout_ PSECTION *SectionPtr, + _Out_ PVOID *ImageBase, + _In_ PUNICODE_STRING FileName, + _In_ BOOLEAN SessionLoad, + _In_ PLDR_DATA_TABLE_ENTRY LdrEntry) { - PROS_SECTION_OBJECT Section = *SectionPtr; + PSECTION Section = *SectionPtr; NTSTATUS Status; PEPROCESS Process; PVOID Base = NULL; @@ -158,7 +158,7 @@ MiLoadImageSection(IN OUT PVOID *SectionPtr, } /* Reserve system PTEs needed */ - PteCount = ROUND_TO_PAGES(Section->ImageSection->ImageInformation.ImageFileSize) >> PAGE_SHIFT; + PteCount = ROUND_TO_PAGES(((PMM_IMAGE_SECTION_OBJECT)Section->Segment)->ImageInformation.ImageFileSize) >> PAGE_SHIFT; PointerPte = MiReserveSystemPtes(PteCount, SystemPteSpace); if (!PointerPte) { @@ -2837,7 +2837,7 @@ MmLoadSystemImage(IN PUNICODE_STRING FileName, PWCHAR MissingDriverName; HANDLE SectionHandle; ACCESS_MASK DesiredAccess; - PVOID Section = NULL; + PSECTION Section = NULL; BOOLEAN LockOwned = FALSE; PLIST_ENTRY NextEntry; IMAGE_INFO ImageInfo; @@ -3054,7 +3054,7 @@ LoaderScan: SECTION_MAP_EXECUTE, MmSectionObjectType, KernelMode, - &Section, + (PVOID*)&Section, NULL); ZwClose(SectionHandle); if (!NT_SUCCESS(Status)) goto Quickie; @@ -3085,7 +3085,7 @@ LoaderScan: ASSERT(Status != STATUS_ALREADY_COMMITTED); /* Get the size of the driver */ - DriverSize = ((PROS_SECTION_OBJECT)Section)->ImageSection->ImageInformation.ImageFileSize; + DriverSize = ((PMM_IMAGE_SECTION_OBJECT)Section->Segment)->ImageInformation.ImageFileSize; /* Make sure we're not being loaded into session space */ if (!Flags) diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index 59b6a22ec36..73711677d41 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -2633,7 +2633,7 @@ MmpFreePageFileSegment(PMM_SECTION_SEGMENT Segment) VOID NTAPI MmpDeleteSection(PVOID ObjectBody) { - PROS_SECTION_OBJECT Section = (PROS_SECTION_OBJECT)ObjectBody; + PROS_SECTION_OBJECT Section = ObjectBody; /* Check if it's an ARM3, or ReactOS section */ if (!MiIsRosSectionObject(Section)) @@ -2656,11 +2656,11 @@ MmpDeleteSection(PVOID ObjectBody) * until the image section is properly initialized we shouldn't * process further here. */ - if (Section->ImageSection == NULL) + if (Section->Segment == NULL) return; - SectionSegments = Section->ImageSection->Segments; - NrSegments = Section->ImageSection->NrSegments; + SectionSegments = ((PMM_IMAGE_SECTION_OBJECT)Section->Segment)->Segments; + NrSegments = ((PMM_IMAGE_SECTION_OBJECT)Section->Segment)->NrSegments; for (i = 0; i < NrSegments; i++) { @@ -3763,7 +3763,7 @@ MmCreateImageSection(PROS_SECTION_OBJECT *SectionObject, return(Status); } - Section->ImageSection = ImageSectionObject; + Section->Segment = (PSEGMENT)ImageSectionObject; ASSERT(ImageSectionObject->Segments); /* @@ -3788,7 +3788,7 @@ MmCreateImageSection(PROS_SECTION_OBJECT *SectionObject, ExFreePool(ImageSectionObject->Segments); ExFreePool(ImageSectionObject); ImageSectionObject = FileObject->SectionObjectPointer->ImageSectionObject; - Section->ImageSection = ImageSectionObject; + Section->Segment = (PSEGMENT)ImageSectionObject; SectionSegments = ImageSectionObject->Segments; for (i = 0; i < ImageSectionObject->NrSegments; i++) @@ -3813,7 +3813,7 @@ MmCreateImageSection(PROS_SECTION_OBJECT *SectionObject, } ImageSectionObject = FileObject->SectionObjectPointer->ImageSectionObject; - Section->ImageSection = ImageSectionObject; + Section->Segment = (PSEGMENT)ImageSectionObject; SectionSegments = ImageSectionObject->Segments; /* @@ -4114,7 +4114,7 @@ MiRosUnmapViewOfSection(IN PEPROCESS Process, PMM_SECTION_SEGMENT Segment; Segment = MemoryArea->SectionData.Segment; - ImageSectionObject = Section->ImageSection; + ImageSectionObject = ((PMM_IMAGE_SECTION_OBJECT)Section->Segment); SectionSegments = ImageSectionObject->Segments; NrSegments = ImageSectionObject->NrSegments; @@ -4314,8 +4314,7 @@ NtQuerySection( { if (RosSection->u.Flags.Image) { - PMM_IMAGE_SECTION_OBJECT ImageSectionObject; - ImageSectionObject = RosSection->ImageSection; + PMM_IMAGE_SECTION_OBJECT ImageSectionObject = ((PMM_IMAGE_SECTION_OBJECT)Section->Segment); *Sii = ImageSectionObject->ImageInformation; } @@ -4504,7 +4503,7 @@ MmMapViewOfSection(IN PVOID SectionObject, PMM_IMAGE_SECTION_OBJECT ImageSectionObject; PMM_SECTION_SEGMENT SectionSegments; - ImageSectionObject = Section->ImageSection; + ImageSectionObject = ((PMM_IMAGE_SECTION_OBJECT)Section->Segment); SectionSegments = ImageSectionObject->Segments; NrSegments = ImageSectionObject->NrSegments; From 8b98ff66eabba884a02993704b46174da1a0141a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Mon, 26 Oct 2020 09:04:49 +0100 Subject: [PATCH 14/99] [NTOSKRNL] Store File Object in the segment instead of Section object --- ntoskrnl/include/internal/mm.h | 6 +++-- ntoskrnl/mm/ARM3/section.c | 9 +++---- ntoskrnl/mm/section.c | 47 ++++++++++++++++++++++++---------- 3 files changed, 41 insertions(+), 21 deletions(-) diff --git a/ntoskrnl/include/internal/mm.h b/ntoskrnl/include/internal/mm.h index 760b514de4a..74cfcf7f11e 100644 --- a/ntoskrnl/include/internal/mm.h +++ b/ntoskrnl/include/internal/mm.h @@ -166,8 +166,9 @@ typedef ULONG_PTR SWAPENTRY; typedef struct _MM_SECTION_SEGMENT { + PFILE_OBJECT FileObject; + FAST_MUTEX Lock; /* lock which protects the page directory */ - PFILE_OBJECT FileObject; LARGE_INTEGER RawLength; /* length of the segment which is part of the mapped file */ LARGE_INTEGER Length; /* absolute length of the segment */ ULONG ReferenceCount; @@ -190,6 +191,8 @@ typedef struct _MM_SECTION_SEGMENT typedef struct _MM_IMAGE_SECTION_OBJECT { + PFILE_OBJECT FileObject; + SECTION_IMAGE_INFORMATION ImageInformation; PVOID BasedAddress; ULONG NrSegments; @@ -199,7 +202,6 @@ typedef struct _MM_IMAGE_SECTION_OBJECT typedef struct _ROS_SECTION_OBJECT { SECTION; - PFILE_OBJECT FileObject; } ROS_SECTION_OBJECT, *PROS_SECTION_OBJECT; #define MA_GetStartingAddress(_MemoryArea) ((_MemoryArea)->VadNode.StartingVpn << PAGE_SHIFT) diff --git a/ntoskrnl/mm/ARM3/section.c b/ntoskrnl/mm/ARM3/section.c index 81ecdc9f8dc..a91cfe2ed4a 100644 --- a/ntoskrnl/mm/ARM3/section.c +++ b/ntoskrnl/mm/ARM3/section.c @@ -1662,7 +1662,7 @@ MiGetFileObjectForSectionAddress( { /* Get the section pointer to the SECTION_OBJECT */ Section = MemoryArea->SectionData.Section; - *FileObject = Section->FileObject; + *FileObject = ((PMM_SECTION_SEGMENT)Section->Segment)->FileObject; } else { @@ -1705,7 +1705,7 @@ PFILE_OBJECT NTAPI MmGetFileObjectForSection(IN PVOID SectionObject) { - PSECTION Section; + PSECTION Section = SectionObject; ASSERT(KeGetCurrentIrql() == PASSIVE_LEVEL); ASSERT(SectionObject != NULL); @@ -1713,12 +1713,11 @@ MmGetFileObjectForSection(IN PVOID SectionObject) if (MiIsRosSectionObject(SectionObject) == FALSE) { /* Return the file pointer stored in the control area */ - Section = SectionObject; return Section->Segment->ControlArea->FilePointer; } /* Return the file object */ - return ((PROS_SECTION_OBJECT)SectionObject)->FileObject; + return ((PMM_SECTION_SEGMENT)Section->Segment)->FileObject; } static @@ -1740,7 +1739,7 @@ MiGetFileObjectForVad( { /* Get the section pointer to the SECTION_OBJECT */ Section = MemoryArea->SectionData.Section; - FileObject = Section->FileObject; + FileObject = ((PMM_SECTION_SEGMENT)Section->Segment)->FileObject; } else { diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index 73711677d41..b975433e271 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -847,6 +847,7 @@ MmFreeSectionSegments(PFILE_OBJECT FileObject) } MmFreePageTablesSectionSegment(&SectionSegments[i], NULL); } + ObDereferenceObject(ImageSectionObject->FileObject); ExFreePool(ImageSectionObject->Segments); ExFreePool(ImageSectionObject); FileObject->SectionObjectPointer->ImageSectionObject = NULL; @@ -863,6 +864,7 @@ MmFreeSectionSegments(PFILE_OBJECT FileObject) DPRINT1("Data segment still referenced\n"); KeBugCheck(MEMORY_MANAGEMENT); } + ObDereferenceObject(Segment->FileObject); MmFreePageTablesSectionSegment(Segment, NULL); ExFreePool(Segment); FileObject->SectionObjectPointer->DataSectionObject = NULL; @@ -941,7 +943,7 @@ MmUnsharePageEntrySectionSegment(PROS_SECTION_OBJECT Section, #endif Page = PFN_FROM_SSE(Entry); - FileObject = Section->FileObject; + FileObject = Segment->FileObject; if (FileObject != NULL && !(Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) { @@ -1047,7 +1049,7 @@ BOOLEAN MiIsPageFromCache(PMEMORY_AREA MemoryArea, { PROS_SHARED_CACHE_MAP SharedCacheMap; PROS_VACB Vacb; - SharedCacheMap = MemoryArea->SectionData.Section->FileObject->SectionObjectPointer->SharedCacheMap; + SharedCacheMap = MemoryArea->SectionData.Segment->FileObject->SectionObjectPointer->SharedCacheMap; Vacb = CcRosLookupVacb(SharedCacheMap, SegOffset + MemoryArea->SectionData.Segment->Image.FileOffset); if (Vacb) { @@ -1106,7 +1108,7 @@ MiReadPage(PMEMORY_AREA MemoryArea, BOOLEAN IsImageSection; LONGLONG Length; - FileObject = MemoryArea->SectionData.Section->FileObject; + FileObject = MemoryArea->SectionData.Segment->FileObject; SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; RawLength = MemoryArea->SectionData.Segment->RawLength.QuadPart; FileOffset = SegOffset + MemoryArea->SectionData.Segment->Image.FileOffset; @@ -2012,7 +2014,7 @@ MmPageOutSectionView(PMMSUPPORT AddressSpace, #ifndef NEWCC FileOffset = Context.Offset.QuadPart + Context.Segment->Image.FileOffset; IsImageSection = Context.Section->u.Flags.Image; - FileObject = Context.Section->FileObject; + FileObject = Context.Segment->FileObject; if (FileObject != NULL && !(Context.Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) @@ -2392,7 +2394,7 @@ MmWritePageSectionView(PMMSUPPORT AddressSpace, Section = MemoryArea->SectionData.Section; IsImageSection = Section->u.Flags.Image; - FileObject = Section->FileObject; + FileObject = Segment->FileObject; DirectMapped = FALSE; if (FileObject != NULL && !(Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) @@ -2709,13 +2711,16 @@ MmpDeleteSection(PVOID ObjectBody) (void)InterlockedDecrementUL(&((PMM_SECTION_SEGMENT)Section->Segment)->ReferenceCount); } - if (Section->FileObject != NULL) + + if (Section->Segment) { -#ifndef NEWCC - CcRosDereferenceCache(Section->FileObject); -#endif - ObDereferenceObject(Section->FileObject); - Section->FileObject = NULL; + PMM_SECTION_SEGMENT Segment = (PMM_SECTION_SEGMENT)Section->Segment; + if (Segment->FileObject != NULL) + { + #ifndef NEWCC + CcRosDereferenceCache(Segment->FileObject); + #endif + } } } @@ -2986,6 +2991,7 @@ MmCreateDataFileSection(PROS_SECTION_OBJECT *SectionObject, Section->Segment = (PSEGMENT)Segment; Segment->ReferenceCount = 1; ExInitializeFastMutex(&Segment->Lock); + Segment->FileObject = FileObject; /* * Set the lock before assigning the segment to the file object */ @@ -3029,9 +3035,12 @@ MmCreateDataFileSection(PROS_SECTION_OBJECT *SectionObject, Segment->RawLength.QuadPart = MaximumSize.QuadPart; Segment->Length.QuadPart = PAGE_ROUND_UP(Segment->RawLength.QuadPart); } + + /* We let the segment reference the file object */ + ObDereferenceObject(FileObject); + FileObject = Segment->FileObject; } MmUnlockSectionSegment(Segment); - Section->FileObject = FileObject; Section->SizeOfSection = MaximumSize; #ifndef NEWCC CcRosReferenceCache(FileObject); @@ -3663,8 +3672,11 @@ ExeFmtpCreateImageSection(PFILE_OBJECT FileObject, ExInitializeFastMutex(&ImageSectionObject->Segments[i].Lock); ImageSectionObject->Segments[i].ReferenceCount = 1; MiInitializeSectionPageTable(&ImageSectionObject->Segments[i]); + ImageSectionObject->Segments[i].FileObject = FileObject; } + ImageSectionObject->FileObject = FileObject; + ASSERT(NT_SUCCESS(Status)); return Status; } @@ -3795,6 +3807,10 @@ MmCreateImageSection(PROS_SECTION_OBJECT *SectionObject, { (void)InterlockedIncrementUL(&SectionSegments[i].ReferenceCount); } + + /* We let the Image Section Object hold the reference */ + ObDereferenceObject(FileObject); + FileObject = ImageSectionObject->FileObject; } Status = StatusExeFmt; @@ -3824,9 +3840,12 @@ MmCreateImageSection(PROS_SECTION_OBJECT *SectionObject, (void)InterlockedIncrementUL(&SectionSegments[i].ReferenceCount); } + /* We let the Image Section Object hold the reference */ + ObDereferenceObject(FileObject); + FileObject = ImageSectionObject->FileObject; + Status = STATUS_SUCCESS; } - Section->FileObject = FileObject; #ifndef NEWCC CcRosReferenceCache(FileObject); #endif @@ -3963,7 +3982,7 @@ MmFreeSectionPage(PVOID Context, MEMORY_AREA* MemoryArea, PVOID Address, if (Page == PFN_FROM_SSE(Entry) && Dirty) { #ifndef NEWCC - FileObject = MemoryArea->SectionData.Section->FileObject; + FileObject = MemoryArea->SectionData.Segment->FileObject; SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; CcRosMarkDirtyFile(SharedCacheMap, Offset.QuadPart + Segment->Image.FileOffset); #endif From b7d988ae5b4948b0ab4be8a094c3b5536fdd187c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Mon, 26 Oct 2020 10:31:46 +0100 Subject: [PATCH 15/99] [NTOSKRNL] Get rid of MM_ROS_SECTION_OBJECT --- ntoskrnl/cache/newcc.h | 2 +- ntoskrnl/cache/section/newmm.h | 4 +- ntoskrnl/include/internal/mm.h | 7 +--- ntoskrnl/mm/ARM3/section.c | 10 ++--- ntoskrnl/mm/section.c | 74 +++++++++++++++++----------------- 5 files changed, 43 insertions(+), 54 deletions(-) diff --git a/ntoskrnl/cache/newcc.h b/ntoskrnl/cache/newcc.h index adbf1c5d926..8768d7aebd2 100644 --- a/ntoskrnl/cache/newcc.h +++ b/ntoskrnl/cache/newcc.h @@ -6,7 +6,7 @@ typedef struct _NOCC_BCB PUBLIC_BCB Bcb; struct _NOCC_CACHE_MAP *Map; - PROS_SECTION_OBJECT SectionObject; + PSECTION SectionObject; LARGE_INTEGER FileOffset; ULONG Length; PVOID BaseAddress; diff --git a/ntoskrnl/cache/section/newmm.h b/ntoskrnl/cache/section/newmm.h index d5f9998993c..edff99f3d85 100644 --- a/ntoskrnl/cache/section/newmm.h +++ b/ntoskrnl/cache/section/newmm.h @@ -96,7 +96,7 @@ typedef struct _MM_REQUIRED_RESOURCES NTSTATUS NTAPI -MmCreateCacheSection(PROS_SECTION_OBJECT *SectionObject, +MmCreateCacheSection(PSECTION *SectionObject, ACCESS_MASK DesiredAccess, POBJECT_ATTRIBUTES ObjectAttributes, PLARGE_INTEGER UMaximumSize, @@ -386,7 +386,7 @@ MiSwapInSectionPage(PMMSUPPORT AddressSpace, NTSTATUS NTAPI -MmExtendCacheSection(PROS_SECTION_OBJECT Section, +MmExtendCacheSection(PSECTION Section, PLARGE_INTEGER NewSize, BOOLEAN ExtendFile); diff --git a/ntoskrnl/include/internal/mm.h b/ntoskrnl/include/internal/mm.h index 74cfcf7f11e..a8536c3d786 100644 --- a/ntoskrnl/include/internal/mm.h +++ b/ntoskrnl/include/internal/mm.h @@ -199,11 +199,6 @@ typedef struct _MM_IMAGE_SECTION_OBJECT PMM_SECTION_SEGMENT Segments; } MM_IMAGE_SECTION_OBJECT, *PMM_IMAGE_SECTION_OBJECT; -typedef struct _ROS_SECTION_OBJECT -{ - SECTION; -} ROS_SECTION_OBJECT, *PROS_SECTION_OBJECT; - #define MA_GetStartingAddress(_MemoryArea) ((_MemoryArea)->VadNode.StartingVpn << PAGE_SHIFT) #define MA_GetEndingAddress(_MemoryArea) (((_MemoryArea)->VadNode.EndingVpn + 1) << PAGE_SHIFT) @@ -219,7 +214,7 @@ typedef struct _MEMORY_AREA struct { - PROS_SECTION_OBJECT Section; + PSECTION Section; LARGE_INTEGER ViewOffset; PMM_SECTION_SEGMENT Segment; LIST_ENTRY RegionListHead; diff --git a/ntoskrnl/mm/ARM3/section.c b/ntoskrnl/mm/ARM3/section.c index a91cfe2ed4a..73c0e3050aa 100644 --- a/ntoskrnl/mm/ARM3/section.c +++ b/ntoskrnl/mm/ARM3/section.c @@ -1655,14 +1655,12 @@ MiGetFileObjectForSectionAddress( if (Vad->u.VadFlags.Spare != 0) { PMEMORY_AREA MemoryArea = (PMEMORY_AREA)Vad; - PROS_SECTION_OBJECT Section; /* Check if it's a section view (RosMm section) */ if (MemoryArea->Type == MEMORY_AREA_SECTION_VIEW) { /* Get the section pointer to the SECTION_OBJECT */ - Section = MemoryArea->SectionData.Section; - *FileObject = ((PMM_SECTION_SEGMENT)Section->Segment)->FileObject; + *FileObject = MemoryArea->SectionData.Segment->FileObject; } else { @@ -1732,14 +1730,12 @@ MiGetFileObjectForVad( if (Vad->u.VadFlags.Spare != 0) { PMEMORY_AREA MemoryArea = (PMEMORY_AREA)Vad; - PROS_SECTION_OBJECT Section; /* Check if it's a section view (RosMm section) */ if (MemoryArea->Type == MEMORY_AREA_SECTION_VIEW) { /* Get the section pointer to the SECTION_OBJECT */ - Section = MemoryArea->SectionData.Section; - FileObject = ((PMM_SECTION_SEGMENT)Section->Segment)->FileObject; + FileObject = MemoryArea->SectionData.Segment->FileObject; } else { @@ -3840,7 +3836,7 @@ NtExtendSection(IN HANDLE SectionHandle, IN OUT PLARGE_INTEGER NewMaximumSize) { LARGE_INTEGER SafeNewMaximumSize; - PROS_SECTION_OBJECT Section; + PSECTION Section; NTSTATUS Status; KPROCESSOR_MODE PreviousMode = ExGetPreviousMode(); diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index b975433e271..f43811e9f50 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -138,7 +138,7 @@ C_ASSERT(PEFMT_FIELDS_EQUAL(IMAGE_OPTIONAL_HEADER32, IMAGE_OPTIONAL_HEADER64, Si typedef struct { - PROS_SECTION_OBJECT Section; + PSECTION Section; PMM_SECTION_SEGMENT Segment; LARGE_INTEGER Offset; BOOLEAN WasDirty; @@ -899,7 +899,7 @@ MmSharePageEntrySectionSegment(PMM_SECTION_SEGMENT Segment, BOOLEAN NTAPI -MmUnsharePageEntrySectionSegment(PROS_SECTION_OBJECT Section, +MmUnsharePageEntrySectionSegment(PSECTION Section, PMM_SECTION_SEGMENT Segment, PLARGE_INTEGER Offset, BOOLEAN Dirty, @@ -1389,7 +1389,7 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, LARGE_INTEGER Offset; PFN_NUMBER Page; NTSTATUS Status; - PROS_SECTION_OBJECT Section; + PSECTION Section; PMM_SECTION_SEGMENT Segment; ULONG_PTR Entry; ULONG_PTR Entry1; @@ -1812,7 +1812,7 @@ MmAccessFaultSectionView(PMMSUPPORT AddressSpace, PVOID Address) { PMM_SECTION_SEGMENT Segment; - PROS_SECTION_OBJECT Section; + PSECTION Section; PFN_NUMBER OldPage; PFN_NUMBER NewPage; NTSTATUS Status; @@ -1955,7 +1955,7 @@ MmPageOutDeleteMapping(PVOID Context, PEPROCESS Process, PVOID Address) if (!PageOutContext->Private) { MmLockSectionSegment(PageOutContext->Segment); - MmUnsharePageEntrySectionSegment((PROS_SECTION_OBJECT)PageOutContext->Section, + MmUnsharePageEntrySectionSegment(PageOutContext->Section, PageOutContext->Segment, &PageOutContext->Offset, PageOutContext->WasDirty, @@ -2367,7 +2367,7 @@ MmWritePageSectionView(PMMSUPPORT AddressSpace, ULONG PageEntry) { LARGE_INTEGER Offset; - PROS_SECTION_OBJECT Section; + PSECTION Section; PMM_SECTION_SEGMENT Segment; PFN_NUMBER Page; SWAPENTRY SwapEntry; @@ -2557,7 +2557,7 @@ MmQuerySectionView(PMEMORY_AREA MemoryArea, { PMM_REGION Region; PVOID RegionBaseAddress; - PROS_SECTION_OBJECT Section; + PSECTION Section; PMM_SECTION_SEGMENT Segment; Region = MmFindRegion((PVOID)MA_GetStartingAddress(MemoryArea), @@ -2635,7 +2635,7 @@ MmpFreePageFileSegment(PMM_SECTION_SEGMENT Segment) VOID NTAPI MmpDeleteSection(PVOID ObjectBody) { - PROS_SECTION_OBJECT Section = ObjectBody; + PSECTION Section = ObjectBody; /* Check if it's an ARM3, or ReactOS section */ if (!MiIsRosSectionObject(Section)) @@ -2739,7 +2739,7 @@ NTSTATUS NTAPI MmCreatePhysicalMemorySection(VOID) { - PROS_SECTION_OBJECT PhysSection; + PSECTION PhysSection; NTSTATUS Status; OBJECT_ATTRIBUTES Obj; UNICODE_STRING Name = RTL_CONSTANT_STRING(L"\\Device\\PhysicalMemory"); @@ -2764,7 +2764,7 @@ MmCreatePhysicalMemorySection(VOID) &Obj, ExGetPreviousMode(), NULL, - sizeof(ROS_SECTION_OBJECT), + sizeof(*PhysSection), 0, 0, (PVOID*)&PhysSection); @@ -2777,7 +2777,7 @@ MmCreatePhysicalMemorySection(VOID) /* * Initialize it */ - RtlZeroMemory(PhysSection, sizeof(ROS_SECTION_OBJECT)); + RtlZeroMemory(PhysSection, sizeof(*PhysSection)); /* Mark this as a "ROS Section" */ PhysSection->u.Flags.filler = 1; @@ -2838,7 +2838,7 @@ MmInitSectionImplementation(VOID) RtlZeroMemory(&ObjectTypeInitializer, sizeof(ObjectTypeInitializer)); RtlInitUnicodeString(&Name, L"Section"); ObjectTypeInitializer.Length = sizeof(ObjectTypeInitializer); - ObjectTypeInitializer.DefaultPagedPoolCharge = sizeof(ROS_SECTION_OBJECT); + ObjectTypeInitializer.DefaultPagedPoolCharge = sizeof(SECTION); ObjectTypeInitializer.PoolType = PagedPool; ObjectTypeInitializer.UseDefaultObject = TRUE; ObjectTypeInitializer.GenericMapping = MmpSectionMapping; @@ -2855,7 +2855,7 @@ MmInitSectionImplementation(VOID) NTSTATUS NTAPI -MmCreateDataFileSection(PROS_SECTION_OBJECT *SectionObject, +MmCreateDataFileSection(PSECTION *SectionObject, ACCESS_MASK DesiredAccess, POBJECT_ATTRIBUTES ObjectAttributes, PLARGE_INTEGER UMaximumSize, @@ -2866,7 +2866,7 @@ MmCreateDataFileSection(PROS_SECTION_OBJECT *SectionObject, * Create a section backed by a data file */ { - PROS_SECTION_OBJECT Section; + PSECTION Section; NTSTATUS Status; LARGE_INTEGER MaximumSize; PMM_SECTION_SEGMENT Segment; @@ -2881,7 +2881,7 @@ MmCreateDataFileSection(PROS_SECTION_OBJECT *SectionObject, ObjectAttributes, ExGetPreviousMode(), NULL, - sizeof(ROS_SECTION_OBJECT), + sizeof(*Section), 0, 0, (PVOID*)&Section); @@ -2893,7 +2893,7 @@ MmCreateDataFileSection(PROS_SECTION_OBJECT *SectionObject, /* * Initialize it */ - RtlZeroMemory(Section, sizeof(ROS_SECTION_OBJECT)); + RtlZeroMemory(Section, sizeof(*Section)); /* Mark this as a "ROS" section */ Section->u.Flags.filler = 1; @@ -3682,7 +3682,7 @@ ExeFmtpCreateImageSection(PFILE_OBJECT FileObject, } NTSTATUS -MmCreateImageSection(PROS_SECTION_OBJECT *SectionObject, +MmCreateImageSection(PSECTION *SectionObject, ACCESS_MASK DesiredAccess, POBJECT_ATTRIBUTES ObjectAttributes, PLARGE_INTEGER UMaximumSize, @@ -3690,7 +3690,7 @@ MmCreateImageSection(PROS_SECTION_OBJECT *SectionObject, ULONG AllocationAttributes, PFILE_OBJECT FileObject) { - PROS_SECTION_OBJECT Section; + PSECTION Section; NTSTATUS Status; PMM_SECTION_SEGMENT SectionSegments; PMM_IMAGE_SECTION_OBJECT ImageSectionObject; @@ -3715,7 +3715,7 @@ MmCreateImageSection(PROS_SECTION_OBJECT *SectionObject, ObjectAttributes, ExGetPreviousMode(), NULL, - sizeof(ROS_SECTION_OBJECT), + sizeof(*Section), 0, 0, (PVOID*)(PVOID)&Section); @@ -3728,7 +3728,7 @@ MmCreateImageSection(PROS_SECTION_OBJECT *SectionObject, /* * Initialize it */ - RtlZeroMemory(Section, sizeof(ROS_SECTION_OBJECT)); + RtlZeroMemory(Section, sizeof(*Section)); /* Mark this as a "ROS" Section */ Section->u.Flags.filler = 1; @@ -3858,7 +3858,7 @@ MmCreateImageSection(PROS_SECTION_OBJECT *SectionObject, static NTSTATUS MmMapViewOfSegment(PMMSUPPORT AddressSpace, - PROS_SECTION_OBJECT Section, + PSECTION Section, PMM_SECTION_SEGMENT Segment, PVOID* BaseAddress, SIZE_T ViewSize, @@ -3944,7 +3944,7 @@ MmFreeSectionPage(PVOID Context, MEMORY_AREA* MemoryArea, PVOID Address, #endif LARGE_INTEGER Offset; SWAPENTRY SavedSwapEntry; - PROS_SECTION_OBJECT Section; + PSECTION Section; PMM_SECTION_SEGMENT Segment; PMMSUPPORT AddressSpace; PEPROCESS Process; @@ -4028,7 +4028,7 @@ MmUnmapViewOfSegment(PMMSUPPORT AddressSpace, { NTSTATUS Status; PMEMORY_AREA MemoryArea; - PROS_SECTION_OBJECT Section; + PSECTION Section; PMM_SECTION_SEGMENT Segment; PLIST_ENTRY CurrentEntry; PMM_REGION CurrentRegion; @@ -4095,7 +4095,7 @@ MiRosUnmapViewOfSection(IN PEPROCESS Process, NTSTATUS Status; PMEMORY_AREA MemoryArea; PMMSUPPORT AddressSpace; - PROS_SECTION_OBJECT Section; + PSECTION Section; PVOID ImageBaseAddress = 0; DPRINT("Opening memory area Process %p BaseAddress %p\n", @@ -4281,8 +4281,6 @@ NtQuerySection( if (MiIsRosSectionObject(Section)) { - PROS_SECTION_OBJECT RosSection = (PROS_SECTION_OBJECT)Section; - switch (SectionInformationClass) { case SectionBasicInformation: @@ -4292,22 +4290,22 @@ NtQuerySection( _SEH2_TRY { Sbi->Attributes = 0; - if (RosSection->u.Flags.Image) + if (Section->u.Flags.Image) Sbi->Attributes |= SEC_IMAGE; - if (RosSection->u.Flags.File) + if (Section->u.Flags.File) Sbi->Attributes |= SEC_FILE; - if (RosSection->u.Flags.NoChange) + if (Section->u.Flags.NoChange) Sbi->Attributes |= SEC_NO_CHANGE; - if (RosSection->u.Flags.Image) + if (Section->u.Flags.Image) { Sbi->BaseAddress = 0; Sbi->Size.QuadPart = 0; } else { - Sbi->BaseAddress = (PVOID)((PMM_SECTION_SEGMENT)RosSection->Segment)->Image.VirtualAddress; - Sbi->Size.QuadPart = ((PMM_SECTION_SEGMENT)RosSection->Segment)->Length.QuadPart; + Sbi->BaseAddress = (PVOID)((PMM_SECTION_SEGMENT)Section->Segment)->Image.VirtualAddress; + Sbi->Size.QuadPart = ((PMM_SECTION_SEGMENT)Section->Segment)->Length.QuadPart; } if (ResultLength != NULL) @@ -4331,7 +4329,7 @@ NtQuerySection( _SEH2_TRY { - if (RosSection->u.Flags.Image) + if (Section->u.Flags.Image) { PMM_IMAGE_SECTION_OBJECT ImageSectionObject = ((PMM_IMAGE_SECTION_OBJECT)Section->Segment); @@ -4474,7 +4472,7 @@ MmMapViewOfSection(IN PVOID SectionObject, IN ULONG AllocationType, IN ULONG Protect) { - PROS_SECTION_OBJECT Section; + PSECTION Section; PMMSUPPORT AddressSpace; ULONG ViewOffset; NTSTATUS Status = STATUS_SUCCESS; @@ -4505,7 +4503,7 @@ MmMapViewOfSection(IN PVOID SectionObject, /* FIXME: We should keep this, but it would break code checking equality */ Protect &= ~PAGE_NOCACHE; - Section = (PROS_SECTION_OBJECT)SectionObject; + Section = SectionObject; AddressSpace = &Process->Vm; if (Section->u.Flags.NoChange) @@ -4815,7 +4813,7 @@ MmMapViewInSystemSpace (IN PVOID SectionObject, OUT PVOID * MappedBase, IN OUT PSIZE_T ViewSize) { - PROS_SECTION_OBJECT Section; + PSECTION Section; PMM_SECTION_SEGMENT Segment; PMMSUPPORT AddressSpace; NTSTATUS Status; @@ -4831,7 +4829,7 @@ MmMapViewInSystemSpace (IN PVOID SectionObject, DPRINT("MmMapViewInSystemSpace() called\n"); - Section = (PROS_SECTION_OBJECT)SectionObject; + Section = SectionObject; Segment = (PMM_SECTION_SEGMENT)Section->Segment; AddressSpace = MmGetKernelAddressSpace(); @@ -4953,7 +4951,7 @@ MmCreateSection (OUT PVOID * Section, { NTSTATUS Status; ULONG Protection; - PROS_SECTION_OBJECT *SectionObject = (PROS_SECTION_OBJECT *)Section; + PSECTION *SectionObject = (PSECTION *)Section; /* Check if an ARM3 section is being created instead */ if (!(AllocationAttributes & (SEC_IMAGE | SEC_PHYSICALMEMORY))) From 4abda863ce273a58ddcd97ecf48ab142b2b11c63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Mon, 26 Oct 2020 11:23:42 +0100 Subject: [PATCH 16/99] [NTOSKRNL/MM] Reduce use of MiIsRosSectionObject --- ntoskrnl/mm/ARM3/section.c | 5 +- ntoskrnl/mm/section.c | 163 +++++++++++++++---------------------- 2 files changed, 67 insertions(+), 101 deletions(-) diff --git a/ntoskrnl/mm/ARM3/section.c b/ntoskrnl/mm/ARM3/section.c index 73c0e3050aa..511f11a5101 100644 --- a/ntoskrnl/mm/ARM3/section.c +++ b/ntoskrnl/mm/ARM3/section.c @@ -3716,7 +3716,7 @@ NtMapViewOfSection(IN HANDLE SectionHandle, return Status; } - if (MiIsRosSectionObject(Section) && Section->u.Flags.PhysicalMemory) + if (Section->u.Flags.PhysicalMemory) { if (PreviousMode == UserMode && SafeSectionOffset.QuadPart + SafeViewSize > MmHighestPhysicalPage << PAGE_SHIFT) @@ -3764,8 +3764,7 @@ NtMapViewOfSection(IN HANDLE SectionHandle, if (NT_SUCCESS(Status)) { /* Check if this is an image for the current process */ - if (MiIsRosSectionObject(Section) && - (Section->u.Flags.Image) && + if ((Section->u.Flags.Image) && (Process == PsGetCurrentProcess()) && (Status != STATUS_IMAGE_NOT_AT_BASE)) { diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index f43811e9f50..5dd758f865d 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -4279,131 +4279,98 @@ NtQuerySection( return Status; } - if (MiIsRosSectionObject(Section)) + switch(SectionInformationClass) { - switch (SectionInformationClass) + case SectionBasicInformation: { - case SectionBasicInformation: + SECTION_BASIC_INFORMATION Sbi; + + Sbi.Size = Section->SizeOfSection; + Sbi.BaseAddress = (PVOID)Section->Address.StartingVpn; + + Sbi.Attributes = 0; + if (Section->u.Flags.Commit) + Sbi.Attributes |= SEC_COMMIT; + if (Section->u.Flags.Reserve) + Sbi.Attributes |= SEC_RESERVE; + if (Section->u.Flags.File) + Sbi.Attributes |= SEC_FILE; + if (Section->u.Flags.Image) + Sbi.Attributes |= SEC_IMAGE; + + /* FIXME : Complete/test the list of flags passed back from NtCreateSection */ + + if (Section->u.Flags.Image) { - PSECTION_BASIC_INFORMATION Sbi = (PSECTION_BASIC_INFORMATION)SectionInformation; - - _SEH2_TRY - { - Sbi->Attributes = 0; - if (Section->u.Flags.Image) - Sbi->Attributes |= SEC_IMAGE; - if (Section->u.Flags.File) - Sbi->Attributes |= SEC_FILE; - if (Section->u.Flags.NoChange) - Sbi->Attributes |= SEC_NO_CHANGE; - - if (Section->u.Flags.Image) - { - Sbi->BaseAddress = 0; - Sbi->Size.QuadPart = 0; - } - else - { - Sbi->BaseAddress = (PVOID)((PMM_SECTION_SEGMENT)Section->Segment)->Image.VirtualAddress; - Sbi->Size.QuadPart = ((PMM_SECTION_SEGMENT)Section->Segment)->Length.QuadPart; - } - - if (ResultLength != NULL) - { - *ResultLength = sizeof(SECTION_BASIC_INFORMATION); - } - Status = STATUS_SUCCESS; - } - _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER) - { - Status = _SEH2_GetExceptionCode(); - } - _SEH2_END; - - break; + Sbi.BaseAddress = 0; + Sbi.Size.QuadPart = 0; + } + else if (MiIsRosSectionObject(Section)) + { + Sbi.BaseAddress = (PVOID)((PMM_SECTION_SEGMENT)Section->Segment)->Image.VirtualAddress; + Sbi.Size.QuadPart = ((PMM_SECTION_SEGMENT)Section->Segment)->Length.QuadPart; + } + else + { + DPRINT1("Unimplemented code path!"); } - case SectionImageInformation: + _SEH2_TRY { - PSECTION_IMAGE_INFORMATION Sii = (PSECTION_IMAGE_INFORMATION)SectionInformation; - - _SEH2_TRY - { - if (Section->u.Flags.Image) - { - PMM_IMAGE_SECTION_OBJECT ImageSectionObject = ((PMM_IMAGE_SECTION_OBJECT)Section->Segment); - - *Sii = ImageSectionObject->ImageInformation; - } - - if (ResultLength != NULL) - { - *ResultLength = sizeof(SECTION_IMAGE_INFORMATION); - } - Status = STATUS_SUCCESS; - } - _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER) - { - Status = _SEH2_GetExceptionCode(); - } - _SEH2_END; - - break; + *((SECTION_BASIC_INFORMATION*)SectionInformation) = Sbi; + if (ResultLength) + *ResultLength = sizeof(Sbi); } + _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER) + { + Status = _SEH2_GetExceptionCode(); + } + _SEH2_END; + break; } - } - else - { - switch(SectionInformationClass) + case SectionImageInformation: { - case SectionBasicInformation: + if (!Section->u.Flags.Image) { - SECTION_BASIC_INFORMATION Sbi; - - Sbi.Size = Section->SizeOfSection; - Sbi.BaseAddress = (PVOID)Section->Address.StartingVpn; - - Sbi.Attributes = 0; - if (Section->u.Flags.Image) - Sbi.Attributes |= SEC_IMAGE; - if (Section->u.Flags.Commit) - Sbi.Attributes |= SEC_COMMIT; - if (Section->u.Flags.Reserve) - Sbi.Attributes |= SEC_RESERVE; - if (Section->u.Flags.File) - Sbi.Attributes |= SEC_FILE; - if (Section->u.Flags.Image) - Sbi.Attributes |= SEC_IMAGE; - - /* FIXME : Complete/test the list of flags passed back from NtCreateSection */ + Status = STATUS_SECTION_NOT_IMAGE; + } + else if (MiIsRosSectionObject(Section)) + { + PMM_IMAGE_SECTION_OBJECT ImageSectionObject = ((PMM_IMAGE_SECTION_OBJECT)Section->Segment); _SEH2_TRY { - *((SECTION_BASIC_INFORMATION*)SectionInformation) = Sbi; - if (ResultLength) - *ResultLength = sizeof(Sbi); + PSECTION_IMAGE_INFORMATION Sii = (PSECTION_IMAGE_INFORMATION)SectionInformation; + *Sii = ImageSectionObject->ImageInformation; + if (ResultLength != NULL) + *ResultLength = sizeof(*Sii); } _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER) { Status = _SEH2_GetExceptionCode(); } _SEH2_END; - break; } - case SectionImageInformation: + else { - if (!Section->u.Flags.Image) + _SEH2_TRY { - Status = STATUS_SECTION_NOT_IMAGE; + PSECTION_IMAGE_INFORMATION Sii = (PSECTION_IMAGE_INFORMATION)SectionInformation; + *Sii = *Section->Segment->u2.ImageInformation; + if (ResultLength != NULL) + *ResultLength = sizeof(*Sii); } - else + _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER) { - /* Currently not supported */ - ASSERT(FALSE); + Status = _SEH2_GetExceptionCode(); } - break; + _SEH2_END; } + break; } + default: + DPRINT1("Unknown SectionInformationClass: %d\n", SectionInformationClass); + Status = STATUS_NOT_SUPPORTED; } ObDereferenceObject(Section); From c3bd70cfd15772877629fd53df547402cde572f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Mon, 26 Oct 2020 12:19:18 +0100 Subject: [PATCH 17/99] [NTOSKRNL/MM] Fix a few wine tests --- ntoskrnl/mm/section.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index 5dd758f865d..f65c5c1ff64 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -4302,13 +4302,22 @@ NtQuerySection( if (Section->u.Flags.Image) { - Sbi.BaseAddress = 0; - Sbi.Size.QuadPart = 0; + if (MiIsRosSectionObject(Section)) + { + PMM_IMAGE_SECTION_OBJECT ImageSectionObject = ((PMM_IMAGE_SECTION_OBJECT)Section->Segment); + Sbi.BaseAddress = 0; + Sbi.Size.QuadPart = ImageSectionObject->ImageInformation.ImageFileSize; + } + else + { + /* Not supported yet */ + ASSERT(FALSE); + } } else if (MiIsRosSectionObject(Section)) { Sbi.BaseAddress = (PVOID)((PMM_SECTION_SEGMENT)Section->Segment)->Image.VirtualAddress; - Sbi.Size.QuadPart = ((PMM_SECTION_SEGMENT)Section->Segment)->Length.QuadPart; + Sbi.Size.QuadPart = ((PMM_SECTION_SEGMENT)Section->Segment)->RawLength.QuadPart; } else { From 4f6fd6c42b4fa712dad5cffc9522e801a38279c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Mon, 26 Oct 2020 17:49:16 +0100 Subject: [PATCH 18/99] [NTOS/MM] Infer that the Memory Area is an image mapping from its VAD node --- ntoskrnl/mm/section.c | 57 +++++++++---------------------------------- 1 file changed, 12 insertions(+), 45 deletions(-) diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index f65c5c1ff64..04cba04d612 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -138,7 +138,7 @@ C_ASSERT(PEFMT_FIELDS_EQUAL(IMAGE_OPTIONAL_HEADER32, IMAGE_OPTIONAL_HEADER64, Si typedef struct { - PSECTION Section; + PMEMORY_AREA MemoryArea; PMM_SECTION_SEGMENT Segment; LARGE_INTEGER Offset; BOOLEAN WasDirty; @@ -899,7 +899,7 @@ MmSharePageEntrySectionSegment(PMM_SECTION_SEGMENT Segment, BOOLEAN NTAPI -MmUnsharePageEntrySectionSegment(PSECTION Section, +MmUnsharePageEntrySectionSegment(PMEMORY_AREA MemoryArea, PMM_SECTION_SEGMENT Segment, PLARGE_INTEGER Offset, BOOLEAN Dirty, @@ -939,7 +939,7 @@ MmUnsharePageEntrySectionSegment(PSECTION Section, LARGE_INTEGER FileOffset; FileOffset.QuadPart = Offset->QuadPart + Segment->Image.FileOffset; - IsImageSection = Section->u.Flags.Image; + IsImageSection = MemoryArea->VadNode.u.VadFlags.VadType == VadImageMap; #endif Page = PFN_FROM_SSE(Entry); @@ -1112,7 +1112,7 @@ MiReadPage(PMEMORY_AREA MemoryArea, SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; RawLength = MemoryArea->SectionData.Segment->RawLength.QuadPart; FileOffset = SegOffset + MemoryArea->SectionData.Segment->Image.FileOffset; - IsImageSection = MemoryArea->SectionData.Section->u.Flags.Image; + IsImageSection = MemoryArea->VadNode.u.VadFlags.VadType == VadImageMap; ASSERT(SharedCacheMap); @@ -1631,7 +1631,7 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, MmCreatePageFileMapping(Process, PAddress, MM_WAIT_ENTRY); MmUnlockAddressSpace(AddressSpace); - if ((Offset.QuadPart >= (LONGLONG)PAGE_ROUND_UP(Segment->RawLength.QuadPart)) && Section->u.Flags.Image) + if ((Offset.QuadPart >= (LONGLONG)PAGE_ROUND_UP(Segment->RawLength.QuadPart)) && (MemoryArea->VadNode.u.VadFlags.VadType == VadImageMap)) { MI_SET_USAGE(MI_USAGE_SECTION); if (Process) MI_SET_PROCESS2(Process->ImageFileName); @@ -1812,7 +1812,6 @@ MmAccessFaultSectionView(PMMSUPPORT AddressSpace, PVOID Address) { PMM_SECTION_SEGMENT Segment; - PSECTION Section; PFN_NUMBER OldPage; PFN_NUMBER NewPage; NTSTATUS Status; @@ -1849,7 +1848,6 @@ MmAccessFaultSectionView(PMMSUPPORT AddressSpace, + MemoryArea->SectionData.ViewOffset.QuadPart; Segment = MemoryArea->SectionData.Segment; - Section = MemoryArea->SectionData.Section; Region = MmFindRegion((PVOID)MA_GetStartingAddress(MemoryArea), &MemoryArea->SectionData.RegionListHead, Address, NULL); @@ -1907,7 +1905,7 @@ MmAccessFaultSectionView(PMMSUPPORT AddressSpace, DPRINT("Swapping page (Old %x New %x)\n", OldPage, NewPage); MmDeleteVirtualMapping(Process, PAddress, NULL, NULL); MmDeleteRmap(OldPage, Process, PAddress); - MmUnsharePageEntrySectionSegment(Section, Segment, &Offset, FALSE, FALSE, NULL); + MmUnsharePageEntrySectionSegment(MemoryArea, Segment, &Offset, FALSE, FALSE, NULL); MmUnlockSectionSegment(Segment); /* @@ -1955,7 +1953,7 @@ MmPageOutDeleteMapping(PVOID Context, PEPROCESS Process, PVOID Address) if (!PageOutContext->Private) { MmLockSectionSegment(PageOutContext->Segment); - MmUnsharePageEntrySectionSegment(PageOutContext->Section, + MmUnsharePageEntrySectionSegment(PageOutContext->MemoryArea, PageOutContext->Segment, &PageOutContext->Offset, PageOutContext->WasDirty, @@ -2000,7 +1998,7 @@ MmPageOutSectionView(PMMSUPPORT AddressSpace, * Get the segment and section. */ Context.Segment = MemoryArea->SectionData.Segment; - Context.Section = MemoryArea->SectionData.Section; + Context.MemoryArea = MemoryArea; Context.SectionEntry = Entry; Context.CallingProcess = Process; @@ -2013,7 +2011,7 @@ MmPageOutSectionView(PMMSUPPORT AddressSpace, #ifndef NEWCC FileOffset = Context.Offset.QuadPart + Context.Segment->Image.FileOffset; - IsImageSection = Context.Section->u.Flags.Image; + IsImageSection = MemoryArea->VadNode.u.VadFlags.VadType == VadImageMap; FileObject = Context.Segment->FileObject; if (FileObject != NULL && @@ -2034,19 +2032,6 @@ MmPageOutSectionView(PMMSUPPORT AddressSpace, } #endif - - /* - * This should never happen since mappings of physical memory are never - * placed in the rmap lists. - */ - if (Context.Section->u.Flags.PhysicalMemory) - { - DPRINT1("Trying to page out from physical memory section address 0x%p " - "process %p\n", Address, - Process ? Process->UniqueProcessId : 0); - KeBugCheck(MEMORY_MANAGEMENT); - } - /* * Get the section segment entry and the physical address. */ @@ -2367,7 +2352,6 @@ MmWritePageSectionView(PMMSUPPORT AddressSpace, ULONG PageEntry) { LARGE_INTEGER Offset; - PSECTION Section; PMM_SECTION_SEGMENT Segment; PFN_NUMBER Page; SWAPENTRY SwapEntry; @@ -2391,8 +2375,7 @@ MmWritePageSectionView(PMMSUPPORT AddressSpace, * Get the segment and section. */ Segment = MemoryArea->SectionData.Segment; - Section = MemoryArea->SectionData.Section; - IsImageSection = Section->u.Flags.Image; + IsImageSection = MemoryArea->VadNode.u.VadFlags.VadType == VadImageMap; FileObject = Segment->FileObject; DirectMapped = FALSE; @@ -2415,18 +2398,6 @@ MmWritePageSectionView(PMMSUPPORT AddressSpace, } } - /* - * This should never happen since mappings of physical memory are never - * placed in the rmap lists. - */ - if (Section->u.Flags.PhysicalMemory) - { - DPRINT1("Trying to write back page from physical memory mapped at %p " - "process %p\n", Address, - Process ? Process->UniqueProcessId : 0); - KeBugCheck(MEMORY_MANAGEMENT); - } - /* * Get the section segment entry and the physical address. */ @@ -2557,7 +2528,6 @@ MmQuerySectionView(PMEMORY_AREA MemoryArea, { PMM_REGION Region; PVOID RegionBaseAddress; - PSECTION Section; PMM_SECTION_SEGMENT Segment; Region = MmFindRegion((PVOID)MA_GetStartingAddress(MemoryArea), @@ -2568,8 +2538,7 @@ MmQuerySectionView(PMEMORY_AREA MemoryArea, return STATUS_UNSUCCESSFUL; } - Section = MemoryArea->SectionData.Section; - if (Section->u.Flags.Image) + if (MemoryArea->VadNode.u.VadFlags.VadType == VadImageMap) { Segment = MemoryArea->SectionData.Segment; Info->AllocationBase = (PUCHAR)MA_GetStartingAddress(MemoryArea) - Segment->Image.VirtualAddress; @@ -3944,7 +3913,6 @@ MmFreeSectionPage(PVOID Context, MEMORY_AREA* MemoryArea, PVOID Address, #endif LARGE_INTEGER Offset; SWAPENTRY SavedSwapEntry; - PSECTION Section; PMM_SECTION_SEGMENT Segment; PMMSUPPORT AddressSpace; PEPROCESS Process; @@ -3957,7 +3925,6 @@ MmFreeSectionPage(PVOID Context, MEMORY_AREA* MemoryArea, PVOID Address, Offset.QuadPart = ((ULONG_PTR)Address - MA_GetStartingAddress(MemoryArea)) + MemoryArea->SectionData.ViewOffset.QuadPart; - Section = MemoryArea->SectionData.Section; Segment = MemoryArea->SectionData.Segment; Entry = MmGetPageEntrySectionSegment(Segment, &Offset); @@ -4017,7 +3984,7 @@ MmFreeSectionPage(PVOID Context, MEMORY_AREA* MemoryArea, PVOID Address, else { MmDeleteRmap(Page, Process, Address); - MmUnsharePageEntrySectionSegment(Section, Segment, &Offset, Dirty, FALSE, NULL); + MmUnsharePageEntrySectionSegment(MemoryArea, Segment, &Offset, Dirty, FALSE, NULL); } } } From f1631b44e120f2c6319fb4d4b113b0a43336b413 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Tue, 27 Oct 2020 17:36:18 +0100 Subject: [PATCH 19/99] [NTOS/MM] Introduce MmMapViewInSystemSpaceEx --- ntoskrnl/include/internal/mm.h | 10 +++++++ ntoskrnl/mm/ARM3/section.c | 38 +++++++++++++++++++------ ntoskrnl/mm/section.c | 51 +++++++++++++++++++++------------- 3 files changed, 71 insertions(+), 28 deletions(-) diff --git a/ntoskrnl/include/internal/mm.h b/ntoskrnl/include/internal/mm.h index a8536c3d786..662f49c9bd7 100644 --- a/ntoskrnl/include/internal/mm.h +++ b/ntoskrnl/include/internal/mm.h @@ -1313,6 +1313,16 @@ VOID NTAPI MmFreeSectionSegments(PFILE_OBJECT FileObject); +/* Exported from NT 6.2 Onward. We keep it internal. */ +NTSTATUS +NTAPI +MmMapViewInSystemSpaceEx ( + _In_ PVOID Section, + _Outptr_result_bytebuffer_ (*ViewSize) PVOID *MappedBase, + _Inout_ PSIZE_T ViewSize, + _Inout_ PLARGE_INTEGER SectionOffset + ); + /* sysldr.c ******************************************************************/ VOID diff --git a/ntoskrnl/mm/ARM3/section.c b/ntoskrnl/mm/ARM3/section.c index 511f11a5101..938a08d6037 100644 --- a/ntoskrnl/mm/ARM3/section.c +++ b/ntoskrnl/mm/ARM3/section.c @@ -416,12 +416,16 @@ NTSTATUS NTAPI MiAddMappedPtes(IN PMMPTE FirstPte, IN PFN_NUMBER PteCount, - IN PCONTROL_AREA ControlArea) + IN PCONTROL_AREA ControlArea, + IN PLARGE_INTEGER SectionOffset) { MMPTE TempPte; PMMPTE PointerPte, ProtoPte, LastProtoPte, LastPte; PSUBSECTION Subsection; + /* Mapping at offset not supported yet */ + ASSERT(SectionOffset->QuadPart == 0); + /* ARM3 doesn't support this yet */ ASSERT(ControlArea->u.Flags.GlobalOnlyPerSession == 0); ASSERT(ControlArea->u.Flags.Rom == 0); @@ -1052,11 +1056,13 @@ NTAPI MiMapViewInSystemSpace(IN PVOID Section, IN PMMSESSION Session, OUT PVOID *MappedBase, - IN OUT PSIZE_T ViewSize) + IN OUT PSIZE_T ViewSize, + IN PLARGE_INTEGER SectionOffset) { PVOID Base; PCONTROL_AREA ControlArea; - ULONG Buckets, SectionSize; + ULONG Buckets; + LONGLONG SectionSize; NTSTATUS Status; PAGED_CODE(); @@ -1073,13 +1079,23 @@ MiMapViewInSystemSpace(IN PVOID Section, ASSERT(NT_SUCCESS(Status)); /* Get the section size at creation time */ - SectionSize = ((PSECTION)Section)->SizeOfSection.LowPart; + SectionSize = ((PSECTION)Section)->SizeOfSection.QuadPart; - /* If the caller didn't specify a view size, assume the whole section */ - if (!(*ViewSize)) *ViewSize = SectionSize; + /* If the caller didn't specify a view size, assume until the end of the section */ + if (!(*ViewSize)) + { + /* Check for overflow first */ + if ((SectionSize - SectionOffset->QuadPart) > SIZE_T_MAX) + { + DPRINT1("Section end is too far away from the specified offset.\n"); + MiDereferenceControlArea(ControlArea); + return STATUS_INVALID_VIEW_SIZE; + } + *ViewSize = SectionSize - SectionOffset->QuadPart; + } /* Check if the caller wanted a larger section than the view */ - if (*ViewSize > SectionSize) + if (SectionOffset->QuadPart + *ViewSize > SectionSize) { /* Fail */ DPRINT1("View is too large\n"); @@ -1129,7 +1145,8 @@ MiMapViewInSystemSpace(IN PVOID Section, /* Create the actual prototype PTEs for this mapping */ Status = MiAddMappedPtes(MiAddressToPte(Base), BYTES_TO_PAGES(*ViewSize), - ControlArea); + ControlArea, + SectionOffset); ASSERT(NT_SUCCESS(Status)); /* Return the base adress of the mapping and success */ @@ -3016,6 +3033,7 @@ MmMapViewInSessionSpace(IN PVOID Section, IN OUT PSIZE_T ViewSize) { PAGED_CODE(); + LARGE_INTEGER SectionOffset; // HACK if (MiIsRosSectionObject(Section)) @@ -3032,10 +3050,12 @@ MmMapViewInSessionSpace(IN PVOID Section, /* Use the system space API, but with the session view instead */ ASSERT(MmIsAddressValid(MmSessionSpace) == TRUE); + SectionOffset.QuadPart = 0; return MiMapViewInSystemSpace(Section, &MmSessionSpace->Session, MappedBase, - ViewSize); + ViewSize, + &SectionOffset); } /* diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index 04cba04d612..fc2c1a3e904 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -88,7 +88,8 @@ NTAPI MiMapViewInSystemSpace(IN PVOID Section, IN PVOID Session, OUT PVOID *MappedBase, - IN OUT PSIZE_T ViewSize); + IN OUT PSIZE_T ViewSize, + IN PLARGE_INTEGER SectionOffset); NTSTATUS NTAPI @@ -3832,7 +3833,7 @@ MmMapViewOfSegment(PMMSUPPORT AddressSpace, PVOID* BaseAddress, SIZE_T ViewSize, ULONG Protect, - ULONG ViewOffset, + LONGLONG ViewOffset, ULONG AllocationType) { PMEMORY_AREA MArea; @@ -4417,7 +4418,6 @@ MmMapViewOfSection(IN PVOID SectionObject, { PSECTION Section; PMMSUPPORT AddressSpace; - ULONG ViewOffset; NTSTATUS Status = STATUS_SUCCESS; BOOLEAN NotAtBase = FALSE; @@ -4530,7 +4530,7 @@ MmMapViewOfSection(IN PVOID SectionObject, Section, &SectionSegments[i], &SBaseAddress, - SectionSegments[i].Length.LowPart, + SectionSegments[i].Length.QuadPart, SectionSegments[i].Protection, 0, 0); @@ -4548,6 +4548,7 @@ MmMapViewOfSection(IN PVOID SectionObject, else { PMM_SECTION_SEGMENT Segment = (PMM_SECTION_SEGMENT)Section->Segment; + LONGLONG ViewOffset; /* check for write access */ if ((Protect & (PAGE_READWRITE|PAGE_EXECUTE_READWRITE)) && @@ -4577,7 +4578,7 @@ MmMapViewOfSection(IN PVOID SectionObject, } else { - ViewOffset = SectionOffset->u.LowPart; + ViewOffset = SectionOffset->QuadPart; } if ((ViewOffset % PAGE_SIZE) != 0) @@ -4588,11 +4589,11 @@ MmMapViewOfSection(IN PVOID SectionObject, if ((*ViewSize) == 0) { - (*ViewSize) = Section->SizeOfSection.u.LowPart - ViewOffset; + (*ViewSize) = Section->SizeOfSection.QuadPart - ViewOffset; } - else if (((*ViewSize)+ViewOffset) > Section->SizeOfSection.u.LowPart) + else if (((*ViewSize)+ViewOffset) > Section->SizeOfSection.QuadPart) { - (*ViewSize) = Section->SizeOfSection.u.LowPart - ViewOffset; + (*ViewSize) = MIN(Section->SizeOfSection.QuadPart - ViewOffset, SIZE_T_MAX - PAGE_SIZE); } *ViewSize = PAGE_ROUND_UP(*ViewSize); @@ -4756,7 +4757,23 @@ MmMapViewInSystemSpace (IN PVOID SectionObject, OUT PVOID * MappedBase, IN OUT PSIZE_T ViewSize) { - PSECTION Section; + LARGE_INTEGER SectionOffset; + + SectionOffset.QuadPart = 0; + + return MmMapViewInSystemSpaceEx(SectionObject, MappedBase, ViewSize, &SectionOffset); +} + +NTSTATUS +NTAPI +MmMapViewInSystemSpaceEx ( + _In_ PVOID SectionObject, + _Outptr_result_bytebuffer_ (*ViewSize) PVOID *MappedBase, + _Inout_ PSIZE_T ViewSize, + _Inout_ PLARGE_INTEGER SectionOffset + ) +{ + PSECTION Section = SectionObject; PMM_SECTION_SEGMENT Segment; PMMSUPPORT AddressSpace; NTSTATUS Status; @@ -4767,10 +4784,11 @@ MmMapViewInSystemSpace (IN PVOID SectionObject, return MiMapViewInSystemSpace(SectionObject, &MmSession, MappedBase, - ViewSize); + ViewSize, + SectionOffset); } - DPRINT("MmMapViewInSystemSpace() called\n"); + DPRINT("MmMapViewInSystemSpaceEx() called\n"); Section = SectionObject; Segment = (PMM_SECTION_SEGMENT)Section->Segment; @@ -4780,25 +4798,20 @@ MmMapViewInSystemSpace (IN PVOID SectionObject, MmLockAddressSpace(AddressSpace); - if ((*ViewSize) == 0) + if ((*ViewSize == 0) || ((SectionOffset->QuadPart + *ViewSize) > Section->SizeOfSection.QuadPart)) { - (*ViewSize) = Section->SizeOfSection.u.LowPart; - } - else if ((*ViewSize) > Section->SizeOfSection.u.LowPart) - { - (*ViewSize) = Section->SizeOfSection.u.LowPart; + *ViewSize = MIN((Section->SizeOfSection.QuadPart - SectionOffset->QuadPart), SIZE_T_MAX); } MmLockSectionSegment(Segment); - Status = MmMapViewOfSegment(AddressSpace, Section, Segment, MappedBase, *ViewSize, PAGE_READWRITE, - 0, + SectionOffset->QuadPart, 0); MmUnlockSectionSegment(Segment); From e4047d1521cd75ffded656aef221d7eb8285a04e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Tue, 27 Oct 2020 17:37:38 +0100 Subject: [PATCH 20/99] [NTOS/MM] Introduce MmArePagesResident and MmMakePagesResident --- ntoskrnl/include/internal/mm.h | 14 ++++ ntoskrnl/mm/section.c | 140 +++++++++++++++++++++++++++++++++ 2 files changed, 154 insertions(+) diff --git a/ntoskrnl/include/internal/mm.h b/ntoskrnl/include/internal/mm.h index 662f49c9bd7..be8b0acc4b0 100644 --- a/ntoskrnl/include/internal/mm.h +++ b/ntoskrnl/include/internal/mm.h @@ -1323,6 +1323,20 @@ MmMapViewInSystemSpaceEx ( _Inout_ PLARGE_INTEGER SectionOffset ); +BOOLEAN +NTAPI +MmArePagesResident( + _In_ PEPROCESS Process, + _In_ PVOID BaseAddress, + _In_ ULONG Length); + +NTSTATUS +NTAPI +MmMakePagesResident( + _In_ PEPROCESS Process, + _In_ PVOID Address, + _In_ ULONG Length); + /* sysldr.c ******************************************************************/ VOID diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index fc2c1a3e904..4d29f5a9ff4 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -5076,4 +5076,144 @@ MmCreateSection (OUT PVOID * Section, return Status; } +BOOLEAN +NTAPI +MmArePagesResident( + _In_ PEPROCESS Process, + _In_ PVOID Address, + _In_ ULONG Length) +{ + PMEMORY_AREA MemoryArea; + BOOLEAN Ret = TRUE; + PMM_SECTION_SEGMENT Segment; + LARGE_INTEGER SegmentOffset, RangeEnd; + + MmLockAddressSpace(&Process->Vm); + + MemoryArea = MmLocateMemoryAreaByAddress(&Process->Vm, Address); + if (MemoryArea == NULL) + { + MmUnlockAddressSpace(&Process->Vm); + return FALSE; + } + + /* Only supported in old Mm for now */ + ASSERT(MemoryArea->Type == MEMORY_AREA_SECTION_VIEW); + /* For file mappings */ + ASSERT(MemoryArea->VadNode.u.VadFlags.VadType != VadImageMap); + + Segment = MemoryArea->SectionData.Segment; + MmLockSectionSegment(Segment); + + SegmentOffset.QuadPart = PAGE_ROUND_DOWN(Address) - MA_GetStartingAddress(MemoryArea) + + MemoryArea->SectionData.ViewOffset.QuadPart; + RangeEnd.QuadPart = PAGE_ROUND_UP((ULONG_PTR)Address + Length) - MA_GetStartingAddress(MemoryArea) + + MemoryArea->SectionData.ViewOffset.QuadPart; + + while (SegmentOffset.QuadPart < RangeEnd.QuadPart) + { + ULONG_PTR Entry = MmGetPageEntrySectionSegment(Segment, &SegmentOffset); + if ((Entry == 0) || IS_SWAP_FROM_SSE(Entry)) + { + Ret = FALSE; + break; + } + SegmentOffset.QuadPart += PAGE_SIZE; + } + + MmUnlockSectionSegment(Segment); + + MmUnlockAddressSpace(&Process->Vm); + return Ret; +} + +NTSTATUS +NTAPI +MmMakePagesResident( + _In_ PEPROCESS Process, + _In_ PVOID Address, + _In_ ULONG Length) +{ + PMEMORY_AREA MemoryArea; + PMM_SECTION_SEGMENT Segment; + LARGE_INTEGER SegmentOffset, RangeEnd; + + MmLockAddressSpace(&Process->Vm); + + MemoryArea = MmLocateMemoryAreaByAddress(&Process->Vm, Address); + if (MemoryArea == NULL) + { + MmUnlockAddressSpace(&Process->Vm); + return FALSE; + } + + /* Only supported in old Mm for now */ + ASSERT(MemoryArea->Type == MEMORY_AREA_SECTION_VIEW); + /* For file mappings */ + ASSERT(MemoryArea->VadNode.u.VadFlags.VadType != VadImageMap); + + Segment = MemoryArea->SectionData.Segment; + MmLockSectionSegment(Segment); + + SegmentOffset.QuadPart = PAGE_ROUND_DOWN(Address) - MA_GetStartingAddress(MemoryArea) + + MemoryArea->SectionData.ViewOffset.QuadPart; + RangeEnd.QuadPart = PAGE_ROUND_UP((ULONG_PTR)Address + Length) - MA_GetStartingAddress(MemoryArea) + + MemoryArea->SectionData.ViewOffset.QuadPart; + + while (SegmentOffset.QuadPart < RangeEnd.QuadPart) + { + ULONG_PTR Entry = MmGetPageEntrySectionSegment(Segment, &SegmentOffset); + + /* Let any pending read proceed */ + while (MM_IS_WAIT_PTE(Entry)) + { + MmUnlockSectionSegment(Segment); + MmUnlockAddressSpace(&Process->Vm); + MiWaitForPageEvent(NULL, NULL); + MmLockAddressSpace(&Process->Vm); + MmLockSectionSegment(Segment); + Entry = MmGetPageEntrySectionSegment(Segment, &SegmentOffset); + } + + /* We are called from Cc, this can't be backed by the page files */ + ASSERT(!IS_SWAP_FROM_SSE(Entry)); + + /* At this point, there may be a valid page there */ + if (Entry == 0) + { + PFN_NUMBER Page; + NTSTATUS Status; + + /* + * Release all our locks and read in the page from disk + */ + MmSetPageEntrySectionSegment(Segment, &SegmentOffset, MAKE_SWAP_SSE(MM_WAIT_ENTRY)); + MmUnlockSectionSegment(Segment); + MmUnlockAddressSpace(&Process->Vm); + + /* FIXME: Read the whole range at once instead of one page at a time */ + Status = MiReadPage(MemoryArea, SegmentOffset.QuadPart, &Page); + if (!NT_SUCCESS(Status)) + { + /* Reset the Segment entry and fail */ + MmLockSectionSegment(Segment); + MmSetPageEntrySectionSegment(Segment, &SegmentOffset, 0); + MmUnlockSectionSegment(Segment); + MiSetPageEvent(Process, Address); + return Status; + } + + MmLockAddressSpace(&Process->Vm); + MmLockSectionSegment(Segment); + MmSetPageEntrySectionSegment(Segment, &SegmentOffset, MAKE_SSE(Page << PAGE_SHIFT, 1)); + } + SegmentOffset.QuadPart += PAGE_SIZE; + } + + MmUnlockSectionSegment(Segment); + + MmUnlockAddressSpace(&Process->Vm); + return STATUS_SUCCESS; +} + /* EOF */ From d8cdb89fb03006595dc40ac23db5267b8d9d9c09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Fri, 6 Nov 2020 09:39:31 +0100 Subject: [PATCH 21/99] [NTOSKRNL] Overhaul Cc and Mm relationship Previously, when creating a file section, Mm requested Cc to cache the file, then Cc would request pages from Mm, then Mm would request them back to serve its file-mapping role Now, Mm does it all by itself. If file cahcing is requested by the FS driver, then Cc creates a file mapping and uses that to serve its purpose. This is a rewrite of Cc --- ntoskrnl/cache/section/sptab.c | 3 - ntoskrnl/cc/copy.c | 561 +++------ ntoskrnl/cc/fs.c | 58 +- ntoskrnl/cc/pin.c | 236 ++-- ntoskrnl/cc/view.c | 495 +++----- ntoskrnl/ex/init.c | 2 - ntoskrnl/ex/sysinfo.c | 12 +- ntoskrnl/include/internal/cc.h | 30 +- ntoskrnl/include/internal/mm.h | 82 +- ntoskrnl/mm/ARM3/section.c | 32 +- ntoskrnl/mm/ARM3/virtual.c | 8 +- ntoskrnl/mm/ARM3/zeropage.c | 2 +- ntoskrnl/mm/balance.c | 12 - ntoskrnl/mm/pagefile.c | 5 +- ntoskrnl/mm/rmap.c | 317 ++--- ntoskrnl/mm/section.c | 1986 +++++++++++++------------------- ntoskrnl/po/power.c | 4 +- 17 files changed, 1499 insertions(+), 2346 deletions(-) diff --git a/ntoskrnl/cache/section/sptab.c b/ntoskrnl/cache/section/sptab.c index 454fb25af42..7a2481a4752 100644 --- a/ntoskrnl/cache/section/sptab.c +++ b/ntoskrnl/cache/section/sptab.c @@ -187,9 +187,6 @@ _MmSetPageEntrySectionSegment(PMM_SECTION_SEGMENT Segment, ASSERT(Segment->Locked); ASSERT(!IS_SWAP_FROM_SSE(Entry) || !IS_DIRTY_SSE(Entry)); - if (Entry && !IS_SWAP_FROM_SSE(Entry)) - MmGetRmapListHeadPage(PFN_FROM_SSE(Entry)); - PageTable = MiSectionPageTableGetOrAllocate(&Segment->PageTable, Offset); if (!PageTable) return STATUS_NO_MEMORY; diff --git a/ntoskrnl/cc/copy.c b/ntoskrnl/cc/copy.c index 1f731716b1f..5bb14cc210a 100644 --- a/ntoskrnl/cc/copy.c +++ b/ntoskrnl/cc/copy.c @@ -20,13 +20,6 @@ static PFN_NUMBER CcZeroPage = 0; #define MAX_ZERO_LENGTH (256 * 1024) -typedef enum _CC_COPY_OPERATION -{ - CcOperationRead, - CcOperationWrite, - CcOperationZero -} CC_COPY_OPERATION; - typedef enum _CC_CAN_WRITE_RETRY { FirstTry = 0, @@ -35,7 +28,7 @@ typedef enum _CC_CAN_WRITE_RETRY RetryMasterLocked = 255, } CC_CAN_WRITE_RETRY; -ULONG CcRosTraceLevel = 0; +ULONG CcRosTraceLevel = CC_API_DEBUG; ULONG CcFastMdlReadWait; ULONG CcFastMdlReadNotPossible; ULONG CcFastReadNotPossible; @@ -76,338 +69,6 @@ CcInitCacheZeroPage ( MiZeroPhysicalPage(CcZeroPage); } -NTSTATUS -NTAPI -CcReadVirtualAddress ( - PROS_VACB Vacb) -{ - ULONG Size; - PMDL Mdl; - NTSTATUS Status; - IO_STATUS_BLOCK IoStatus; - KEVENT Event; - ULARGE_INTEGER LargeSize; - - LargeSize.QuadPart = Vacb->SharedCacheMap->SectionSize.QuadPart - Vacb->FileOffset.QuadPart; - if (LargeSize.QuadPart > VACB_MAPPING_GRANULARITY) - { - LargeSize.QuadPart = VACB_MAPPING_GRANULARITY; - } - Size = LargeSize.LowPart; - - Size = ROUND_TO_PAGES(Size); - ASSERT(Size <= VACB_MAPPING_GRANULARITY); - ASSERT(Size > 0); - - Mdl = IoAllocateMdl(Vacb->BaseAddress, Size, FALSE, FALSE, NULL); - if (!Mdl) - { - return STATUS_INSUFFICIENT_RESOURCES; - } - - Status = STATUS_SUCCESS; - _SEH2_TRY - { - MmProbeAndLockPages(Mdl, KernelMode, IoWriteAccess); - } - _SEH2_EXCEPT (EXCEPTION_EXECUTE_HANDLER) - { - Status = _SEH2_GetExceptionCode(); - DPRINT1("MmProbeAndLockPages failed with: %lx for %p (%p, %p)\n", Status, Mdl, Vacb, Vacb->BaseAddress); - KeBugCheck(CACHE_MANAGER); - } _SEH2_END; - - if (NT_SUCCESS(Status)) - { - Mdl->MdlFlags |= MDL_IO_PAGE_READ; - KeInitializeEvent(&Event, NotificationEvent, FALSE); - Status = IoPageRead(Vacb->SharedCacheMap->FileObject, Mdl, &Vacb->FileOffset, &Event, &IoStatus); - if (Status == STATUS_PENDING) - { - KeWaitForSingleObject(&Event, Executive, KernelMode, FALSE, NULL); - Status = IoStatus.Status; - } - - MmUnlockPages(Mdl); - } - - IoFreeMdl(Mdl); - - if (!NT_SUCCESS(Status) && (Status != STATUS_END_OF_FILE)) - { - DPRINT1("IoPageRead failed, Status %x\n", Status); - return Status; - } - - if (Size < VACB_MAPPING_GRANULARITY) - { - RtlZeroMemory((char*)Vacb->BaseAddress + Size, - VACB_MAPPING_GRANULARITY - Size); - } - - return STATUS_SUCCESS; -} - -NTSTATUS -NTAPI -CcWriteVirtualAddress ( - PROS_VACB Vacb) -{ - ULONG Size; - PMDL Mdl; - NTSTATUS Status; - IO_STATUS_BLOCK IoStatus; - KEVENT Event; - ULARGE_INTEGER LargeSize; - - LargeSize.QuadPart = Vacb->SharedCacheMap->SectionSize.QuadPart - Vacb->FileOffset.QuadPart; - if (LargeSize.QuadPart > VACB_MAPPING_GRANULARITY) - { - LargeSize.QuadPart = VACB_MAPPING_GRANULARITY; - } - Size = LargeSize.LowPart; - // - // Nonpaged pool PDEs in ReactOS must actually be synchronized between the - // MmGlobalPageDirectory and the real system PDE directory. What a mess... - // - { - ULONG i = 0; - do - { - MmGetPfnForProcess(NULL, (PVOID)((ULONG_PTR)Vacb->BaseAddress + (i << PAGE_SHIFT))); - } while (++i < (Size >> PAGE_SHIFT)); - } - - ASSERT(Size <= VACB_MAPPING_GRANULARITY); - ASSERT(Size > 0); - - Mdl = IoAllocateMdl(Vacb->BaseAddress, Size, FALSE, FALSE, NULL); - if (!Mdl) - { - return STATUS_INSUFFICIENT_RESOURCES; - } - - Status = STATUS_SUCCESS; - _SEH2_TRY - { - MmProbeAndLockPages(Mdl, KernelMode, IoReadAccess); - } - _SEH2_EXCEPT (EXCEPTION_EXECUTE_HANDLER) - { - Status = _SEH2_GetExceptionCode(); - DPRINT1("MmProbeAndLockPages failed with: %lx for %p (%p, %p)\n", Status, Mdl, Vacb, Vacb->BaseAddress); - KeBugCheck(CACHE_MANAGER); - } _SEH2_END; - - if (NT_SUCCESS(Status)) - { - KeInitializeEvent(&Event, NotificationEvent, FALSE); - Status = IoSynchronousPageWrite(Vacb->SharedCacheMap->FileObject, Mdl, &Vacb->FileOffset, &Event, &IoStatus); - if (Status == STATUS_PENDING) - { - KeWaitForSingleObject(&Event, Executive, KernelMode, FALSE, NULL); - Status = IoStatus.Status; - } - - MmUnlockPages(Mdl); - } - IoFreeMdl(Mdl); - if (!NT_SUCCESS(Status) && (Status != STATUS_END_OF_FILE)) - { - DPRINT1("IoPageWrite failed, Status %x\n", Status); - return Status; - } - - return STATUS_SUCCESS; -} - -NTSTATUS -ReadWriteOrZero( - _Inout_ PVOID BaseAddress, - _Inout_opt_ PVOID Buffer, - _In_ ULONG Length, - _In_ CC_COPY_OPERATION Operation) -{ - NTSTATUS Status = STATUS_SUCCESS; - - if (Operation == CcOperationZero) - { - /* Zero */ - RtlZeroMemory(BaseAddress, Length); - } - else - { - _SEH2_TRY - { - if (Operation == CcOperationWrite) - RtlCopyMemory(BaseAddress, Buffer, Length); - else - RtlCopyMemory(Buffer, BaseAddress, Length); - } - _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER) - { - Status = _SEH2_GetExceptionCode(); - } - _SEH2_END; - } - return Status; -} - -BOOLEAN -CcCopyData ( - _In_ PFILE_OBJECT FileObject, - _In_ LONGLONG FileOffset, - _Inout_ PVOID Buffer, - _In_ LONGLONG Length, - _In_ CC_COPY_OPERATION Operation, - _In_ BOOLEAN Wait, - _Out_ PIO_STATUS_BLOCK IoStatus) -{ - NTSTATUS Status; - LONGLONG CurrentOffset; - ULONG BytesCopied; - KIRQL OldIrql; - PROS_SHARED_CACHE_MAP SharedCacheMap; - PLIST_ENTRY ListEntry; - PROS_VACB Vacb; - ULONG PartialLength; - PVOID BaseAddress; - BOOLEAN Valid; - PPRIVATE_CACHE_MAP PrivateCacheMap; - - SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; - PrivateCacheMap = FileObject->PrivateCacheMap; - CurrentOffset = FileOffset; - BytesCopied = 0; - - if (!Wait) - { - /* test if the requested data is available */ - KeAcquireSpinLock(&SharedCacheMap->CacheMapLock, &OldIrql); - /* FIXME: this loop doesn't take into account areas that don't have - * a VACB in the list yet */ - ListEntry = SharedCacheMap->CacheMapVacbListHead.Flink; - while (ListEntry != &SharedCacheMap->CacheMapVacbListHead) - { - Vacb = CONTAINING_RECORD(ListEntry, - ROS_VACB, - CacheMapVacbListEntry); - ListEntry = ListEntry->Flink; - if (!Vacb->Valid && - DoRangesIntersect(Vacb->FileOffset.QuadPart, - VACB_MAPPING_GRANULARITY, - CurrentOffset, Length)) - { - KeReleaseSpinLock(&SharedCacheMap->CacheMapLock, OldIrql); - /* data not available */ - return FALSE; - } - if (Vacb->FileOffset.QuadPart >= CurrentOffset + Length) - break; - } - KeReleaseSpinLock(&SharedCacheMap->CacheMapLock, OldIrql); - } - - PartialLength = CurrentOffset % VACB_MAPPING_GRANULARITY; - if (PartialLength != 0) - { - PartialLength = min(Length, VACB_MAPPING_GRANULARITY - PartialLength); - Status = CcRosRequestVacb(SharedCacheMap, - ROUND_DOWN(CurrentOffset, - VACB_MAPPING_GRANULARITY), - &BaseAddress, - &Valid, - &Vacb); - if (!NT_SUCCESS(Status)) - ExRaiseStatus(Status); - if (!Valid) - { - Status = CcReadVirtualAddress(Vacb); - if (!NT_SUCCESS(Status)) - { - CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE, FALSE); - ExRaiseStatus(Status); - } - } - Status = ReadWriteOrZero((PUCHAR)BaseAddress + CurrentOffset % VACB_MAPPING_GRANULARITY, - Buffer, - PartialLength, - Operation); - - CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, Operation != CcOperationRead, FALSE); - - if (!NT_SUCCESS(Status)) - ExRaiseStatus(STATUS_INVALID_USER_BUFFER); - - Length -= PartialLength; - CurrentOffset += PartialLength; - BytesCopied += PartialLength; - - if (Operation != CcOperationZero) - Buffer = (PVOID)((ULONG_PTR)Buffer + PartialLength); - } - - while (Length > 0) - { - ASSERT(CurrentOffset % VACB_MAPPING_GRANULARITY == 0); - PartialLength = min(VACB_MAPPING_GRANULARITY, Length); - Status = CcRosRequestVacb(SharedCacheMap, - CurrentOffset, - &BaseAddress, - &Valid, - &Vacb); - if (!NT_SUCCESS(Status)) - ExRaiseStatus(Status); - if (!Valid && - (Operation == CcOperationRead || - PartialLength < VACB_MAPPING_GRANULARITY)) - { - Status = CcReadVirtualAddress(Vacb); - if (!NT_SUCCESS(Status)) - { - CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE, FALSE); - ExRaiseStatus(Status); - } - } - Status = ReadWriteOrZero(BaseAddress, Buffer, PartialLength, Operation); - - CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, Operation != CcOperationRead, FALSE); - - if (!NT_SUCCESS(Status)) - ExRaiseStatus(STATUS_INVALID_USER_BUFFER); - - Length -= PartialLength; - CurrentOffset += PartialLength; - BytesCopied += PartialLength; - - if (Operation != CcOperationZero) - Buffer = (PVOID)((ULONG_PTR)Buffer + PartialLength); - } - - /* If that was a successful sync read operation, let's handle read ahead */ - if (Operation == CcOperationRead && Length == 0 && Wait) - { - /* If file isn't random access and next read may get us cross VACB boundary, - * schedule next read - */ - if (!BooleanFlagOn(FileObject->Flags, FO_RANDOM_ACCESS) && - (CurrentOffset - 1) / VACB_MAPPING_GRANULARITY != (CurrentOffset + BytesCopied - 1) / VACB_MAPPING_GRANULARITY) - { - CcScheduleReadAhead(FileObject, (PLARGE_INTEGER)&FileOffset, BytesCopied); - } - - /* And update read history in private cache map */ - PrivateCacheMap->FileOffset1.QuadPart = PrivateCacheMap->FileOffset2.QuadPart; - PrivateCacheMap->BeyondLastByte1.QuadPart = PrivateCacheMap->BeyondLastByte2.QuadPart; - PrivateCacheMap->FileOffset2.QuadPart = FileOffset; - PrivateCacheMap->BeyondLastByte2.QuadPart = FileOffset + BytesCopied; - } - - IoStatus->Status = STATUS_SUCCESS; - IoStatus->Information = BytesCopied; - return TRUE; -} - VOID CcPostDeferredWrites(VOID) { @@ -492,8 +153,6 @@ CcPerformReadAhead( PROS_SHARED_CACHE_MAP SharedCacheMap; PROS_VACB Vacb; ULONG PartialLength; - PVOID BaseAddress; - BOOLEAN Valid; ULONG Length; PPRIVATE_CACHE_MAP PrivateCacheMap; BOOLEAN Locked; @@ -556,10 +215,7 @@ CcPerformReadAhead( { PartialLength = min(Length, VACB_MAPPING_GRANULARITY - PartialLength); Status = CcRosRequestVacb(SharedCacheMap, - ROUND_DOWN(CurrentOffset, - VACB_MAPPING_GRANULARITY), - &BaseAddress, - &Valid, + ROUND_DOWN(CurrentOffset, VACB_MAPPING_GRANULARITY), &Vacb); if (!NT_SUCCESS(Status)) { @@ -567,15 +223,13 @@ CcPerformReadAhead( goto Clear; } - if (!Valid) + Status = CcRosEnsureVacbResident(Vacb, TRUE, FALSE, + CurrentOffset % VACB_MAPPING_GRANULARITY, PartialLength); + if (!NT_SUCCESS(Status)) { - Status = CcReadVirtualAddress(Vacb); - if (!NT_SUCCESS(Status)) - { - CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE, FALSE); - DPRINT1("Failed to read data: %lx!\n", Status); - goto Clear; - } + CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE, FALSE); + DPRINT1("Failed to read data: %lx!\n", Status); + goto Clear; } CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); @@ -590,8 +244,6 @@ CcPerformReadAhead( PartialLength = min(VACB_MAPPING_GRANULARITY, Length); Status = CcRosRequestVacb(SharedCacheMap, CurrentOffset, - &BaseAddress, - &Valid, &Vacb); if (!NT_SUCCESS(Status)) { @@ -599,15 +251,12 @@ CcPerformReadAhead( goto Clear; } - if (!Valid) + Status = CcRosEnsureVacbResident(Vacb, TRUE, FALSE, 0, PartialLength); + if (!NT_SUCCESS(Status)) { - Status = CcReadVirtualAddress(Vacb); - if (!NT_SUCCESS(Status)) - { - CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE, FALSE); - DPRINT1("Failed to read data: %lx!\n", Status); - goto Clear; - } + CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE, FALSE); + DPRINT1("Failed to read data: %lx!\n", Status); + goto Clear; } CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); @@ -811,6 +460,12 @@ CcCopyRead ( OUT PVOID Buffer, OUT PIO_STATUS_BLOCK IoStatus) { + PROS_VACB Vacb; + PROS_SHARED_CACHE_MAP SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; + NTSTATUS Status; + LONGLONG CurrentOffset; + LONGLONG ReadEnd = FileOffset->QuadPart + Length; + CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%I64d Length=%lu Wait=%d\n", FileObject, FileOffset->QuadPart, Length, Wait); @@ -819,13 +474,58 @@ CcCopyRead ( FileObject, FileOffset->QuadPart, Length, Wait, Buffer, IoStatus); - return CcCopyData(FileObject, - FileOffset->QuadPart, - Buffer, - Length, - CcOperationRead, - Wait, - IoStatus); + if (!SharedCacheMap) + return FALSE; + + /* Documented to ASSERT, but KMTests test this case... */ + // ASSERT((FileOffset->QuadPart + Length) <= SharedCacheMap->FileSize.QuadPart); + + IoStatus->Status = STATUS_SUCCESS; + IoStatus->Information = 0; + + CurrentOffset = FileOffset->QuadPart; + while(CurrentOffset < ReadEnd) + { + Status = CcRosGetVacb(SharedCacheMap, CurrentOffset, &Vacb); + if (!NT_SUCCESS(Status)) + { + ExRaiseStatus(Status); + return FALSE; + } + + _SEH2_TRY + { + ULONG VacbOffset = CurrentOffset % VACB_MAPPING_GRANULARITY; + ULONG VacbLength = min(Length, VACB_MAPPING_GRANULARITY - VacbOffset); + SIZE_T CopyLength = VacbLength; + + if (!CcRosEnsureVacbResident(Vacb, Wait, FALSE, VacbOffset, VacbLength)) + return FALSE; + + /* Do not copy past the section */ + if (CurrentOffset + VacbLength > SharedCacheMap->SectionSize.QuadPart) + CopyLength = SharedCacheMap->SectionSize.QuadPart - CurrentOffset; + if (CopyLength != 0) + RtlCopyMemory(Buffer, (PUCHAR)Vacb->BaseAddress + VacbOffset, CopyLength); + + /* Zero-out the buffer tail if needed */ + if (CopyLength < VacbLength) + RtlZeroMemory((PUCHAR)Buffer + CopyLength, VacbLength - CopyLength); + + IoStatus->Information += VacbLength; + + Buffer = (PVOID)((ULONG_PTR)Buffer + VacbLength); + CurrentOffset += VacbLength; + Length -= VacbLength; + } + _SEH2_FINALLY + { + CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); + } + _SEH2_END; + } + + return TRUE; } /* @@ -840,7 +540,11 @@ CcCopyWrite ( IN BOOLEAN Wait, IN PVOID Buffer) { - IO_STATUS_BLOCK IoStatus; + PROS_VACB Vacb; + PROS_SHARED_CACHE_MAP SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; + NTSTATUS Status; + LONGLONG CurrentOffset; + LONGLONG WriteEnd = FileOffset->QuadPart + Length; CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%I64d Length=%lu Wait=%d Buffer=%p\n", FileObject, FileOffset->QuadPart, Length, Wait, Buffer); @@ -849,13 +553,48 @@ CcCopyWrite ( "Length %lu, Wait %u, Buffer 0x%p)\n", FileObject, FileOffset->QuadPart, Length, Wait, Buffer); - return CcCopyData(FileObject, - FileOffset->QuadPart, - Buffer, - Length, - CcOperationWrite, - Wait, - &IoStatus); + if (!SharedCacheMap) + return FALSE; + + /* FIXME: Honor FileObject FO_WRITE_THROUGH flag */ + + ASSERT((FileOffset->QuadPart + Length) <= SharedCacheMap->FileSize.QuadPart); + + CurrentOffset = FileOffset->QuadPart; + while(CurrentOffset < WriteEnd) + { + ULONG VacbOffset = CurrentOffset % VACB_MAPPING_GRANULARITY; + ULONG VacbLength = min(Length, VACB_MAPPING_GRANULARITY - VacbOffset); + + Status = CcRosGetVacb(SharedCacheMap, CurrentOffset, &Vacb); + if (!NT_SUCCESS(Status)) + { + ExRaiseStatus(Status); + return FALSE; + } + + _SEH2_TRY + { + if (!CcRosEnsureVacbResident(Vacb, Wait, FALSE, VacbOffset, VacbLength)) + { + CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); + return FALSE; + } + + RtlCopyMemory((PVOID)((ULONG_PTR)Vacb->BaseAddress + VacbOffset), Buffer, VacbLength); + + Buffer = (PVOID)((ULONG_PTR)Buffer + VacbLength); + CurrentOffset += VacbLength; + Length -= VacbLength; + } + _SEH2_FINALLY + { + CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, TRUE, FALSE); + } + _SEH2_END; + } + + return TRUE; } /* @@ -999,11 +738,8 @@ CcZeroData ( NTSTATUS Status; LARGE_INTEGER WriteOffset; LONGLONG Length; - ULONG CurrentLength; - PMDL Mdl; - ULONG i; - IO_STATUS_BLOCK Iosb; - KEVENT Event; + PROS_VACB Vacb; + PROS_SHARED_CACHE_MAP SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; CCTRACE(CC_API_DEBUG, "FileObject=%p StartOffset=%I64u EndOffset=%I64u Wait=%d\n", FileObject, StartOffset->QuadPart, EndOffset->QuadPart, Wait); @@ -1015,9 +751,14 @@ CcZeroData ( Length = EndOffset->QuadPart - StartOffset->QuadPart; WriteOffset.QuadPart = StartOffset->QuadPart; - if (FileObject->SectionObjectPointer->SharedCacheMap == NULL) + if (!SharedCacheMap || (FileObject->Flags & FO_WRITE_THROUGH)) { - /* File is not cached */ + /* Make this a non-cached write */ + IO_STATUS_BLOCK Iosb; + KEVENT Event; + PMDL Mdl; + ULONG i; + ULONG CurrentLength; Mdl = _alloca(MmSizeOfMdl(NULL, MAX_ZERO_LENGTH)); @@ -1032,7 +773,7 @@ CcZeroData ( CurrentLength = Length; } MmInitializeMdl(Mdl, (PVOID)(ULONG_PTR)WriteOffset.QuadPart, CurrentLength); - Mdl->MdlFlags |= (MDL_PAGES_LOCKED | MDL_IO_PAGE_READ); + Mdl->MdlFlags |= MDL_PAGES_LOCKED; for (i = 0; i < ((Mdl->Size - sizeof(MDL)) / sizeof(ULONG)); i++) { ((PPFN_NUMBER)(Mdl + 1))[i] = CcZeroPage; @@ -1055,18 +796,42 @@ CcZeroData ( WriteOffset.QuadPart += CurrentLength; Length -= CurrentLength; } - } - else - { - IO_STATUS_BLOCK IoStatus; - return CcCopyData(FileObject, - WriteOffset.QuadPart, - NULL, - Length, - CcOperationZero, - Wait, - &IoStatus); + return TRUE; + } + + ASSERT(EndOffset->QuadPart <= SharedCacheMap->FileSize.QuadPart); + + while(WriteOffset.QuadPart < EndOffset->QuadPart) + { + ULONG VacbOffset = WriteOffset.QuadPart % VACB_MAPPING_GRANULARITY; + ULONG VacbLength = min(Length, VACB_MAPPING_GRANULARITY - VacbOffset); + + Status = CcRosGetVacb(SharedCacheMap, WriteOffset.QuadPart, &Vacb); + if (!NT_SUCCESS(Status)) + { + ExRaiseStatus(Status); + return FALSE; + } + + _SEH2_TRY + { + if (!CcRosEnsureVacbResident(Vacb, Wait, FALSE, VacbOffset, VacbLength)) + { + CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); + return FALSE; + } + + RtlZeroMemory((PVOID)((ULONG_PTR)Vacb->BaseAddress + VacbOffset), VacbLength); + + WriteOffset.QuadPart += VacbLength; + Length -= VacbLength; + } + _SEH2_FINALLY + { + CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, TRUE, FALSE); + } + _SEH2_END; } return TRUE; diff --git a/ntoskrnl/cc/fs.c b/ntoskrnl/cc/fs.c index 5798f56a91a..5a74ecee48c 100644 --- a/ntoskrnl/cc/fs.c +++ b/ntoskrnl/cc/fs.c @@ -10,13 +10,10 @@ /* INCLUDES ******************************************************************/ #include + #define NDEBUG #include -/* GLOBALS *****************************************************************/ - -NTSTATUS CcRosInternalFreeVacb(PROS_VACB Vacb); - /* FUNCTIONS *****************************************************************/ /* @@ -272,8 +269,9 @@ CcSetFileSizes ( IN PFILE_OBJECT FileObject, IN PCC_FILE_SIZES FileSizes) { - KIRQL oldirql; + KIRQL OldIrql; PROS_SHARED_CACHE_MAP SharedCacheMap; + LARGE_INTEGER OldSectionSize; CCTRACE(CC_API_DEBUG, "FileObject=%p FileSizes=%p\n", FileObject, FileSizes); @@ -294,7 +292,14 @@ CcSetFileSizes ( if (SharedCacheMap == NULL) return; - if (FileSizes->AllocationSize.QuadPart < SharedCacheMap->SectionSize.QuadPart) + /* Update the relevant fields */ + KeAcquireSpinLock(&SharedCacheMap->CacheMapLock, &OldIrql); + OldSectionSize = SharedCacheMap->SectionSize; + SharedCacheMap->SectionSize = FileSizes->AllocationSize; + SharedCacheMap->FileSize = FileSizes->FileSize; + KeReleaseSpinLock(&SharedCacheMap->CacheMapLock, OldIrql); + + if (FileSizes->AllocationSize.QuadPart < OldSectionSize.QuadPart) { CcPurgeCacheSection(FileObject->SectionObjectPointer, &FileSizes->AllocationSize, @@ -303,46 +308,9 @@ CcSetFileSizes ( } else { - PROS_VACB LastVacb; - - /* - * If file (allocation) size has increased, then we need to check whether - * it just grows in a single VACB (the last one). - * If so, we must mark the VACB as invalid to trigger a read to the - * FSD at the next VACB usage, and thus avoid returning garbage - */ - - /* Check for allocation size and the last VACB */ - if (SharedCacheMap->SectionSize.QuadPart < FileSizes->AllocationSize.QuadPart && - SharedCacheMap->SectionSize.QuadPart % VACB_MAPPING_GRANULARITY) - { - LastVacb = CcRosLookupVacb(SharedCacheMap, - SharedCacheMap->SectionSize.QuadPart); - if (LastVacb != NULL) - { - /* Mark it as invalid */ - CcRosReleaseVacb(SharedCacheMap, LastVacb, LastVacb->Dirty ? LastVacb->Valid : FALSE, FALSE, FALSE); - } - } - - /* Check for file size and the last VACB */ - if (SharedCacheMap->FileSize.QuadPart < FileSizes->FileSize.QuadPart && - SharedCacheMap->FileSize.QuadPart % VACB_MAPPING_GRANULARITY) - { - LastVacb = CcRosLookupVacb(SharedCacheMap, - SharedCacheMap->FileSize.QuadPart); - if (LastVacb != NULL) - { - /* Mark it as invalid */ - CcRosReleaseVacb(SharedCacheMap, LastVacb, LastVacb->Dirty ? LastVacb->Valid : FALSE, FALSE, FALSE); - } - } + /* Extend our section object */ + MmExtendSection(SharedCacheMap->Section, &SharedCacheMap->SectionSize); } - - KeAcquireSpinLock(&SharedCacheMap->CacheMapLock, &oldirql); - SharedCacheMap->SectionSize = FileSizes->AllocationSize; - SharedCacheMap->FileSize = FileSizes->FileSize; - KeReleaseSpinLock(&SharedCacheMap->CacheMapLock, oldirql); } /* diff --git a/ntoskrnl/cc/pin.c b/ntoskrnl/cc/pin.c index b839e71dafd..189ba480305 100644 --- a/ntoskrnl/cc/pin.c +++ b/ntoskrnl/cc/pin.c @@ -67,91 +67,6 @@ CcpFindBcb( return (Found ? Bcb : NULL); } -static -BOOLEAN -NTAPI -CcpMapData( - IN PROS_SHARED_CACHE_MAP SharedCacheMap, - IN PLARGE_INTEGER FileOffset, - IN ULONG Length, - IN ULONG Flags, - OUT PROS_VACB *pVacb, - OUT PVOID *pBuffer) -{ - LONGLONG ReadOffset, BaseOffset; - BOOLEAN Valid; - PROS_VACB Vacb; - NTSTATUS Status; - LONGLONG ROffset; - - ReadOffset = FileOffset->QuadPart; - - DPRINT("SectionSize %I64x, FileSize %I64x\n", - SharedCacheMap->SectionSize.QuadPart, - SharedCacheMap->FileSize.QuadPart); - - if (ReadOffset % VACB_MAPPING_GRANULARITY + Length > VACB_MAPPING_GRANULARITY) - { - CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> FALSE\n", - SharedCacheMap->FileObject, FileOffset, Length, Flags); - return FALSE; - } - - if (!BooleanFlagOn(Flags, MAP_NO_READ)) - { - static int Warned = 0; - - SetFlag(Flags, MAP_NO_READ); - if (!Warned) - { - DPRINT1("Mapping/pinning with no read not implemented. Forcing read, might fail if wait not allowed\n"); - Warned++; - } - } - - /* Properly round offset and call internal helper for getting a VACB */ - ROffset = ROUND_DOWN(ReadOffset, VACB_MAPPING_GRANULARITY); - Status = CcRosGetVacb(SharedCacheMap, - ROffset, - &BaseOffset, - pBuffer, - &Valid, - &Vacb); - if (!NT_SUCCESS(Status)) - { - CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> FALSE\n", - SharedCacheMap->FileObject, FileOffset, Length, Flags); - ExRaiseStatus(Status); - return FALSE; - } - - if (!Valid && BooleanFlagOn(Flags, MAP_NO_READ)) - { - if (!BooleanFlagOn(Flags, MAP_WAIT)) - { - CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE, FALSE); - CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> FALSE\n", - SharedCacheMap->FileObject, FileOffset, Length, Flags); - return FALSE; - } - - Status = CcReadVirtualAddress(Vacb); - if (!NT_SUCCESS(Status)) - { - CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE, FALSE); - CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> FALSE\n", - SharedCacheMap->FileObject, FileOffset, Length, Flags); - ExRaiseStatus(Status); - return FALSE; - } - } - - *pBuffer = (PUCHAR)*pBuffer + ReadOffset % VACB_MAPPING_GRANULARITY; - *pVacb = Vacb; - - return TRUE; -} - static VOID CcpDereferenceBcb( @@ -304,44 +219,44 @@ CcpPinData( OUT PVOID * Buffer) { PINTERNAL_BCB NewBcb; - BOOLEAN Result; - PROS_VACB Vacb; KIRQL OldIrql; - ULONG MapFlags; + ULONG VacbOffset; + NTSTATUS Status; + BOOLEAN Result; + + VacbOffset = (ULONG)(FileOffset->QuadPart % VACB_MAPPING_GRANULARITY); + /* This seems to be valid, according to KMTests */ + if ((VacbOffset + Length) > VACB_MAPPING_GRANULARITY) + Length = VACB_MAPPING_GRANULARITY - VacbOffset; KeAcquireSpinLock(&SharedCacheMap->BcbSpinLock, &OldIrql); NewBcb = CcpFindBcb(SharedCacheMap, FileOffset, Length, TRUE); if (NewBcb != NULL) { + BOOLEAN Result; + ++NewBcb->RefCount; KeReleaseSpinLock(&SharedCacheMap->BcbSpinLock, OldIrql); if (BooleanFlagOn(Flags, PIN_EXCLUSIVE)) - { Result = ExAcquireResourceExclusiveLite(&NewBcb->Lock, BooleanFlagOn(Flags, PIN_WAIT)); - } else - { Result = ExAcquireSharedStarveExclusive(&NewBcb->Lock, BooleanFlagOn(Flags, PIN_WAIT)); - } if (!Result) { CcpDereferenceBcb(SharedCacheMap, NewBcb); - NewBcb = NULL; - } - else - { - NewBcb->PinCount++; - *Bcb = NewBcb; - *Buffer = (PUCHAR)NewBcb->Vacb->BaseAddress + FileOffset->QuadPart % VACB_MAPPING_GRANULARITY; + return FALSE; } - return Result; + NewBcb->PinCount++; } else { + LONGLONG ROffset; + PROS_VACB Vacb; + KeReleaseSpinLock(&SharedCacheMap->BcbSpinLock, OldIrql); if (BooleanFlagOn(Flags, PIN_IF_BCB)) @@ -349,29 +264,49 @@ CcpPinData( return FALSE; } - MapFlags = Flags & PIN_WAIT; - if (BooleanFlagOn(Flags, PIN_NO_READ)) + /* Properly round offset and call internal helper for getting a VACB */ + ROffset = ROUND_DOWN(FileOffset->QuadPart, VACB_MAPPING_GRANULARITY); + Status = CcRosGetVacb(SharedCacheMap, ROffset, &Vacb); + if (!NT_SUCCESS(Status)) { - SetFlag(MapFlags, MAP_NO_READ); + CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> FALSE\n", + SharedCacheMap->FileObject, FileOffset, Length, Flags); + ExRaiseStatus(Status); + return FALSE; } - Result = CcpMapData(SharedCacheMap, FileOffset, Length, MapFlags, &Vacb, Buffer); - if (Result) + NewBcb = CcpGetAppropriateBcb(SharedCacheMap, Vacb, FileOffset, Length, Flags, TRUE); + if (NewBcb == NULL) { - NewBcb = CcpGetAppropriateBcb(SharedCacheMap, Vacb, FileOffset, Length, Flags, TRUE); - if (NewBcb == NULL) - { - CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); - Result = FALSE; - } - else - { - *Bcb = NewBcb; - } + CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); + return FALSE; } } - return Result; + Result = FALSE; + _SEH2_TRY + { + /* Ensure the pages are resident */ + Result = CcRosEnsureVacbResident(NewBcb->Vacb, + BooleanFlagOn(Flags, PIN_WAIT), + BooleanFlagOn(Flags, PIN_NO_READ), + VacbOffset, Length); + } + _SEH2_FINALLY + { + if (!Result) + { + CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> FALSE\n", + SharedCacheMap->FileObject, FileOffset, Length, Flags); + CcUnpinData(NewBcb); + return FALSE; + } + } + _SEH2_END; + + *Bcb = NewBcb; + *Buffer = (PVOID)((ULONG_PTR)NewBcb->Vacb->BaseAddress + VacbOffset); + return TRUE; } /* @@ -387,13 +322,15 @@ CcMapData ( OUT PVOID *pBcb, OUT PVOID *pBuffer) { - BOOLEAN Ret; KIRQL OldIrql; PINTERNAL_BCB iBcb; PROS_VACB Vacb; PROS_SHARED_CACHE_MAP SharedCacheMap; + ULONG VacbOffset; + NTSTATUS Status; + BOOLEAN Result; - DPRINT("CcMapData(FileObject 0x%p, FileOffset %I64x, Length %lu, Flags 0x%lx," + CCTRACE(CC_API_DEBUG, "CcMapData(FileObject 0x%p, FileOffset 0x%I64x, Length %lu, Flags 0x%lx," " pBcb 0x%p, pBuffer 0x%p)\n", FileObject, FileOffset->QuadPart, Length, Flags, pBcb, pBuffer); @@ -413,6 +350,11 @@ CcMapData ( ++CcMapDataNoWait; } + VacbOffset = (ULONG)(FileOffset->QuadPart % VACB_MAPPING_GRANULARITY); + /* KMTests seem to show that it is allowed to call accross mapping granularity */ + if ((VacbOffset + Length) > VACB_MAPPING_GRANULARITY) + Length = VACB_MAPPING_GRANULARITY - VacbOffset; + KeAcquireSpinLock(&SharedCacheMap->BcbSpinLock, &OldIrql); iBcb = CcpFindBcb(SharedCacheMap, FileOffset, Length, FALSE); @@ -420,34 +362,54 @@ CcMapData ( { KeReleaseSpinLock(&SharedCacheMap->BcbSpinLock, OldIrql); - Ret = CcpMapData(SharedCacheMap, FileOffset, Length, Flags, &Vacb, pBuffer); - if (Ret) + /* Call internal helper for getting a VACB */ + Status = CcRosGetVacb(SharedCacheMap, FileOffset->QuadPart, &Vacb); + if (!NT_SUCCESS(Status)) { - iBcb = CcpGetAppropriateBcb(SharedCacheMap, Vacb, FileOffset, Length, 0, FALSE); - if (iBcb == NULL) - { - CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); - Ret = FALSE; - } - else - { - *pBcb = iBcb; - } + CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> FALSE\n", + SharedCacheMap->FileObject, FileOffset, Length, Flags); + ExRaiseStatus(Status); + return FALSE; + } + + iBcb = CcpGetAppropriateBcb(SharedCacheMap, Vacb, FileOffset, Length, 0, FALSE); + if (iBcb == NULL) + { + CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); + CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> FALSE\n", + SharedCacheMap->FileObject, FileOffset, Length, Flags); + return FALSE; } } else { ++iBcb->RefCount; KeReleaseSpinLock(&SharedCacheMap->BcbSpinLock, OldIrql); - - *pBcb = iBcb; - *pBuffer = (PUCHAR)iBcb->Vacb->BaseAddress + FileOffset->QuadPart % VACB_MAPPING_GRANULARITY; - Ret = TRUE; } - CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> %d Bcb=%p\n", - FileObject, FileOffset, Length, Flags, Ret, *pBcb); - return Ret; + _SEH2_TRY + { + Result = FALSE; + /* Ensure the pages are resident */ + Result = CcRosEnsureVacbResident(iBcb->Vacb, BooleanFlagOn(Flags, MAP_WAIT), + BooleanFlagOn(Flags, MAP_NO_READ), VacbOffset, Length); + } + _SEH2_FINALLY + { + if (!Result) + { + CcpDereferenceBcb(SharedCacheMap, iBcb); + return FALSE; + } + } + _SEH2_END; + + *pBcb = iBcb; + *pBuffer = (PVOID)((ULONG_PTR)iBcb->Vacb->BaseAddress + VacbOffset); + + CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> TRUE Bcb=%p, Buffer %p\n", + FileObject, FileOffset, Length, Flags, *pBcb, *pBuffer); + return Result; } /* diff --git a/ntoskrnl/cc/view.c b/ntoskrnl/cc/view.c index adb28bb84ab..ddbe77e22af 100644 --- a/ntoskrnl/cc/view.c +++ b/ntoskrnl/cc/view.c @@ -156,20 +156,29 @@ CcRosTraceCacheMap ( #endif } +NTSTATUS +NTAPI +MmFlushVirtualMemory(IN PEPROCESS Process, + IN OUT PVOID *BaseAddress, + IN OUT PSIZE_T RegionSize, + OUT PIO_STATUS_BLOCK IoStatusBlock); + NTSTATUS NTAPI CcRosFlushVacb ( PROS_VACB Vacb) { + IO_STATUS_BLOCK Iosb; + SIZE_T FlushSize = min(VACB_MAPPING_GRANULARITY, + Vacb->SharedCacheMap->SectionSize.QuadPart - Vacb->FileOffset.QuadPart); NTSTATUS Status; CcRosUnmarkDirtyVacb(Vacb, TRUE); - Status = CcWriteVirtualAddress(Vacb); + Status = MmFlushVirtualMemory(NULL, &Vacb->BaseAddress, &FlushSize, &Iosb); + if (!NT_SUCCESS(Status)) - { CcRosMarkDirtyVacb(Vacb); - } return Status; } @@ -234,6 +243,8 @@ CcRosFlushDirtyPages ( current->SharedCacheMap->LazyWriteContext, Wait); if (!Locked) { + DPRINT("Not locked!"); + ASSERT(!Wait); OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); CcRosVacbDecRefCount(current); continue; @@ -264,15 +275,18 @@ CcRosFlushDirtyPages ( PagesFreed = VACB_MAPPING_GRANULARITY / PAGE_SIZE; (*Count) += PagesFreed; - /* Make sure we don't overflow target! */ - if (Target < PagesFreed) + if (!Wait) { - /* If we would have, jump to zero directly */ - Target = 0; - } - else - { - Target -= PagesFreed; + /* Make sure we don't overflow target! */ + if (Target < PagesFreed) + { + /* If we would have, jump to zero directly */ + Target = 0; + } + else + { + Target -= PagesFreed; + } } } @@ -286,136 +300,6 @@ CcRosFlushDirtyPages ( return STATUS_SUCCESS; } -NTSTATUS -CcRosTrimCache ( - ULONG Target, - ULONG Priority, - PULONG NrFreed) -/* - * FUNCTION: Try to free some memory from the file cache. - * ARGUMENTS: - * Target - The number of pages to be freed. - * Priority - The priority of free (currently unused). - * NrFreed - Points to a variable where the number of pages - * actually freed is returned. - */ -{ - PLIST_ENTRY current_entry; - PROS_VACB current; - ULONG PagesFreed; - KIRQL oldIrql; - LIST_ENTRY FreeList; - PFN_NUMBER Page; - ULONG i; - BOOLEAN FlushedPages = FALSE; - - DPRINT("CcRosTrimCache(Target %lu)\n", Target); - - InitializeListHead(&FreeList); - - *NrFreed = 0; - -retry: - oldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); - - current_entry = VacbLruListHead.Flink; - while (current_entry != &VacbLruListHead) - { - ULONG Refs; - - current = CONTAINING_RECORD(current_entry, - ROS_VACB, - VacbLruListEntry); - current_entry = current_entry->Flink; - - KeAcquireSpinLockAtDpcLevel(¤t->SharedCacheMap->CacheMapLock); - - /* Reference the VACB */ - CcRosVacbIncRefCount(current); - - /* Check if it's mapped and not dirty */ - if (InterlockedCompareExchange((PLONG)¤t->MappedCount, 0, 0) > 0 && !current->Dirty) - { - /* We have to break these locks because Cc sucks */ - KeReleaseSpinLockFromDpcLevel(¤t->SharedCacheMap->CacheMapLock); - KeReleaseQueuedSpinLock(LockQueueMasterLock, oldIrql); - - /* Page out the VACB */ - for (i = 0; i < VACB_MAPPING_GRANULARITY / PAGE_SIZE; i++) - { - Page = (PFN_NUMBER)(MmGetPhysicalAddress((PUCHAR)current->BaseAddress + (i * PAGE_SIZE)).QuadPart >> PAGE_SHIFT); - - MmPageOutPhysicalAddress(Page); - } - - /* Reacquire the locks */ - oldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); - KeAcquireSpinLockAtDpcLevel(¤t->SharedCacheMap->CacheMapLock); - } - - /* Dereference the VACB */ - Refs = CcRosVacbDecRefCount(current); - - /* Check if we can free this entry now */ - if (Refs < 2) - { - ASSERT(!current->Dirty); - ASSERT(!current->MappedCount); - ASSERT(Refs == 1); - - RemoveEntryList(¤t->CacheMapVacbListEntry); - RemoveEntryList(¤t->VacbLruListEntry); - InitializeListHead(¤t->VacbLruListEntry); - InsertHeadList(&FreeList, ¤t->CacheMapVacbListEntry); - - /* Calculate how many pages we freed for Mm */ - PagesFreed = min(VACB_MAPPING_GRANULARITY / PAGE_SIZE, Target); - Target -= PagesFreed; - (*NrFreed) += PagesFreed; - } - - KeReleaseSpinLockFromDpcLevel(¤t->SharedCacheMap->CacheMapLock); - } - - KeReleaseQueuedSpinLock(LockQueueMasterLock, oldIrql); - - /* Try flushing pages if we haven't met our target */ - if ((Target > 0) && !FlushedPages) - { - /* Flush dirty pages to disk */ - CcRosFlushDirtyPages(Target, &PagesFreed, FALSE, FALSE); - FlushedPages = TRUE; - - /* We can only swap as many pages as we flushed */ - if (PagesFreed < Target) Target = PagesFreed; - - /* Check if we flushed anything */ - if (PagesFreed != 0) - { - /* Try again after flushing dirty pages */ - DPRINT("Flushed %lu dirty cache pages to disk\n", PagesFreed); - goto retry; - } - } - - while (!IsListEmpty(&FreeList)) - { - ULONG Refs; - - current_entry = RemoveHeadList(&FreeList); - current = CONTAINING_RECORD(current_entry, - ROS_VACB, - CacheMapVacbListEntry); - InitializeListHead(¤t->CacheMapVacbListEntry); - Refs = CcRosVacbDecRefCount(current); - ASSERT(Refs == 0); - } - - DPRINT("Evicted %lu cache pages\n", (*NrFreed)); - - return STATUS_SUCCESS; -} - NTSTATUS NTAPI CcRosReleaseVacb ( @@ -504,6 +388,7 @@ CcRosMarkDirtyVacb ( { KIRQL oldIrql; PROS_SHARED_CACHE_MAP SharedCacheMap; + ULONG Length = VACB_MAPPING_GRANULARITY; SharedCacheMap = Vacb->SharedCacheMap; @@ -513,8 +398,12 @@ CcRosMarkDirtyVacb ( ASSERT(!Vacb->Dirty); InsertTailList(&DirtyVacbListHead, &Vacb->DirtyVacbListEntry); - CcTotalDirtyPages += VACB_MAPPING_GRANULARITY / PAGE_SIZE; - Vacb->SharedCacheMap->DirtyPages += VACB_MAPPING_GRANULARITY / PAGE_SIZE; +#if 0 + if (Vacb->FileOffset.QuadPart + Length > SharedCacheMap->SectionSize.QuadPart) + Length = SharedCacheMap->SectionSize.QuadPart - Vacb->FileOffset.QuadPart; +#endif + CcTotalDirtyPages += PAGE_ROUND_UP(Length) / PAGE_SIZE; + Vacb->SharedCacheMap->DirtyPages += PAGE_ROUND_UP(Length) / PAGE_SIZE; CcRosVacbIncRefCount(Vacb); /* Move to the tail of the LRU list */ @@ -531,6 +420,9 @@ CcRosMarkDirtyVacb ( CcScheduleLazyWriteScan(FALSE); } KeReleaseQueuedSpinLock(LockQueueMasterLock, oldIrql); + + /* Tell Mm */ + MmMakePagesDirty(NULL, Vacb->BaseAddress, Length); } VOID @@ -541,6 +433,7 @@ CcRosUnmarkDirtyVacb ( { KIRQL oldIrql; PROS_SHARED_CACHE_MAP SharedCacheMap; + ULONG Length = VACB_MAPPING_GRANULARITY; SharedCacheMap = Vacb->SharedCacheMap; @@ -556,8 +449,14 @@ CcRosUnmarkDirtyVacb ( RemoveEntryList(&Vacb->DirtyVacbListEntry); InitializeListHead(&Vacb->DirtyVacbListEntry); - CcTotalDirtyPages -= VACB_MAPPING_GRANULARITY / PAGE_SIZE; - Vacb->SharedCacheMap->DirtyPages -= VACB_MAPPING_GRANULARITY / PAGE_SIZE; + +#if 0 + if (Vacb->FileOffset.QuadPart + Length > SharedCacheMap->SectionSize.QuadPart) + Length = SharedCacheMap->SectionSize.QuadPart - Vacb->FileOffset.QuadPart; +#endif + CcTotalDirtyPages -= PAGE_ROUND_UP(Length) / PAGE_SIZE; + Vacb->SharedCacheMap->DirtyPages -= PAGE_ROUND_UP(Length) / PAGE_SIZE; + CcRosVacbDecRefCount(Vacb); if (LockViews) @@ -626,73 +525,6 @@ CcRosUnmapVacb ( return STATUS_SUCCESS; } -static -NTSTATUS -CcRosMapVacbInKernelSpace( - PROS_VACB Vacb) -{ - ULONG i; - NTSTATUS Status; - ULONG_PTR NumberOfPages; - PVOID BaseAddress = NULL; - - /* Create a memory area. */ - MmLockAddressSpace(MmGetKernelAddressSpace()); - Status = MmCreateMemoryArea(MmGetKernelAddressSpace(), - 0, // nothing checks for VACB mareas, so set to 0 - &BaseAddress, - VACB_MAPPING_GRANULARITY, - PAGE_READWRITE, - (PMEMORY_AREA*)&Vacb->MemoryArea, - 0, - PAGE_SIZE); - ASSERT(Vacb->BaseAddress == NULL); - Vacb->BaseAddress = BaseAddress; - MmUnlockAddressSpace(MmGetKernelAddressSpace()); - if (!NT_SUCCESS(Status)) - { - DPRINT1("MmCreateMemoryArea failed with %lx for VACB %p\n", Status, Vacb); - return Status; - } - - ASSERT(((ULONG_PTR)Vacb->BaseAddress % PAGE_SIZE) == 0); - ASSERT((ULONG_PTR)Vacb->BaseAddress > (ULONG_PTR)MmSystemRangeStart); - ASSERT((ULONG_PTR)Vacb->BaseAddress + VACB_MAPPING_GRANULARITY - 1 > (ULONG_PTR)MmSystemRangeStart); - - /* Create a virtual mapping for this memory area */ - NumberOfPages = BYTES_TO_PAGES(VACB_MAPPING_GRANULARITY); - for (i = 0; i < NumberOfPages; i++) - { - PFN_NUMBER PageFrameNumber; - - MI_SET_USAGE(MI_USAGE_CACHE); - Status = MmRequestPageMemoryConsumer(MC_CACHE, TRUE, &PageFrameNumber); - if (PageFrameNumber == 0) - { - DPRINT1("Unable to allocate page\n"); - KeBugCheck(MEMORY_MANAGEMENT); - } - - ASSERT(BaseAddress == Vacb->BaseAddress); - ASSERT(i * PAGE_SIZE < VACB_MAPPING_GRANULARITY); - ASSERT((ULONG_PTR)Vacb->BaseAddress + (i * PAGE_SIZE) >= (ULONG_PTR)BaseAddress); - ASSERT((ULONG_PTR)Vacb->BaseAddress + (i * PAGE_SIZE) > (ULONG_PTR)MmSystemRangeStart); - - Status = MmCreateVirtualMapping(NULL, - (PVOID)((ULONG_PTR)Vacb->BaseAddress + (i * PAGE_SIZE)), - PAGE_READWRITE, - &PageFrameNumber, - 1); - if (!NT_SUCCESS(Status)) - { - DPRINT1("Unable to create virtual mapping\n"); - KeBugCheck(MEMORY_MANAGEMENT); - } - } - - return STATUS_SUCCESS; -} - static BOOLEAN CcRosFreeUnusedVacb ( @@ -789,6 +621,7 @@ CcRosCreateVacb ( KIRQL oldIrql; ULONG Refs; BOOLEAN Retried; + SIZE_T ViewSize = VACB_MAPPING_GRANULARITY; ASSERT(SharedCacheMap); @@ -823,8 +656,9 @@ CcRosCreateVacb ( Retried = FALSE; Retry: - /* Map VACB in kernel space */ - Status = CcRosMapVacbInKernelSpace(current); + /* Map VACB in system space */ + Status = MmMapViewInSystemSpaceEx(SharedCacheMap->Section, ¤t->BaseAddress, &ViewSize, ¤t->FileOffset); + if (!NT_SUCCESS(Status)) { ULONG Freed; @@ -932,14 +766,50 @@ Retry: return Status; } +BOOLEAN +NTAPI +CcRosEnsureVacbResident( + _In_ PROS_VACB Vacb, + _In_ BOOLEAN Wait, + _In_ BOOLEAN NoRead, + _In_ ULONG Offset, + _In_ ULONG Length +) +{ + PVOID BaseAddress; + + ASSERT((Offset + Length) <= VACB_MAPPING_GRANULARITY); + + if ((Vacb->FileOffset.QuadPart + Offset) > Vacb->SharedCacheMap->FileSize.QuadPart) + return FALSE; + + BaseAddress = (PVOID)((ULONG_PTR)Vacb->BaseAddress + Offset); + + /* Check if the pages are resident */ + if (!MmArePagesResident(NULL, BaseAddress, Length)) + { + if (!Wait) + { + return FALSE; + } + + if (!NoRead) + { + NTSTATUS Status = MmMakePagesResident(NULL, BaseAddress, Length); + if (!NT_SUCCESS(Status)) + ExRaiseStatus(Status); + } + } + + return TRUE; +} + + NTSTATUS NTAPI CcRosGetVacb ( PROS_SHARED_CACHE_MAP SharedCacheMap, LONGLONG FileOffset, - PLONGLONG BaseOffset, - PVOID* BaseAddress, - PBOOLEAN UptoDate, PROS_VACB *Vacb) { PROS_VACB current; @@ -978,13 +848,9 @@ CcRosGetVacb ( KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); /* - * Return information about the VACB to the caller. + * Return the VACB to the caller. */ - *UptoDate = current->Valid; - *BaseAddress = current->BaseAddress; - DPRINT("*BaseAddress %p\n", *BaseAddress); *Vacb = current; - *BaseOffset = current->FileOffset.QuadPart; ASSERT(Refs > 1); @@ -996,14 +862,11 @@ NTAPI CcRosRequestVacb ( PROS_SHARED_CACHE_MAP SharedCacheMap, LONGLONG FileOffset, - PVOID* BaseAddress, - PBOOLEAN UptoDate, PROS_VACB *Vacb) /* * FUNCTION: Request a page mapping for a shared cache map */ { - LONGLONG BaseOffset; ASSERT(SharedCacheMap); @@ -1016,30 +879,9 @@ CcRosRequestVacb ( return CcRosGetVacb(SharedCacheMap, FileOffset, - &BaseOffset, - BaseAddress, - UptoDate, Vacb); } -static -VOID -CcFreeCachePage ( - PVOID Context, - MEMORY_AREA* MemoryArea, - PVOID Address, - PFN_NUMBER Page, - SWAPENTRY SwapEntry, - BOOLEAN Dirty) -{ - ASSERT(SwapEntry == 0); - if (Page != 0) - { - ASSERT(MmGetReferenceCountPage(Page) == 1); - MmReleasePageMemoryConsumer(MC_CACHE, Page); - } -} - NTSTATUS CcRosInternalFreeVacb ( PROS_VACB Vacb) @@ -1047,6 +889,8 @@ CcRosInternalFreeVacb ( * FUNCTION: Releases a VACB associated with a shared cache map */ { + NTSTATUS Status; + DPRINT("Freeing VACB 0x%p\n", Vacb); #if DBG if (Vacb->SharedCacheMap->Trace) @@ -1055,12 +899,14 @@ CcRosInternalFreeVacb ( } #endif - MmLockAddressSpace(MmGetKernelAddressSpace()); - MmFreeMemoryArea(MmGetKernelAddressSpace(), - Vacb->MemoryArea, - CcFreeCachePage, - NULL); - MmUnlockAddressSpace(MmGetKernelAddressSpace()); + /* Delete the mapping */ + Status = MmUnmapViewInSystemSpace(Vacb->BaseAddress); + if (!NT_SUCCESS(Status)) + { + DPRINT1("Failed to unmap VACB from System address space! Status 0x%08X\n", Status); + ASSERT(FALSE); + /* Proceed with the deĺetion anyway */ + } if (Vacb->ReferenceCount != 0) { @@ -1097,11 +943,8 @@ CcFlushCache ( PROS_VACB current; NTSTATUS Status; - CCTRACE(CC_API_DEBUG, "SectionObjectPointers=%p FileOffset=%p Length=%lu\n", - SectionObjectPointers, FileOffset, Length); - - DPRINT("CcFlushCache(SectionObjectPointers 0x%p, FileOffset 0x%p, Length %lu, IoStatus 0x%p)\n", - SectionObjectPointers, FileOffset, Length, IoStatus); + CCTRACE(CC_API_DEBUG, "SectionObjectPointers=%p FileOffset=0x%I64X Length=%lu\n", + SectionObjectPointers, FileOffset ? FileOffset->QuadPart : 0LL, Length); if (SectionObjectPointers && SectionObjectPointers->SharedCacheMap) { @@ -1217,6 +1060,8 @@ CcRosDeleteFileCache ( KeReleaseSpinLockFromDpcLevel(&SharedCacheMap->CacheMapLock); KeReleaseQueuedSpinLock(LockQueueMasterLock, *OldIrql); + if(SharedCacheMap->Section) + ObDereferenceObject(SharedCacheMap->Section); ObDereferenceObject(SharedCacheMap->FileObject); while (!IsListEmpty(&FreeList)) @@ -1299,36 +1144,6 @@ CcRosRemoveIfClosed ( KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); } - -VOID -NTAPI -CcRosDereferenceCache ( - PFILE_OBJECT FileObject) -{ - PROS_SHARED_CACHE_MAP SharedCacheMap; - KIRQL OldIrql; - - OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); - SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; - ASSERT(SharedCacheMap); - if (SharedCacheMap->OpenCount > 0) - { - SharedCacheMap->OpenCount--; - if (SharedCacheMap->OpenCount == 0) - { - KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); - MmFreeSectionSegments(SharedCacheMap->FileObject); - - OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); - CcRosDeleteFileCache(FileObject, SharedCacheMap, &OldIrql); - KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); - - return; - } - } - KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); -} - NTSTATUS NTAPI CcRosReleaseFileCache ( @@ -1373,20 +1188,12 @@ CcRosReleaseFileCache ( PrivateMap->NodeTypeCode = 0; } - if (SharedCacheMap->OpenCount > 0) + ASSERT(SharedCacheMap->OpenCount > 0); + + SharedCacheMap->OpenCount--; + if (SharedCacheMap->OpenCount == 0) { - SharedCacheMap->OpenCount--; - if (SharedCacheMap->OpenCount == 0) - { - KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); - MmFreeSectionSegments(SharedCacheMap->FileObject); - - OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); - CcRosDeleteFileCache(FileObject, SharedCacheMap, &OldIrql); - KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); - - return STATUS_SUCCESS; - } + CcRosDeleteFileCache(FileObject, SharedCacheMap, &OldIrql); } } } @@ -1412,6 +1219,8 @@ CcRosInitializeFileCache ( DPRINT("CcRosInitializeFileCache(FileObject 0x%p)\n", FileObject); + OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); + Allocated = FALSE; SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; if (SharedCacheMap == NULL) @@ -1437,27 +1246,37 @@ CcRosInitializeFileCache ( KeInitializeSpinLock(&SharedCacheMap->CacheMapLock); InitializeListHead(&SharedCacheMap->CacheMapVacbListHead); InitializeListHead(&SharedCacheMap->BcbList); - } - OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); - if (Allocated) + SharedCacheMap->Flags = SHARED_CACHE_MAP_IN_CREATION; + + ObReferenceObjectByPointer(FileObject, + FILE_ALL_ACCESS, + NULL, + KernelMode); + + FileObject->SectionObjectPointer->SharedCacheMap = SharedCacheMap; + + // CcRosTraceCacheMap(SharedCacheMap, TRUE); + } + else if (SharedCacheMap->Flags & SHARED_CACHE_MAP_IN_CREATION) { - if (FileObject->SectionObjectPointer->SharedCacheMap == NULL) - { - ObReferenceObjectByPointer(FileObject, - FILE_ALL_ACCESS, - NULL, - KernelMode); - FileObject->SectionObjectPointer->SharedCacheMap = SharedCacheMap; + /* The shared cache map is being created somewhere else. Wait for that to happen */ + KEVENT Waiter; + PKEVENT PreviousWaiter = SharedCacheMap->CreateEvent; - InsertTailList(&CcCleanSharedCacheMapList, &SharedCacheMap->SharedCacheMapLinks); - } - else - { - ExFreeToNPagedLookasideList(&SharedCacheMapLookasideList, SharedCacheMap); - SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; - } + KeInitializeEvent(&Waiter, NotificationEvent, FALSE); + SharedCacheMap->CreateEvent = &Waiter; + + KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); + + KeWaitForSingleObject(&Waiter, Executive, KernelMode, FALSE, NULL); + + if (PreviousWaiter) + KeSetEvent(PreviousWaiter, IO_NO_INCREMENT, FALSE); + + OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); } + if (FileObject->PrivateCacheMap == NULL) { PPRIVATE_CACHE_MAP PrivateMap; @@ -1503,8 +1322,48 @@ CcRosInitializeFileCache ( FileObject->PrivateCacheMap = PrivateMap; SharedCacheMap->OpenCount++; } + KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); + /* Create the section */ + if (Allocated) + { + NTSTATUS Status; + + ASSERT(SharedCacheMap->Section == NULL); + + Status = MmCreateSection( + &SharedCacheMap->Section, + SECTION_ALL_ACCESS, + NULL, + &SharedCacheMap->SectionSize, + PAGE_READWRITE, + 0, + NULL, + FileObject); + + ASSERT(NT_SUCCESS(Status)); + + if (!NT_SUCCESS(Status)) + { + CcRosReleaseFileCache(FileObject); + return Status; + } + + OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); + + InsertTailList(&CcCleanSharedCacheMapList, &SharedCacheMap->SharedCacheMapLinks); + SharedCacheMap->Flags &= ~SHARED_CACHE_MAP_IN_CREATION; + + if (SharedCacheMap->CreateEvent) + { + KeSetEvent(SharedCacheMap->CreateEvent, IO_NO_INCREMENT, FALSE); + SharedCacheMap->CreateEvent = NULL; + } + + KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); + } + return STATUS_SUCCESS; } @@ -1564,8 +1423,6 @@ CcInitView ( TAG_VACB, 20); - MmInitializeMemoryConsumer(MC_CACHE, CcRosTrimCache); - CcInitCacheZeroPage(); } diff --git a/ntoskrnl/ex/init.c b/ntoskrnl/ex/init.c index 8eb867b09cd..fc98cccd430 100644 --- a/ntoskrnl/ex/init.c +++ b/ntoskrnl/ex/init.c @@ -1970,7 +1970,6 @@ Phase1InitializationDiscard(IN PVOID Context) InbvEnableDisplayString(TRUE); /* Launch initial process */ - DPRINT("Free non-cache pages: %lx\n", MmAvailablePages + MiMemoryConsumers[MC_CACHE].PagesUsed); ProcessInfo = &InitBuffer->ProcessInfo; ExpLoadInitialProcess(InitBuffer, &ProcessParameters, &Environment); @@ -2009,7 +2008,6 @@ Phase1InitializationDiscard(IN PVOID Context) /* Free the boot buffer */ ExFreePoolWithTag(InitBuffer, TAG_INIT); - DPRINT("Free non-cache pages: %lx\n", MmAvailablePages + MiMemoryConsumers[MC_CACHE].PagesUsed); } VOID diff --git a/ntoskrnl/ex/sysinfo.c b/ntoskrnl/ex/sysinfo.c index 64cbffc5ba1..f595623b4c8 100644 --- a/ntoskrnl/ex/sysinfo.c +++ b/ntoskrnl/ex/sysinfo.c @@ -719,7 +719,6 @@ QSI_DEF(SystemPerformanceInformation) * Not sure this is right. 8^\ */ Spi->CommittedPages = MiMemoryConsumers[MC_SYSTEM].PagesUsed + - MiMemoryConsumers[MC_CACHE].PagesUsed + MiMemoryConsumers[MC_USER].PagesUsed + MiUsedSwapPages; /* @@ -767,7 +766,7 @@ QSI_DEF(SystemPerformanceInformation) Spi->TotalSystemDriverPages = 0; /* FIXME */ Spi->Spare3Count = 0; /* FIXME */ - Spi->ResidentSystemCachePage = MiMemoryConsumers[MC_CACHE].PagesUsed; + Spi->ResidentSystemCachePage = 0; /* FIXME */ Spi->ResidentPagedPoolPage = 0; /* FIXME */ Spi->ResidentSystemDriverPage = 0; /* FIXME */ @@ -1477,13 +1476,10 @@ QSI_DEF(SystemFileCacheInformation) RtlZeroMemory(Sci, sizeof(SYSTEM_FILECACHE_INFORMATION)); /* Return the Byte size not the page size. */ - Sci->CurrentSize = - MiMemoryConsumers[MC_CACHE].PagesUsed * PAGE_SIZE; - Sci->PeakSize = - MiMemoryConsumers[MC_CACHE].PagesUsed * PAGE_SIZE; /* FIXME */ + Sci->CurrentSize = 0; /* FIXME */ + Sci->PeakSize = 0; /* FIXME */ /* Taskmgr multiplies this one by page size right away */ - Sci->CurrentSizeIncludingTransitionInPages = - MiMemoryConsumers[MC_CACHE].PagesUsed; /* FIXME: Should be */ + Sci->CurrentSizeIncludingTransitionInPages = 0; /* FIXME: Should be */ /* system working set and standby pages. */ Sci->PageFaultCount = 0; /* FIXME */ Sci->MinimumWorkingSet = 0; /* FIXME */ diff --git a/ntoskrnl/include/internal/cc.h b/ntoskrnl/include/internal/cc.h index 18c0b9effa2..9e186a61d11 100644 --- a/ntoskrnl/include/internal/cc.h +++ b/ntoskrnl/include/internal/cc.h @@ -3,7 +3,7 @@ // // Define this if you want debugging support // -#define _CC_DEBUG_ 0x00 +#define _CC_DEBUG_ 0x0 // // These define the Debug Masks Supported @@ -179,6 +179,8 @@ typedef struct _ROS_SHARED_CACHE_MAP ULONG DirtyPages; LIST_ENTRY SharedCacheMapLinks; ULONG Flags; + PVOID Section; + PKEVENT CreateEvent; PCACHE_MANAGER_CALLBACKS Callbacks; PVOID LazyWriteContext; LIST_ENTRY PrivateList; @@ -197,13 +199,12 @@ typedef struct _ROS_SHARED_CACHE_MAP #define READAHEAD_DISABLED 0x1 #define WRITEBEHIND_DISABLED 0x2 +#define SHARED_CACHE_MAP_IN_CREATION 0x4 typedef struct _ROS_VACB { /* Base address of the region where the view's data is mapped. */ PVOID BaseAddress; - /* Memory area representing the region where the view's data is mapped. */ - struct _MEMORY_AREA* MemoryArea; /* Are the contents of the view valid. */ BOOLEAN Valid; /* Are the contents of the view newer than those on disk. */ @@ -316,12 +317,19 @@ NTAPI CcRosGetVacb( PROS_SHARED_CACHE_MAP SharedCacheMap, LONGLONG FileOffset, - PLONGLONG BaseOffset, - PVOID *BaseAddress, - PBOOLEAN UptoDate, PROS_VACB *Vacb ); +BOOLEAN +NTAPI +CcRosEnsureVacbResident( + _In_ PROS_VACB Vacb, + _In_ BOOLEAN Wait, + _In_ BOOLEAN NoRead, + _In_ ULONG Offset, + _In_ ULONG Length +); + VOID NTAPI CcInitView(VOID); @@ -330,14 +338,6 @@ VOID NTAPI CcShutdownLazyWriter(VOID); -NTSTATUS -NTAPI -CcReadVirtualAddress(PROS_VACB Vacb); - -NTSTATUS -NTAPI -CcWriteVirtualAddress(PROS_VACB Vacb); - BOOLEAN NTAPI CcInitializeCacheManager(VOID); @@ -415,8 +415,6 @@ NTAPI CcRosRequestVacb( PROS_SHARED_CACHE_MAP SharedCacheMap, LONGLONG FileOffset, - PVOID* BaseAddress, - PBOOLEAN UptoDate, PROS_VACB *Vacb ); diff --git a/ntoskrnl/include/internal/mm.h b/ntoskrnl/include/internal/mm.h index be8b0acc4b0..55e9fcfc29c 100644 --- a/ntoskrnl/include/internal/mm.h +++ b/ntoskrnl/include/internal/mm.h @@ -89,12 +89,9 @@ typedef ULONG_PTR SWAPENTRY; #define SEC_PHYSICALMEMORY (0x80000000) -#define MM_DATAFILE_SEGMENT (0x2) - -#define MC_CACHE (0) -#define MC_USER (1) -#define MC_SYSTEM (2) -#define MC_MAXIMUM (3) +#define MC_USER (0) +#define MC_SYSTEM (1) +#define MC_MAXIMUM (2) #define PAGED_POOL_MASK 1 #define MUST_SUCCEED_POOL_MASK 2 @@ -171,10 +168,10 @@ typedef struct _MM_SECTION_SEGMENT FAST_MUTEX Lock; /* lock which protects the page directory */ LARGE_INTEGER RawLength; /* length of the segment which is part of the mapped file */ LARGE_INTEGER Length; /* absolute length of the segment */ - ULONG ReferenceCount; - ULONG CacheCount; + PULONG ReferenceCount; + ULONG SectionCount; ULONG Protection; - ULONG Flags; + PULONG Flags; BOOLEAN WriteCopy; BOOLEAN Locked; @@ -185,6 +182,9 @@ typedef struct _MM_SECTION_SEGMENT ULONG Characteristics; } Image; + ULONG RefCount; + ULONG SegFlags; + LIST_ENTRY ListOfSegments; RTL_GENERIC_TABLE PageTable; } MM_SECTION_SEGMENT, *PMM_SECTION_SEGMENT; @@ -193,12 +193,20 @@ typedef struct _MM_IMAGE_SECTION_OBJECT { PFILE_OBJECT FileObject; + ULONG RefCount; + ULONG SegFlags; + SECTION_IMAGE_INFORMATION ImageInformation; PVOID BasedAddress; ULONG NrSegments; PMM_SECTION_SEGMENT Segments; } MM_IMAGE_SECTION_OBJECT, *PMM_IMAGE_SECTION_OBJECT; +#define MM_DATAFILE_SEGMENT (0x2) +#define MM_SEGMENT_INDELETE (0x4) +#define MM_SEGMENT_INCREATE (0x8) + + #define MA_GetStartingAddress(_MemoryArea) ((_MemoryArea)->VadNode.StartingVpn << PAGE_SHIFT) #define MA_GetEndingAddress(_MemoryArea) (((_MemoryArea)->VadNode.EndingVpn + 1) << PAGE_SHIFT) @@ -862,11 +870,6 @@ MmInitializeRmapList(VOID); VOID NTAPI MmSetCleanAllRmaps(PFN_NUMBER Page); - -VOID -NTAPI -MmSetDirtyAllRmaps(PFN_NUMBER Page); - BOOLEAN NTAPI MmIsDirtyPageRmap(PFN_NUMBER Page); @@ -1288,15 +1291,6 @@ MmNotPresentFaultSectionView( BOOLEAN Locked ); -NTSTATUS -NTAPI -MmPageOutSectionView( - PMMSUPPORT AddressSpace, - PMEMORY_AREA MemoryArea, - PVOID Address, - ULONG_PTR Entry -); - NTSTATUS NTAPI MmCreatePhysicalMemorySection(VOID); @@ -1337,6 +1331,48 @@ MmMakePagesResident( _In_ PVOID Address, _In_ ULONG Length); +NTSTATUS +NTAPI +MmMakePagesDirty( + _In_ PEPROCESS Process, + _In_ PVOID Address, + _In_ ULONG Length); + +NTSTATUS +NTAPI +MmRosFlushVirtualMemory( + _In_ PEPROCESS Process, + _Inout_ PVOID* Address, + _Inout_ PSIZE_T Length, + _Out_ PIO_STATUS_BLOCK Iosb); + +BOOLEAN +NTAPI +MmCheckDirtySegment( + PMM_SECTION_SEGMENT Segment, + PLARGE_INTEGER Offset, + BOOLEAN ForceDirty, + BOOLEAN PageOut); + +BOOLEAN +NTAPI +MmUnsharePageEntrySectionSegment(PMEMORY_AREA MemoryArea, + PMM_SECTION_SEGMENT Segment, + PLARGE_INTEGER Offset, + BOOLEAN Dirty, + BOOLEAN PageOut, + ULONG_PTR *InEntry); + +VOID +NTAPI +MmDereferenceSegment(PMM_SECTION_SEGMENT Segment); + +NTSTATUS +NTAPI +MmExtendSection( + _In_ PVOID Section, + _Inout_ PLARGE_INTEGER NewSize); + /* sysldr.c ******************************************************************/ VOID diff --git a/ntoskrnl/mm/ARM3/section.c b/ntoskrnl/mm/ARM3/section.c index 938a08d6037..9ea36637a22 100644 --- a/ntoskrnl/mm/ARM3/section.c +++ b/ntoskrnl/mm/ARM3/section.c @@ -2934,7 +2934,7 @@ MmMapViewOfArm3Section(IN PVOID SectionObject, if (!(*ViewSize)) { /* Compute it for the caller */ - CalculatedViewSize = Section->SizeOfSection.QuadPart - + CalculatedViewSize = Section->SizeOfSection.QuadPart - SectionOffset->QuadPart; /* Check if it's larger than 4GB or overflows into kernel-mode */ @@ -3891,30 +3891,24 @@ NtExtendSection(IN HANDLE SectionHandle, NULL); if (!NT_SUCCESS(Status)) return Status; - /* Really this should go in MmExtendSection */ - if (!Section->u.Flags.File || Section->u.Flags.Image) - { - DPRINT1("Not extending a file\n"); - ObDereferenceObject(Section); - return STATUS_SECTION_NOT_EXTENDED; - } - - /* FIXME: Do the work */ + Status = MmExtendSection(Section, &SafeNewMaximumSize); /* Dereference the section */ ObDereferenceObject(Section); - /* Enter SEH */ - _SEH2_TRY + if (NT_SUCCESS(Status)) { - /* Write back the new size */ - *NewMaximumSize = SafeNewMaximumSize; + _SEH2_TRY + { + /* Write back the new size */ + *NewMaximumSize = SafeNewMaximumSize; + } + _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER) + { + Status = _SEH2_GetExceptionCode(); + } + _SEH2_END; } - _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER) - { - /* Nothing to do */ - } - _SEH2_END; /* Return the status */ return STATUS_NOT_IMPLEMENTED; diff --git a/ntoskrnl/mm/ARM3/virtual.c b/ntoskrnl/mm/ARM3/virtual.c index 19eb8a69b01..00271e03e7b 100644 --- a/ntoskrnl/mm/ARM3/virtual.c +++ b/ntoskrnl/mm/ARM3/virtual.c @@ -1356,12 +1356,8 @@ MmFlushVirtualMemory(IN PEPROCESS Process, OUT PIO_STATUS_BLOCK IoStatusBlock) { PAGED_CODE(); - UNIMPLEMENTED; - - // - // Fake success - // - return STATUS_SUCCESS; + /* For now we call the old Mm */ + return MmRosFlushVirtualMemory(Process, BaseAddress, RegionSize, IoStatusBlock); } ULONG diff --git a/ntoskrnl/mm/ARM3/zeropage.c b/ntoskrnl/mm/ARM3/zeropage.c index 5fe22294199..80725e8a5a7 100644 --- a/ntoskrnl/mm/ARM3/zeropage.c +++ b/ntoskrnl/mm/ARM3/zeropage.c @@ -46,7 +46,7 @@ MmZeroPageThread(VOID) /* Get the discardable sections to free them */ MiFindInitializationCode(&StartAddress, &EndAddress); if (StartAddress) MiFreeInitializationCode(StartAddress, EndAddress); - DPRINT("Free non-cache pages: %lx\n", MmAvailablePages + MiMemoryConsumers[MC_CACHE].PagesUsed); + DPRINT("Free pages: %lx\n", MmAvailablePages); /* Set our priority to 0 */ Thread->BasePriority = 0; diff --git a/ntoskrnl/mm/balance.c b/ntoskrnl/mm/balance.c index 5cd5e267a23..29c0dfc6f47 100644 --- a/ntoskrnl/mm/balance.c +++ b/ntoskrnl/mm/balance.c @@ -54,18 +54,6 @@ MmInitializeBalancer(ULONG NrAvailablePages, ULONG NrSystemPages) /* Set up targets. */ MiMinimumAvailablePages = 256; MiMinimumPagesPerRun = 256; - if ((NrAvailablePages + NrSystemPages) >= 8192) - { - MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 4 * 3; - } - else if ((NrAvailablePages + NrSystemPages) >= 4096) - { - MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 3 * 2; - } - else - { - MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 8; - } MiMemoryConsumers[MC_USER].PagesTarget = NrAvailablePages - MiMinimumAvailablePages; } diff --git a/ntoskrnl/mm/pagefile.c b/ntoskrnl/mm/pagefile.c index 37870520998..e0f9e087150 100644 --- a/ntoskrnl/mm/pagefile.c +++ b/ntoskrnl/mm/pagefile.c @@ -94,9 +94,6 @@ NTAPI MmBuildMdlFromPages(PMDL Mdl, PPFN_NUMBER Pages) { memcpy(Mdl + 1, Pages, sizeof(PFN_NUMBER) * (PAGE_ROUND_UP(Mdl->ByteOffset+Mdl->ByteCount)/PAGE_SIZE)); - - /* FIXME: this flag should be set by the caller perhaps? */ - Mdl->MdlFlags |= MDL_IO_PAGE_READ; } @@ -230,7 +227,7 @@ MiReadPageFile( MmInitializeMdl(Mdl, NULL, PAGE_SIZE); MmBuildMdlFromPages(Mdl, &Page); - Mdl->MdlFlags |= MDL_PAGES_LOCKED; + Mdl->MdlFlags |= MDL_PAGES_LOCKED | MDL_IO_PAGE_READ; file_offset.QuadPart = PageFileOffset * PAGE_SIZE; diff --git a/ntoskrnl/mm/rmap.c b/ntoskrnl/mm/rmap.c index 9f5938d7496..d1cfb584dce 100644 --- a/ntoskrnl/mm/rmap.c +++ b/ntoskrnl/mm/rmap.c @@ -19,7 +19,6 @@ /* GLOBALS ******************************************************************/ static NPAGED_LOOKASIDE_LIST RmapLookasideList; -FAST_MUTEX RmapListLock; /* FUNCTIONS ****************************************************************/ @@ -38,7 +37,6 @@ VOID NTAPI MmInitializeRmapList(VOID) { - ExInitializeFastMutex(&RmapListLock); ExInitializeNPagedLookasideList (&RmapLookasideList, NULL, RmapListFree, @@ -55,37 +53,27 @@ MmPageOutPhysicalAddress(PFN_NUMBER Page) PMM_RMAP_ENTRY entry; PMEMORY_AREA MemoryArea; PMMSUPPORT AddressSpace; - ULONG Type; PVOID Address; PEPROCESS Process; - ULONGLONG Offset; NTSTATUS Status = STATUS_SUCCESS; + PMM_SECTION_SEGMENT Segment; + LARGE_INTEGER SegmentOffset; + KIRQL OldIrql; + + OldIrql = MiAcquirePfnLock(); - ExAcquireFastMutex(&RmapListLock); entry = MmGetRmapListHeadPage(Page); -#ifdef NEWCC - // Special case for NEWCC: we can have a page that's only in a segment - // page table - if (entry && RMAP_IS_SEGMENT(entry->Address) && entry->Next == NULL) - { - /* NEWCC does locking itself */ - ExReleaseFastMutex(&RmapListLock); - return MmpPageOutPhysicalAddress(Page); - } -#endif - while (entry && RMAP_IS_SEGMENT(entry->Address)) entry = entry->Next; if (entry == NULL) { - ExReleaseFastMutex(&RmapListLock); - return(STATUS_UNSUCCESSFUL); + MiReleasePfnLock(OldIrql); + return STATUS_UNSUCCESSFUL; } Process = entry->Process; - Address = entry->Address; if ((((ULONG_PTR)Address) & 0xFFF) != 0) @@ -97,12 +85,12 @@ MmPageOutPhysicalAddress(PFN_NUMBER Page) { if (!ExAcquireRundownProtection(&Process->RundownProtect)) { - ExReleaseFastMutex(&RmapListLock); + MiReleasePfnLock(OldIrql); return STATUS_PROCESS_IS_TERMINATING; } Status = ObReferenceObjectByPointer(Process, PROCESS_ALL_ACCESS, NULL, KernelMode); - ExReleaseFastMutex(&RmapListLock); + MiReleasePfnLock(OldIrql); if (!NT_SUCCESS(Status)) { ExReleaseRundownProtection(&Process->RundownProtect); @@ -112,11 +100,12 @@ MmPageOutPhysicalAddress(PFN_NUMBER Page) } else { - ExReleaseFastMutex(&RmapListLock); + MiReleasePfnLock(OldIrql); AddressSpace = MmGetKernelAddressSpace(); } MmLockAddressSpace(AddressSpace); + MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace, Address); if (MemoryArea == NULL || MemoryArea->DeleteInProgress) { @@ -128,23 +117,27 @@ MmPageOutPhysicalAddress(PFN_NUMBER Page) } return(STATUS_UNSUCCESSFUL); } - Type = MemoryArea->Type; - if (Type == MEMORY_AREA_SECTION_VIEW) + + if (MemoryArea->Type == MEMORY_AREA_SECTION_VIEW) { ULONG_PTR Entry; - Offset = MemoryArea->SectionData.ViewOffset.QuadPart + + BOOLEAN Dirty; + PFN_NUMBER MapPage; + LARGE_INTEGER Offset; + BOOLEAN Released; + + Offset.QuadPart = MemoryArea->SectionData.ViewOffset.QuadPart + ((ULONG_PTR)Address - MA_GetStartingAddress(MemoryArea)); - MmLockSectionSegment(MemoryArea->SectionData.Segment); + Segment = MemoryArea->SectionData.Segment; - /* - * Get or create a pageop - */ - Entry = MmGetPageEntrySectionSegment(MemoryArea->SectionData.Segment, - (PLARGE_INTEGER)&Offset); + MmLockSectionSegment(Segment); + + Entry = MmGetPageEntrySectionSegment(Segment, &Offset); if (Entry && MM_IS_WAIT_PTE(Entry)) { - MmUnlockSectionSegment(MemoryArea->SectionData.Segment); + /* The segment is being read or something. Give up */ + MmUnlockSectionSegment(Segment); MmUnlockAddressSpace(AddressSpace); if (Address < MmSystemRangeStart) { @@ -154,18 +147,101 @@ MmPageOutPhysicalAddress(PFN_NUMBER Page) return(STATUS_UNSUCCESSFUL); } - MmSetPageEntrySectionSegment(MemoryArea->SectionData.Segment, (PLARGE_INTEGER)&Offset, MAKE_SWAP_SSE(MM_WAIT_ENTRY)); + /* Delete this virtual mapping in the process */ + MmDeleteVirtualMapping(Process, Address, &Dirty, &MapPage); + ASSERT(MapPage == Page); - /* - * Release locks now we have a page op. - */ - MmUnlockSectionSegment(MemoryArea->SectionData.Segment); + if (Page != PFN_FROM_SSE(Entry)) + { + SWAPENTRY SwapEntry; + + /* This page is private to the process */ + MmUnlockSectionSegment(Segment); + + /* Check if we should write it back to the page file */ + SwapEntry = MmGetSavedSwapEntryPage(Page); + + if ((SwapEntry == 0) && Dirty) + { + /* We don't have a Swap entry, yet the page is dirty. Get one */ + SwapEntry = MmAllocSwapPage(); + if (!SwapEntry) + { + PMM_REGION Region = MmFindRegion((PVOID)MA_GetStartingAddress(MemoryArea), + &MemoryArea->SectionData.RegionListHead, + Address, NULL); + + /* We can't, so let this page in the Process VM */ + MmCreateVirtualMapping(Process, Address, Region->Protect, &Page, 1); + MmSetDirtyPage(Process, Address); + + MmUnlockAddressSpace(AddressSpace); + return STATUS_UNSUCCESSFUL; + } + } + + if (Dirty) + { + Status = MmWriteToSwapPage(SwapEntry, Page); + if (!NT_SUCCESS(Status)) + { + /* We failed at saving the content of this page. Keep it in */ + PMM_REGION Region = MmFindRegion((PVOID)MA_GetStartingAddress(MemoryArea), + &MemoryArea->SectionData.RegionListHead, + Address, NULL); + + /* This Swap Entry is useless to us */ + MmSetSavedSwapEntryPage(Page, 0); + MmFreeSwapPage(SwapEntry); + + /* We can't, so let this page in the Process VM */ + MmCreateVirtualMapping(Process, Address, Region->Protect, &Page, 1); + MmSetDirtyPage(Process, Address); + + MmUnlockAddressSpace(AddressSpace); + return STATUS_UNSUCCESSFUL; + } + } + + if (SwapEntry) + { + /* Keep this in the process VM */ + MmCreatePageFileMapping(Process, Address, SwapEntry); + MmSetSavedSwapEntryPage(Page, 0); + } + + MmUnlockAddressSpace(AddressSpace); + + /* We can finally let this page go */ + MmDeleteRmap(Page, Process, Address); + MmReleasePageMemoryConsumer(MC_USER, Page); + + ASSERT(MmGetRmapListHeadPage(Page) == NULL); + + if (Address < MmSystemRangeStart) + { + ExReleaseRundownProtection(&Process->RundownProtect); + ObDereferenceObject(Process); + } + return STATUS_SUCCESS; + } + + /* Delete this RMAP */ + MmDeleteRmap(Page, Process, Address); + + /* One less mapping referencing this segment */ + Released = MmUnsharePageEntrySectionSegment(MemoryArea, Segment, &Offset, Dirty, FALSE, NULL); + + MmUnlockSectionSegment(Segment); MmUnlockAddressSpace(AddressSpace); - /* - * Do the actual page out work. - */ - Status = MmPageOutSectionView(AddressSpace, MemoryArea, Address, Entry); + if (Address < MmSystemRangeStart) + { + ExReleaseRundownProtection(&Process->RundownProtect); + ObDereferenceObject(Process); + } + + if (Released) return STATUS_SUCCESS; } #ifdef NEWCC else if (Type == MEMORY_AREA_CACHE) @@ -185,7 +261,29 @@ MmPageOutPhysicalAddress(PFN_NUMBER Page) ExReleaseRundownProtection(&Process->RundownProtect); ObDereferenceObject(Process); } - return(Status); + + /* Now write this page to file, if needed */ + Segment = MmGetSectionAssociation(Page, &SegmentOffset); + if (Segment) + { + BOOLEAN Released; + + MmLockSectionSegment(Segment); + + Released = MmCheckDirtySegment(Segment, &SegmentOffset, FALSE, TRUE); + + MmUnlockSectionSegment(Segment); + + MmDereferenceSegment(Segment); + + if (Released) + { + return STATUS_SUCCESS; + } + } + + /* If we are here, then we didn't release the page */ + return STATUS_UNSUCCESSFUL; } VOID @@ -193,12 +291,13 @@ NTAPI MmSetCleanAllRmaps(PFN_NUMBER Page) { PMM_RMAP_ENTRY current_entry; + KIRQL OldIrql; - ExAcquireFastMutex(&RmapListLock); + OldIrql = MiAcquirePfnLock(); current_entry = MmGetRmapListHeadPage(Page); if (current_entry == NULL) { - DPRINT1("MmIsDirtyRmap: No rmaps.\n"); + DPRINT1("MmSetCleanAllRmaps: No rmaps.\n"); KeBugCheck(MEMORY_MANAGEMENT); } while (current_entry != NULL) @@ -207,29 +306,7 @@ MmSetCleanAllRmaps(PFN_NUMBER Page) MmSetCleanPage(current_entry->Process, current_entry->Address); current_entry = current_entry->Next; } - ExReleaseFastMutex(&RmapListLock); -} - -VOID -NTAPI -MmSetDirtyAllRmaps(PFN_NUMBER Page) -{ - PMM_RMAP_ENTRY current_entry; - - ExAcquireFastMutex(&RmapListLock); - current_entry = MmGetRmapListHeadPage(Page); - if (current_entry == NULL) - { - DPRINT1("MmIsDirtyRmap: No rmaps.\n"); - KeBugCheck(MEMORY_MANAGEMENT); - } - while (current_entry != NULL) - { - if (!RMAP_IS_SEGMENT(current_entry->Address)) - MmSetDirtyPage(current_entry->Process, current_entry->Address); - current_entry = current_entry->Next; - } - ExReleaseFastMutex(&RmapListLock); + MiReleasePfnLock(OldIrql); } BOOLEAN @@ -237,27 +314,31 @@ NTAPI MmIsDirtyPageRmap(PFN_NUMBER Page) { PMM_RMAP_ENTRY current_entry; + KIRQL OldIrql; + BOOLEAN Dirty = FALSE; - ExAcquireFastMutex(&RmapListLock); + OldIrql = MiAcquirePfnLock(); current_entry = MmGetRmapListHeadPage(Page); if (current_entry == NULL) { - ExReleaseFastMutex(&RmapListLock); - return(FALSE); + DPRINT1("MmIsDirtyPageRmap: No rmaps.\n"); + KeBugCheck(MEMORY_MANAGEMENT); } while (current_entry != NULL) { - if ( - !RMAP_IS_SEGMENT(current_entry->Address) && - MmIsDirtyPage(current_entry->Process, current_entry->Address)) + if (!RMAP_IS_SEGMENT(current_entry->Address)) { - ExReleaseFastMutex(&RmapListLock); - return(TRUE); + if (MmIsDirtyPage(current_entry->Process, current_entry->Address)) + { + Dirty = TRUE; + break; + } } current_entry = current_entry->Next; } - ExReleaseFastMutex(&RmapListLock); - return(FALSE); + MiReleasePfnLock(OldIrql); + + return Dirty; } VOID @@ -268,6 +349,8 @@ MmInsertRmap(PFN_NUMBER Page, PEPROCESS Process, PMM_RMAP_ENTRY current_entry; PMM_RMAP_ENTRY new_entry; ULONG PrevSize; + KIRQL OldIrql; + if (!RMAP_IS_SEGMENT(Address)) Address = (PVOID)PAGE_ROUND_DOWN(Address); @@ -298,7 +381,7 @@ MmInsertRmap(PFN_NUMBER Page, PEPROCESS Process, KeBugCheck(MEMORY_MANAGEMENT); } - ExAcquireFastMutex(&RmapListLock); + OldIrql = MiAcquirePfnLock(); current_entry = MmGetRmapListHeadPage(Page); new_entry->Next = current_entry; #if DBG @@ -318,7 +401,8 @@ MmInsertRmap(PFN_NUMBER Page, PEPROCESS Process, } #endif MmSetRmapListHeadPage(Page, new_entry); - ExReleaseFastMutex(&RmapListLock); + MiReleasePfnLock(OldIrql); + if (!RMAP_IS_SEGMENT(Address)) { if (Process == NULL) @@ -336,63 +420,15 @@ MmInsertRmap(PFN_NUMBER Page, PEPROCESS Process, } } -VOID -NTAPI -MmDeleteAllRmaps(PFN_NUMBER Page, PVOID Context, - VOID (*DeleteMapping)(PVOID Context, PEPROCESS Process, - PVOID Address)) -{ - PMM_RMAP_ENTRY current_entry; - PMM_RMAP_ENTRY previous_entry; - PEPROCESS Process; - - ExAcquireFastMutex(&RmapListLock); - current_entry = MmGetRmapListHeadPage(Page); - if (current_entry == NULL) - { - DPRINT1("MmDeleteAllRmaps: No rmaps.\n"); - KeBugCheck(MEMORY_MANAGEMENT); - } - MmSetRmapListHeadPage(Page, NULL); - ExReleaseFastMutex(&RmapListLock); - - while (current_entry != NULL) - { - previous_entry = current_entry; - current_entry = current_entry->Next; - if (!RMAP_IS_SEGMENT(previous_entry->Address)) - { - if (DeleteMapping) - { - DeleteMapping(Context, previous_entry->Process, - previous_entry->Address); - } - Process = previous_entry->Process; - ExFreeToNPagedLookasideList(&RmapLookasideList, previous_entry); - if (Process == NULL) - { - Process = PsInitialSystemProcess; - } - if (Process) - { - (void)InterlockedExchangeAddUL(&Process->Vm.WorkingSetSize, -PAGE_SIZE); - } - } - else - { - ExFreeToNPagedLookasideList(&RmapLookasideList, previous_entry); - } - } -} - VOID NTAPI MmDeleteRmap(PFN_NUMBER Page, PEPROCESS Process, PVOID Address) { PMM_RMAP_ENTRY current_entry, previous_entry; + KIRQL OldIrql; - ExAcquireFastMutex(&RmapListLock); + OldIrql = MiAcquirePfnLock(); previous_entry = NULL; current_entry = MmGetRmapListHeadPage(Page); @@ -409,7 +445,8 @@ MmDeleteRmap(PFN_NUMBER Page, PEPROCESS Process, { previous_entry->Next = current_entry->Next; } - ExReleaseFastMutex(&RmapListLock); + MiReleasePfnLock(OldIrql); + ExFreeToNPagedLookasideList(&RmapLookasideList, current_entry); if (!RMAP_IS_SEGMENT(Address)) { @@ -450,8 +487,8 @@ MmGetSegmentRmap(PFN_NUMBER Page, PULONG RawOffset) { PCACHE_SECTION_PAGE_TABLE Result = NULL; PMM_RMAP_ENTRY current_entry;//, previous_entry; + KIRQL OldIrql = MiAcquirePfnLock(); - ExAcquireFastMutex(&RmapListLock); //previous_entry = NULL; current_entry = MmGetRmapListHeadPage(Page); while (current_entry != NULL) @@ -460,14 +497,20 @@ MmGetSegmentRmap(PFN_NUMBER Page, PULONG RawOffset) { Result = (PCACHE_SECTION_PAGE_TABLE)current_entry->Process; *RawOffset = (ULONG_PTR)current_entry->Address & ~RMAP_SEGMENT_MASK; - InterlockedIncrementUL(&Result->Segment->ReferenceCount); - ExReleaseFastMutex(&RmapListLock); + if (*Result->Segment->Flags & MM_SEGMENT_INDELETE) + { + MiReleasePfnLock(OldIrql); + return NULL; + } + + InterlockedIncrementUL(Result->Segment->ReferenceCount); + MiReleasePfnLock(OldIrql); return Result; } //previous_entry = current_entry; current_entry = current_entry->Next; } - ExReleaseFastMutex(&RmapListLock); + MiReleasePfnLock(OldIrql); return NULL; } @@ -482,8 +525,8 @@ NTAPI MmDeleteSectionAssociation(PFN_NUMBER Page) { PMM_RMAP_ENTRY current_entry, previous_entry; + KIRQL OldIrql = MiAcquirePfnLock(); - ExAcquireFastMutex(&RmapListLock); previous_entry = NULL; current_entry = MmGetRmapListHeadPage(Page); while (current_entry != NULL) @@ -498,12 +541,12 @@ MmDeleteSectionAssociation(PFN_NUMBER Page) { previous_entry->Next = current_entry->Next; } - ExReleaseFastMutex(&RmapListLock); + MiReleasePfnLock(OldIrql); ExFreeToNPagedLookasideList(&RmapLookasideList, current_entry); return; } previous_entry = current_entry; current_entry = current_entry->Next; } - ExReleaseFastMutex(&RmapListLock); + MiReleasePfnLock(OldIrql); } diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index 4d29f5a9ff4..39ae8bf75f5 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -193,6 +193,58 @@ static GENERIC_MAPPING MmpSectionMapping = /* FUNCTIONS *****************************************************************/ + +NTSTATUS +NTAPI +MiWritePage(PMM_SECTION_SEGMENT Segment, + LONGLONG SegOffset, + PFN_NUMBER Page) +/* + * FUNCTION: write a page for a section backed memory area. + * PARAMETERS: + * MemoryArea - Memory area to write the page for. + * Offset - Offset of the page to write. + * Page - Page which contains the data to write. + */ +{ + NTSTATUS Status; + IO_STATUS_BLOCK IoStatus; + KEVENT Event; + UCHAR MdlBase[sizeof(MDL) + sizeof(PFN_NUMBER)]; + PMDL Mdl = (PMDL)MdlBase; + PFILE_OBJECT FileObject = Segment->FileObject; + LARGE_INTEGER FileOffset; + PFSRTL_COMMON_FCB_HEADER Fcb = FileObject->FsContext; + + FileOffset.QuadPart = Segment->Image.FileOffset + SegOffset; + + /* Check if we are not writing off-limit */ + if (FileOffset.QuadPart >= Fcb->AllocationSize.QuadPart) + { + return STATUS_SUCCESS; + } + + RtlZeroMemory(MdlBase, sizeof(MdlBase)); + MmInitializeMdl(Mdl, NULL, PAGE_SIZE); + MmBuildMdlFromPages(Mdl, &Page); + Mdl->MdlFlags |= MDL_PAGES_LOCKED; + + KeInitializeEvent(&Event, NotificationEvent, FALSE); + Status = IoSynchronousPageWrite(FileObject, Mdl, &FileOffset, &Event, &IoStatus); + if (Status == STATUS_PENDING) + { + KeWaitForSingleObject(&Event, Executive, KernelMode, FALSE, NULL); + Status = IoStatus.Status; + } + if (Mdl->MdlFlags & MDL_MAPPED_TO_SYSTEM_VA) + { + MmUnmapLockedPages (Mdl->MappedSystemVa, Mdl); + } + + return Status; +} + + /* References: [1] Microsoft Corporation, "Microsoft Portable Executable and Common Object @@ -713,6 +765,8 @@ l_ReadHeaderFromFile: nStatus = STATUS_INVALID_IMAGE_FORMAT; + ASSERT(ImageSectionObject->RefCount > 0); + /* convert the executable sections into segments. See also [1], section 4 */ for(i = 0; i < ImageSectionObject->NrSegments - 1; ++ i) { @@ -824,51 +878,146 @@ MmspWaitForFileLock(PFILE_OBJECT File) //return KeWaitForSingleObject(&File->Lock, 0, KernelMode, FALSE, NULL); } + + VOID NTAPI -MmFreeSectionSegments(PFILE_OBJECT FileObject) +MmpFreePageFileSegment(PMM_SECTION_SEGMENT Segment) { - if (FileObject->SectionObjectPointer->ImageSectionObject != NULL) + ULONG Length; + LARGE_INTEGER Offset; + ULONG_PTR Entry; + SWAPENTRY SavedSwapEntry; + PFN_NUMBER Page; + + Page = 0; + + MmLockSectionSegment(Segment); + + Length = PAGE_ROUND_UP(Segment->Length.QuadPart); + for (Offset.QuadPart = 0; Offset.QuadPart < Length; Offset.QuadPart += PAGE_SIZE) { - PMM_IMAGE_SECTION_OBJECT ImageSectionObject; + Entry = MmGetPageEntrySectionSegment(Segment, &Offset); + if (Entry) + { + MmSetPageEntrySectionSegment(Segment, &Offset, 0); + if (IS_SWAP_FROM_SSE(Entry)) + { + MmFreeSwapPage(SWAPENTRY_FROM_SSE(Entry)); + } + else + { + Page = PFN_FROM_SSE(Entry); + SavedSwapEntry = MmGetSavedSwapEntryPage(Page); + if (SavedSwapEntry != 0) + { + MmSetSavedSwapEntryPage(Page, 0); + MmFreeSwapPage(SavedSwapEntry); + } + MmReleasePageMemoryConsumer(MC_USER, Page); + } + } + } + + MmUnlockSectionSegment(Segment); +} + +static +VOID +NTAPI +FreeSegmentPage(PMM_SECTION_SEGMENT Segment, PLARGE_INTEGER Offset) +{ + ULONG_PTR Entry; + PFN_NUMBER Page; + + MmLockSectionSegment(Segment); + + Entry = MmGetPageEntrySectionSegment(Segment, Offset); + + MmUnlockSectionSegment(Segment); + + /* This must be either a valid entry or nothing */ + ASSERT(!IS_SWAP_FROM_SSE(Entry)); + + /* There should be no reference anymore */ + ASSERT(SHARE_COUNT_FROM_SSE(Entry) == 0); + + Page = PFN_FROM_SSE(Entry); + /* If there is a page, this must be because it's still dirty */ + ASSERT(Page != 0); + + /* Write the page */ + if (IS_DIRTY_SSE(Entry)) + MiWritePage(Segment, Offset->QuadPart, Page); + + MmReleasePageMemoryConsumer(MC_USER, Page); +} + +VOID +NTAPI +MmDereferenceSegment(PMM_SECTION_SEGMENT Segment) +{ + KIRQL OldIrql; + + /* Lock the PFN lock because we mess around with SectionObjectPointers */ + OldIrql = MiAcquirePfnLock(); + + if (InterlockedDecrementUL(Segment->ReferenceCount) > 0) + { + /* Nothing to do yet */ + MiReleasePfnLock(OldIrql); + return; + } + + *Segment->Flags |= MM_SEGMENT_INDELETE; + + MiReleasePfnLock(OldIrql); + + /* Flush the segment */ + if (*Segment->Flags & MM_DATAFILE_SEGMENT) + { + /* Free the page table. This will flush any remaining dirty data */ + MmFreePageTablesSectionSegment(Segment, FreeSegmentPage); + + OldIrql = MiAcquirePfnLock(); + /* Delete the pointer on the file */ + ASSERT(Segment->FileObject->SectionObjectPointer->DataSectionObject == Segment); + Segment->FileObject->SectionObjectPointer->DataSectionObject = NULL; + MiReleasePfnLock(OldIrql); + ObDereferenceObject(Segment->FileObject); + + ExFreePoolWithTag(Segment, TAG_MM_SECTION_SEGMENT); + } + else + { + /* Most grotesque thing ever */ + PMM_IMAGE_SECTION_OBJECT ImageSectionObject = CONTAINING_RECORD(Segment->ReferenceCount, MM_IMAGE_SECTION_OBJECT, RefCount); PMM_SECTION_SEGMENT SectionSegments; ULONG NrSegments; ULONG i; - ImageSectionObject = (PMM_IMAGE_SECTION_OBJECT)FileObject->SectionObjectPointer->ImageSectionObject; + OldIrql = MiAcquirePfnLock(); + /* Delete the pointer on the file */ + ASSERT(ImageSectionObject->FileObject->SectionObjectPointer->ImageSectionObject == ImageSectionObject); + ImageSectionObject->FileObject->SectionObjectPointer->ImageSectionObject = NULL; + MiReleasePfnLock(OldIrql); + + ObDereferenceObject(ImageSectionObject->FileObject); + NrSegments = ImageSectionObject->NrSegments; SectionSegments = ImageSectionObject->Segments; for (i = 0; i < NrSegments; i++) { - if (SectionSegments[i].ReferenceCount != 0) + if (SectionSegments[i].Image.Characteristics & IMAGE_SCN_MEM_SHARED) { - DPRINT1("Image segment %lu still referenced (was %lu)\n", i, - SectionSegments[i].ReferenceCount); - KeBugCheck(MEMORY_MANAGEMENT); + MmpFreePageFileSegment(&SectionSegments[i]); } + MmFreePageTablesSectionSegment(&SectionSegments[i], NULL); } - ObDereferenceObject(ImageSectionObject->FileObject); - ExFreePool(ImageSectionObject->Segments); - ExFreePool(ImageSectionObject); - FileObject->SectionObjectPointer->ImageSectionObject = NULL; - } - if (FileObject->SectionObjectPointer->DataSectionObject != NULL) - { - PMM_SECTION_SEGMENT Segment; - Segment = (PMM_SECTION_SEGMENT)FileObject->SectionObjectPointer-> - DataSectionObject; - - if (Segment->ReferenceCount != 0) - { - DPRINT1("Data segment still referenced\n"); - KeBugCheck(MEMORY_MANAGEMENT); - } - ObDereferenceObject(Segment->FileObject); - MmFreePageTablesSectionSegment(Segment, NULL); - ExFreePool(Segment); - FileObject->SectionObjectPointer->DataSectionObject = NULL; + ExFreePoolWithTag(ImageSectionObject->Segments, TAG_MM_SECTION_SEGMENT); + ExFreePoolWithTag(ImageSectionObject, TAG_MM_SECTION_SEGMENT); } } @@ -908,7 +1057,9 @@ MmUnsharePageEntrySectionSegment(PMEMORY_AREA MemoryArea, ULONG_PTR *InEntry) { ULONG_PTR Entry = InEntry ? *InEntry : MmGetPageEntrySectionSegment(Segment, Offset); - BOOLEAN IsDirectMapped = FALSE; + PFN_NUMBER Page = PFN_FROM_SSE(Entry); + ULONG_PTR NewEntry = 0; + SWAPENTRY SwapEntry; if (Entry == 0) { @@ -924,142 +1075,66 @@ MmUnsharePageEntrySectionSegment(PMEMORY_AREA MemoryArea, { KeBugCheck(MEMORY_MANAGEMENT); } + Dirty = Dirty || IS_DIRTY_SSE(Entry); Entry = MAKE_SSE(PAGE_FROM_SSE(Entry), SHARE_COUNT_FROM_SSE(Entry) - 1); - /* - * If we reducing the share count of this entry to zero then set the entry - * to zero and tell the cache the page is no longer mapped. - */ - if (SHARE_COUNT_FROM_SSE(Entry) == 0) - { - PFILE_OBJECT FileObject; - SWAPENTRY SavedSwapEntry; - PFN_NUMBER Page; -#ifndef NEWCC - PROS_SHARED_CACHE_MAP SharedCacheMap; - BOOLEAN IsImageSection; - LARGE_INTEGER FileOffset; - - FileOffset.QuadPart = Offset->QuadPart + Segment->Image.FileOffset; - IsImageSection = MemoryArea->VadNode.u.VadFlags.VadType == VadImageMap; -#endif - - Page = PFN_FROM_SSE(Entry); - FileObject = Segment->FileObject; - if (FileObject != NULL && - !(Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) - { - -#ifndef NEWCC - if ((FileOffset.QuadPart % PAGE_SIZE) == 0 && - (Offset->QuadPart + PAGE_SIZE <= Segment->RawLength.QuadPart || !IsImageSection)) - { - NTSTATUS Status; - SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; - IsDirectMapped = TRUE; -#ifndef NEWCC - Status = CcRosUnmapVacb(SharedCacheMap, FileOffset.QuadPart, Dirty); -#else - Status = STATUS_SUCCESS; -#endif - if (!NT_SUCCESS(Status)) - { - DPRINT1("CcRosUnmapVacb failed, status = %x\n", Status); - KeBugCheck(MEMORY_MANAGEMENT); - } - } -#endif - } - - SavedSwapEntry = MmGetSavedSwapEntryPage(Page); - if (SavedSwapEntry == 0) - { - if (!PageOut && (Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) - { - /* - * FIXME: - * Try to page out this page and set the swap entry - * within the section segment. There exist no rmap entry - * for this page. The pager thread can't page out a - * page without a rmap entry. - */ - MmSetPageEntrySectionSegment(Segment, Offset, Entry); - if (InEntry) *InEntry = Entry; - MiSetPageEvent(NULL, NULL); - } - else - { - MmSetPageEntrySectionSegment(Segment, Offset, 0); - if (InEntry) *InEntry = 0; - MiSetPageEvent(NULL, NULL); - if (!IsDirectMapped) - { - MmReleasePageMemoryConsumer(MC_USER, Page); - } - } - } - else - { - if (Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED) - { - if (!PageOut) - { - if (Dirty) - { - /* - * FIXME: - * We hold all locks. Nobody can do something with the current - * process and the current segment (also not within an other process). - */ - NTSTATUS Status; - Status = MmWriteToSwapPage(SavedSwapEntry, Page); - if (!NT_SUCCESS(Status)) - { - DPRINT1("MM: Failed to write to swap page (Status was 0x%.8X)\n", Status); - KeBugCheck(MEMORY_MANAGEMENT); - } - } - MmSetPageEntrySectionSegment(Segment, Offset, MAKE_SWAP_SSE(SavedSwapEntry)); - if (InEntry) *InEntry = MAKE_SWAP_SSE(SavedSwapEntry); - MmSetSavedSwapEntryPage(Page, 0); - MiSetPageEvent(NULL, NULL); - } - MmReleasePageMemoryConsumer(MC_USER, Page); - } - else - { - DPRINT1("Found a swapentry for a non private page in an image or data file sgment\n"); - KeBugCheck(MEMORY_MANAGEMENT); - } - } - } - else + if (Dirty) Entry = DIRTY_SSE(Entry); + + if (SHARE_COUNT_FROM_SSE(Entry) > 0) { + /* Update the page mapping in the segment and we're done */ if (InEntry) *InEntry = Entry; else MmSetPageEntrySectionSegment(Segment, Offset, Entry); + return FALSE; } - return(SHARE_COUNT_FROM_SSE(Entry) > 0); -} -BOOLEAN MiIsPageFromCache(PMEMORY_AREA MemoryArea, - LONGLONG SegOffset) -{ -#ifndef NEWCC - if (!(MemoryArea->SectionData.Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) + if (IS_DIRTY_SSE(Entry) && !(Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) { - PROS_SHARED_CACHE_MAP SharedCacheMap; - PROS_VACB Vacb; - SharedCacheMap = MemoryArea->SectionData.Segment->FileObject->SectionObjectPointer->SharedCacheMap; - Vacb = CcRosLookupVacb(SharedCacheMap, SegOffset + MemoryArea->SectionData.Segment->Image.FileOffset); - if (Vacb) + ASSERT(!Segment->WriteCopy); + ASSERT(MmGetSavedSwapEntryPage(Page) == 0); + + /* The entry must be written back to the disk, so let this in the segment, the page-out thread will take care of this */ + MmSetPageEntrySectionSegment(Segment, Offset, Entry); + return FALSE; + } + + SwapEntry = MmGetSavedSwapEntryPage(Page); + if (Dirty && !SwapEntry) + { + SwapEntry = MmAllocSwapPage(); + if (!SwapEntry) { - CcRosReleaseVacb(SharedCacheMap, Vacb, Vacb->Valid, FALSE, TRUE); - return TRUE; + /* We can't have a swap entry for this page. Let the segment keep it */ + MmSetPageEntrySectionSegment(Segment, Offset, Entry); + return FALSE; } } -#endif - return FALSE; + + if (Dirty) + { + NTSTATUS Status = MmWriteToSwapPage(SwapEntry, Page); + if (!NT_SUCCESS(Status)) + { + /* We failed. Clean up */ + MmSetSavedSwapEntryPage(Page, 0); + MmFreeSwapPage(SwapEntry); + MmSetPageEntrySectionSegment(Segment, Offset, Entry); + return FALSE; + } + } + + if (SwapEntry) + { + NewEntry = MAKE_SWAP_SSE(SwapEntry); + MmSetSavedSwapEntryPage(Page, 0); + } + + /* We can let this go */ + MmSetPageEntrySectionSegment(Segment, Offset, NewEntry); + MmReleasePageMemoryConsumer(MC_USER, Page); + MiSetPageEvent(NULL, NULL); + return TRUE; } NTSTATUS @@ -1097,175 +1172,46 @@ MiReadPage(PMEMORY_AREA MemoryArea, * Page - Variable that receives a page contains the read data. */ { - LONGLONG BaseOffset; - LONGLONG FileOffset; - PVOID BaseAddress; - BOOLEAN UptoDate; - PROS_VACB Vacb; - PFILE_OBJECT FileObject; NTSTATUS Status; - LONGLONG RawLength; - PROS_SHARED_CACHE_MAP SharedCacheMap; - BOOLEAN IsImageSection; - LONGLONG Length; + IO_STATUS_BLOCK IoStatus; + KEVENT Event; + UCHAR MdlBase[sizeof(MDL) + sizeof(PFN_NUMBER)]; + PMDL Mdl = (PMDL)MdlBase; + PFILE_OBJECT FileObject = MemoryArea->SectionData.Segment->FileObject; + LARGE_INTEGER FileOffset; + PFSRTL_COMMON_FCB_HEADER Fcb = FileObject->FsContext; - FileObject = MemoryArea->SectionData.Segment->FileObject; - SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; - RawLength = MemoryArea->SectionData.Segment->RawLength.QuadPart; - FileOffset = SegOffset + MemoryArea->SectionData.Segment->Image.FileOffset; - IsImageSection = MemoryArea->VadNode.u.VadFlags.VadType == VadImageMap; + FileOffset.QuadPart = MemoryArea->SectionData.Segment->Image.FileOffset + SegOffset; - ASSERT(SharedCacheMap); + Status = MmRequestPageMemoryConsumer(MC_USER, FALSE, Page); + if (!NT_SUCCESS(Status)) + return Status; - DPRINT("%S %I64x\n", FileObject->FileName.Buffer, FileOffset); + /* Check if we are beyond the file */ + if (FileOffset.QuadPart > Fcb->FileSize.QuadPart) + return STATUS_SUCCESS; - /* - * If the file system is letting us go directly to the cache and the - * memory area was mapped at an offset in the file which is page aligned - * then get the related VACB. - */ - if (((FileOffset % PAGE_SIZE) == 0) && - ((SegOffset + PAGE_SIZE <= RawLength) || !IsImageSection) && - !(MemoryArea->SectionData.Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) + RtlZeroMemory(MdlBase, sizeof(MdlBase)); + MmInitializeMdl(Mdl, NULL, PAGE_SIZE); + MmBuildMdlFromPages(Mdl, Page); + Mdl->MdlFlags |= MDL_PAGES_LOCKED | MDL_IO_PAGE_READ; + + KeInitializeEvent(&Event, NotificationEvent, FALSE); + + Status = IoPageRead(FileObject, Mdl, &FileOffset, &Event, &IoStatus); + if (Status == STATUS_PENDING) { - - /* - * Get the related VACB; we use a lower level interface than - * filesystems do because it is safe for us to use an offset with an - * alignment less than the file system block size. - */ - Status = CcRosGetVacb(SharedCacheMap, - FileOffset, - &BaseOffset, - &BaseAddress, - &UptoDate, - &Vacb); - if (!NT_SUCCESS(Status)) - { - return(Status); - } - if (!UptoDate) - { - /* - * If the VACB isn't up to date then call the file - * system to read in the data. - */ - Status = CcReadVirtualAddress(Vacb); - if (!NT_SUCCESS(Status)) - { - CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE, FALSE); - return Status; - } - } - - /* Probe the page, since it's PDE might not be synced */ - (void)*((volatile char*)BaseAddress + FileOffset - BaseOffset); - - /* - * Retrieve the page from the view that we actually want. - */ - (*Page) = MmGetPhysicalAddress((char*)BaseAddress + - FileOffset - BaseOffset).LowPart >> PAGE_SHIFT; - - CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, TRUE); + KeWaitForSingleObject(&Event, WrPageIn, KernelMode, FALSE, NULL); + Status = IoStatus.Status; } - else + if (Mdl->MdlFlags & MDL_MAPPED_TO_SYSTEM_VA) { - PEPROCESS Process; - KIRQL Irql; - PVOID PageAddr; - LONGLONG VacbOffset; - - /* - * Allocate a page, this is rather complicated by the possibility - * we might have to move other things out of memory - */ - MI_SET_USAGE(MI_USAGE_SECTION); - MI_SET_PROCESS2(PsGetCurrentProcess()->ImageFileName); - Status = MmRequestPageMemoryConsumer(MC_USER, TRUE, Page); - if (!NT_SUCCESS(Status)) - { - return(Status); - } - Status = CcRosGetVacb(SharedCacheMap, - FileOffset, - &BaseOffset, - &BaseAddress, - &UptoDate, - &Vacb); - if (!NT_SUCCESS(Status)) - { - return(Status); - } - if (!UptoDate) - { - /* - * If the VACB isn't up to date then call the file - * system to read in the data. - */ - Status = CcReadVirtualAddress(Vacb); - if (!NT_SUCCESS(Status)) - { - CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE, FALSE); - return Status; - } - } - - Process = PsGetCurrentProcess(); - PageAddr = MiMapPageInHyperSpace(Process, *Page, &Irql); - VacbOffset = BaseOffset + VACB_MAPPING_GRANULARITY - FileOffset; - Length = RawLength - SegOffset; - if (Length <= VacbOffset && Length <= PAGE_SIZE) - { - memcpy(PageAddr, (char*)BaseAddress + FileOffset - BaseOffset, Length); - } - else if (VacbOffset >= PAGE_SIZE) - { - memcpy(PageAddr, (char*)BaseAddress + FileOffset - BaseOffset, PAGE_SIZE); - } - else - { - memcpy(PageAddr, (char*)BaseAddress + FileOffset - BaseOffset, VacbOffset); - MiUnmapPageInHyperSpace(Process, PageAddr, Irql); - CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); - Status = CcRosGetVacb(SharedCacheMap, - FileOffset + VacbOffset, - &BaseOffset, - &BaseAddress, - &UptoDate, - &Vacb); - if (!NT_SUCCESS(Status)) - { - return(Status); - } - if (!UptoDate) - { - /* - * If the VACB isn't up to date then call the file - * system to read in the data. - */ - Status = CcReadVirtualAddress(Vacb); - if (!NT_SUCCESS(Status)) - { - CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE, FALSE); - return Status; - } - } - PageAddr = MiMapPageInHyperSpace(Process, *Page, &Irql); - if (Length < PAGE_SIZE) - { - memcpy((char*)PageAddr + VacbOffset, BaseAddress, Length - VacbOffset); - } - else - { - memcpy((char*)PageAddr + VacbOffset, BaseAddress, PAGE_SIZE - VacbOffset); - } - } - MiUnmapPageInHyperSpace(Process, PageAddr, Irql); - CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); + MmUnmapLockedPages (Mdl->MappedSystemVa, Mdl); } - return(STATUS_SUCCESS); + + return Status; } + #else NTSTATUS NTAPI @@ -1571,7 +1517,7 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, /* * Add the page to the process's working set */ - MmInsertRmap(Page, Process, Address); + if (Process) MmInsertRmap(Page, Process, Address); /* * Finish the operation */ @@ -1684,7 +1630,8 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, KeBugCheck(MEMORY_MANAGEMENT); } ASSERT(MmIsPagePresent(Process, PAddress)); - MmInsertRmap(Page, Process, Address); + if (Process) + MmInsertRmap(Page, Process, Address); /* Set this section offset has being backed by our new page. */ Entry = MAKE_SSE(Page << PAGE_SHIFT, 1); @@ -1765,7 +1712,8 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, DPRINT1("Unable to create virtual mapping\n"); KeBugCheck(MEMORY_MANAGEMENT); } - MmInsertRmap(Page, Process, Address); + if (Process) + MmInsertRmap(Page, Process, Address); /* * Mark the offset within the section as having valid, in-memory @@ -1794,7 +1742,9 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, DPRINT1("Unable to create virtual mapping\n"); KeBugCheck(MEMORY_MANAGEMENT); } - MmInsertRmap(Page, Process, Address); + + if (Process) + MmInsertRmap(Page, Process, Address); /* Take a reference on it */ MmSharePageEntrySectionSegment(Segment, &Offset); @@ -1905,7 +1855,8 @@ MmAccessFaultSectionView(PMMSUPPORT AddressSpace, */ DPRINT("Swapping page (Old %x New %x)\n", OldPage, NewPage); MmDeleteVirtualMapping(Process, PAddress, NULL, NULL); - MmDeleteRmap(OldPage, Process, PAddress); + if (Process) + MmDeleteRmap(OldPage, Process, PAddress); MmUnsharePageEntrySectionSegment(MemoryArea, Segment, &Offset, FALSE, FALSE, NULL); MmUnlockSectionSegment(Segment); @@ -1923,567 +1874,15 @@ MmAccessFaultSectionView(PMMSUPPORT AddressSpace, KeBugCheck(MEMORY_MANAGEMENT); return(Status); } - MmInsertRmap(NewPage, Process, PAddress); + + if (Process) + MmInsertRmap(NewPage, Process, PAddress); MiSetPageEvent(Process, Address); DPRINT("Address 0x%p\n", Address); return(STATUS_SUCCESS); } -VOID -MmPageOutDeleteMapping(PVOID Context, PEPROCESS Process, PVOID Address) -{ - MM_SECTION_PAGEOUT_CONTEXT* PageOutContext; - BOOLEAN WasDirty; - PFN_NUMBER Page = 0; - - PageOutContext = (MM_SECTION_PAGEOUT_CONTEXT*)Context; - if (Process) - { - MmLockAddressSpace(&Process->Vm); - } - - MmDeleteVirtualMapping(Process, - Address, - &WasDirty, - &Page); - if (WasDirty) - { - PageOutContext->WasDirty = TRUE; - } - if (!PageOutContext->Private) - { - MmLockSectionSegment(PageOutContext->Segment); - MmUnsharePageEntrySectionSegment(PageOutContext->MemoryArea, - PageOutContext->Segment, - &PageOutContext->Offset, - PageOutContext->WasDirty, - TRUE, - &PageOutContext->SectionEntry); - MmUnlockSectionSegment(PageOutContext->Segment); - } - if (Process) - { - MmUnlockAddressSpace(&Process->Vm); - } - - if (PageOutContext->Private) - { - MmReleasePageMemoryConsumer(MC_USER, Page); - } -} - -NTSTATUS -NTAPI -MmPageOutSectionView(PMMSUPPORT AddressSpace, - MEMORY_AREA* MemoryArea, - PVOID Address, ULONG_PTR Entry) -{ - PFN_NUMBER Page; - MM_SECTION_PAGEOUT_CONTEXT Context; - SWAPENTRY SwapEntry; - NTSTATUS Status; -#ifndef NEWCC - ULONGLONG FileOffset; - PFILE_OBJECT FileObject; - PROS_SHARED_CACHE_MAP SharedCacheMap = NULL; - BOOLEAN IsImageSection; -#endif - BOOLEAN DirectMapped; - PEPROCESS Process = MmGetAddressSpaceOwner(AddressSpace); - KIRQL OldIrql; - - Address = (PVOID)PAGE_ROUND_DOWN(Address); - - /* - * Get the segment and section. - */ - Context.Segment = MemoryArea->SectionData.Segment; - Context.MemoryArea = MemoryArea; - Context.SectionEntry = Entry; - Context.CallingProcess = Process; - - Context.Offset.QuadPart = (ULONG_PTR)Address - MA_GetStartingAddress(MemoryArea) - + MemoryArea->SectionData.ViewOffset.QuadPart; - - DirectMapped = FALSE; - - MmLockSectionSegment(Context.Segment); - -#ifndef NEWCC - FileOffset = Context.Offset.QuadPart + Context.Segment->Image.FileOffset; - IsImageSection = MemoryArea->VadNode.u.VadFlags.VadType == VadImageMap; - FileObject = Context.Segment->FileObject; - - if (FileObject != NULL && - !(Context.Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) - { - SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; - - /* - * If the file system is letting us go directly to the cache and the - * memory area was mapped at an offset in the file which is page aligned - * then note this is a direct mapped page. - */ - if ((FileOffset % PAGE_SIZE) == 0 && - (Context.Offset.QuadPart + PAGE_SIZE <= Context.Segment->RawLength.QuadPart || !IsImageSection)) - { - DirectMapped = TRUE; - } - } -#endif - - /* - * Get the section segment entry and the physical address. - */ - if (!MmIsPagePresent(Process, Address)) - { - DPRINT1("Trying to page out not-present page at (%p,0x%p).\n", - Process ? Process->UniqueProcessId : 0, Address); - KeBugCheck(MEMORY_MANAGEMENT); - } - Page = MmGetPfnForProcess(Process, Address); - SwapEntry = MmGetSavedSwapEntryPage(Page); - - /* - * Check the reference count to ensure this page can be paged out - */ - if (MmGetReferenceCountPage(Page) != 1) - { - DPRINT("Cannot page out locked section page: 0x%lu (RefCount: %lu)\n", - Page, MmGetReferenceCountPage(Page)); - MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, Entry); - MmUnlockSectionSegment(Context.Segment); - return STATUS_UNSUCCESSFUL; - } - - /* - * Prepare the context structure for the rmap delete call. - */ - MmUnlockSectionSegment(Context.Segment); - Context.WasDirty = FALSE; - if (IS_SWAP_FROM_SSE(Entry) || PFN_FROM_SSE(Entry) != Page) - { - Context.Private = TRUE; - } - else - { - Context.Private = FALSE; - } - - /* - * Take an additional reference to the page or the VACB. - */ - if (DirectMapped && !Context.Private) - { - if(!MiIsPageFromCache(MemoryArea, Context.Offset.QuadPart)) - { - DPRINT1("Direct mapped non private page is not associated with the cache.\n"); - KeBugCheck(MEMORY_MANAGEMENT); - } - } - else - { - OldIrql = MiAcquirePfnLock(); - MmReferencePage(Page); - MiReleasePfnLock(OldIrql); - } - - MmDeleteAllRmaps(Page, (PVOID)&Context, MmPageOutDeleteMapping); - - /* Since we passed in a surrogate, we'll get back the page entry - * state in our context. This is intended to make intermediate - * decrements of share count not release the wait entry. - */ - Entry = Context.SectionEntry; - - /* - * If this wasn't a private page then we should have reduced the entry to - * zero by deleting all the rmaps. - */ - if (!Context.Private && Entry != 0) - { - if (!(Context.Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) - { - KeBugCheckEx(MEMORY_MANAGEMENT, Entry, (ULONG_PTR)Process, (ULONG_PTR)Address, 0); - } - } - - /* - * If the page wasn't dirty then we can just free it as for a readonly page. - * Since we unmapped all the mappings above we know it will not suddenly - * become dirty. - * If the page is from a pagefile section and has no swap entry, - * we can't free the page at this point. - */ - SwapEntry = MmGetSavedSwapEntryPage(Page); - if (Context.Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED) - { - if (Context.Private) - { - DPRINT1("Found a %s private page (address %p) in a shared section segment.\n", - Context.WasDirty ? "dirty" : "clean", Address); - KeBugCheckEx(MEMORY_MANAGEMENT, Page, (ULONG_PTR)Process, (ULONG_PTR)Address, 0); - } - if (!Context.WasDirty || SwapEntry != 0) - { - MmSetSavedSwapEntryPage(Page, 0); - if (SwapEntry != 0) - { - MmLockSectionSegment(Context.Segment); - MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, MAKE_SWAP_SSE(SwapEntry)); - MmUnlockSectionSegment(Context.Segment); - } - MmReleasePageMemoryConsumer(MC_USER, Page); - MiSetPageEvent(NULL, NULL); - return(STATUS_SUCCESS); - } - } - else if (!Context.Private && DirectMapped) - { - if (SwapEntry != 0) - { - DPRINT1("Found a swapentry for a non private and direct mapped page (address %p)\n", - Address); - KeBugCheckEx(MEMORY_MANAGEMENT, STATUS_UNSUCCESSFUL, SwapEntry, (ULONG_PTR)Process, (ULONG_PTR)Address); - } -#ifndef NEWCC - Status = CcRosUnmapVacb(SharedCacheMap, FileOffset, FALSE); -#else - Status = STATUS_SUCCESS; -#endif -#ifndef NEWCC - if (!NT_SUCCESS(Status)) - { - DPRINT1("CcRosUnmapVacb failed, status = %x\n", Status); - KeBugCheckEx(MEMORY_MANAGEMENT, Status, (ULONG_PTR)SharedCacheMap, (ULONG_PTR)FileOffset, (ULONG_PTR)Address); - } -#endif - MiSetPageEvent(NULL, NULL); - return(STATUS_SUCCESS); - } - else if (!Context.WasDirty && !DirectMapped && !Context.Private) - { - if (SwapEntry != 0) - { - DPRINT1("Found a swap entry for a non dirty, non private and not direct mapped page (address %p)\n", - Address); - KeBugCheckEx(MEMORY_MANAGEMENT, SwapEntry, Page, (ULONG_PTR)Process, (ULONG_PTR)Address); - } - MmReleasePageMemoryConsumer(MC_USER, Page); - MiSetPageEvent(NULL, NULL); - return(STATUS_SUCCESS); - } - else if (!Context.WasDirty && Context.Private && SwapEntry != 0) - { - DPRINT("Not dirty and private and not swapped (%p:%p)\n", Process, Address); - MmSetSavedSwapEntryPage(Page, 0); - MmLockAddressSpace(AddressSpace); - Status = MmCreatePageFileMapping(Process, - Address, - SwapEntry); - MmUnlockAddressSpace(AddressSpace); - if (!NT_SUCCESS(Status)) - { - DPRINT1("Status %x Swapping out %p:%p\n", Status, Process, Address); - KeBugCheckEx(MEMORY_MANAGEMENT, Status, (ULONG_PTR)Process, (ULONG_PTR)Address, SwapEntry); - } - MmReleasePageMemoryConsumer(MC_USER, Page); - MiSetPageEvent(NULL, NULL); - return(STATUS_SUCCESS); - } - - /* - * If necessary, allocate an entry in the paging file for this page - */ - if (SwapEntry == 0) - { - SwapEntry = MmAllocSwapPage(); - if (SwapEntry == 0) - { - MmShowOutOfSpaceMessagePagingFile(); - MmLockAddressSpace(AddressSpace); - /* - * For private pages restore the old mappings. - */ - if (Context.Private) - { - Status = MmCreateVirtualMapping(Process, - Address, - MmProtectToValue[MemoryArea->VadNode.u.VadFlags.Protection], - &Page, - 1); - MmSetDirtyPage(Process, Address); - MmInsertRmap(Page, - Process, - Address); - } - else - { - ULONG_PTR OldEntry; - - MmLockSectionSegment(Context.Segment); - - /* - * For non-private pages if the page wasn't direct mapped then - * set it back into the section segment entry so we don't loose - * our copy. Otherwise it will be handled by the cache manager. - */ - Status = MmCreateVirtualMapping(Process, - Address, - MmProtectToValue[MemoryArea->VadNode.u.VadFlags.Protection], - &Page, - 1); - MmSetDirtyPage(Process, Address); - MmInsertRmap(Page, - Process, - Address); - // If we got here, the previous entry should have been a wait - Entry = MAKE_SSE(Page << PAGE_SHIFT, 1); - OldEntry = MmGetPageEntrySectionSegment(Context.Segment, &Context.Offset); - ASSERT(OldEntry == 0 || OldEntry == MAKE_SWAP_SSE(MM_WAIT_ENTRY)); - MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, Entry); - MmUnlockSectionSegment(Context.Segment); - } - MmUnlockAddressSpace(AddressSpace); - MiSetPageEvent(NULL, NULL); - return(STATUS_PAGEFILE_QUOTA); - } - } - - /* - * Write the page to the pagefile - */ - Status = MmWriteToSwapPage(SwapEntry, Page); - if (!NT_SUCCESS(Status)) - { - DPRINT1("MM: Failed to write to swap page (Status was 0x%.8X)\n", - Status); - /* - * As above: undo our actions. - * FIXME: Also free the swap page. - */ - MmLockAddressSpace(AddressSpace); - if (Context.Private) - { - Status = MmCreateVirtualMapping(Process, - Address, - MmProtectToValue[MemoryArea->VadNode.u.VadFlags.Protection], - &Page, - 1); - MmSetDirtyPage(Process, Address); - MmInsertRmap(Page, - Process, - Address); - } - else - { - MmLockSectionSegment(Context.Segment); - Status = MmCreateVirtualMapping(Process, - Address, - MmProtectToValue[MemoryArea->VadNode.u.VadFlags.Protection], - &Page, - 1); - MmSetDirtyPage(Process, Address); - MmInsertRmap(Page, - Process, - Address); - Entry = MAKE_SSE(Page << PAGE_SHIFT, 1); - MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, Entry); - MmUnlockSectionSegment(Context.Segment); - } - MmUnlockAddressSpace(AddressSpace); - MiSetPageEvent(NULL, NULL); - return(STATUS_UNSUCCESSFUL); - } - - /* - * Otherwise we have succeeded. - */ - DPRINT("MM: Wrote section page 0x%.8X to swap!\n", Page << PAGE_SHIFT); - MmSetSavedSwapEntryPage(Page, 0); - if (Context.Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED) - { - MmLockSectionSegment(Context.Segment); - MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, MAKE_SWAP_SSE(SwapEntry)); - MmUnlockSectionSegment(Context.Segment); - } - else - { - MmReleasePageMemoryConsumer(MC_USER, Page); - } - - if (Context.Private) - { - MmLockAddressSpace(AddressSpace); - MmLockSectionSegment(Context.Segment); - Status = MmCreatePageFileMapping(Process, - Address, - SwapEntry); - /* We had placed a wait entry upon entry ... replace it before leaving */ - MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, Entry); - MmUnlockSectionSegment(Context.Segment); - MmUnlockAddressSpace(AddressSpace); - if (!NT_SUCCESS(Status)) - { - DPRINT1("Status %x Creating page file mapping for %p:%p\n", Status, Process, Address); - KeBugCheckEx(MEMORY_MANAGEMENT, Status, (ULONG_PTR)Process, (ULONG_PTR)Address, SwapEntry); - } - } - else - { - MmLockAddressSpace(AddressSpace); - MmLockSectionSegment(Context.Segment); - Entry = MAKE_SWAP_SSE(SwapEntry); - /* We had placed a wait entry upon entry ... replace it before leaving */ - MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, Entry); - MmUnlockSectionSegment(Context.Segment); - MmUnlockAddressSpace(AddressSpace); - } - - MiSetPageEvent(NULL, NULL); - return(STATUS_SUCCESS); -} - -NTSTATUS -NTAPI -MmWritePageSectionView(PMMSUPPORT AddressSpace, - PMEMORY_AREA MemoryArea, - PVOID Address, - ULONG PageEntry) -{ - LARGE_INTEGER Offset; - PMM_SECTION_SEGMENT Segment; - PFN_NUMBER Page; - SWAPENTRY SwapEntry; - ULONG_PTR Entry; - BOOLEAN Private; - NTSTATUS Status; - PFILE_OBJECT FileObject; -#ifndef NEWCC - PROS_SHARED_CACHE_MAP SharedCacheMap = NULL; -#endif - BOOLEAN DirectMapped; - BOOLEAN IsImageSection; - PEPROCESS Process = MmGetAddressSpaceOwner(AddressSpace); - - Address = (PVOID)PAGE_ROUND_DOWN(Address); - - Offset.QuadPart = (ULONG_PTR)Address - MA_GetStartingAddress(MemoryArea) - + MemoryArea->SectionData.ViewOffset.QuadPart; - - /* - * Get the segment and section. - */ - Segment = MemoryArea->SectionData.Segment; - IsImageSection = MemoryArea->VadNode.u.VadFlags.VadType == VadImageMap; - - FileObject = Segment->FileObject; - DirectMapped = FALSE; - if (FileObject != NULL && - !(Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) - { -#ifndef NEWCC - SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; -#endif - - /* - * If the file system is letting us go directly to the cache and the - * memory area was mapped at an offset in the file which is page aligned - * then note this is a direct mapped page. - */ - if (((Offset.QuadPart + Segment->Image.FileOffset) % PAGE_SIZE) == 0 && - (Offset.QuadPart + PAGE_SIZE <= Segment->RawLength.QuadPart || !IsImageSection)) - { - DirectMapped = TRUE; - } - } - - /* - * Get the section segment entry and the physical address. - */ - Entry = MmGetPageEntrySectionSegment(Segment, &Offset); - if (!MmIsPagePresent(Process, Address)) - { - DPRINT1("Trying to page out not-present page at (%p,0x%p).\n", - Process ? Process->UniqueProcessId : 0, Address); - KeBugCheck(MEMORY_MANAGEMENT); - } - Page = MmGetPfnForProcess(Process, Address); - SwapEntry = MmGetSavedSwapEntryPage(Page); - - /* - * Check for a private (COWed) page. - */ - if (IS_SWAP_FROM_SSE(Entry) || PFN_FROM_SSE(Entry) != Page) - { - Private = TRUE; - } - else - { - Private = FALSE; - } - - /* - * Speculatively set all mappings of the page to clean. - */ - MmSetCleanAllRmaps(Page); - - /* - * If this page was direct mapped from the cache then the cache manager - * will take care of writing it back to disk. - */ - if (DirectMapped && !Private) - { - //LARGE_INTEGER SOffset; - ASSERT(SwapEntry == 0); - //SOffset.QuadPart = Offset.QuadPart + Segment->Image.FileOffset; -#ifndef NEWCC - CcRosMarkDirtyFile(SharedCacheMap, Offset.QuadPart); -#endif - MmLockSectionSegment(Segment); - MmSetPageEntrySectionSegment(Segment, &Offset, PageEntry); - MmUnlockSectionSegment(Segment); - MiSetPageEvent(NULL, NULL); - return(STATUS_SUCCESS); - } - - /* - * If necessary, allocate an entry in the paging file for this page - */ - if (SwapEntry == 0) - { - SwapEntry = MmAllocSwapPage(); - if (SwapEntry == 0) - { - MmSetDirtyAllRmaps(Page); - MiSetPageEvent(NULL, NULL); - return(STATUS_PAGEFILE_QUOTA); - } - MmSetSavedSwapEntryPage(Page, SwapEntry); - } - - /* - * Write the page to the pagefile - */ - Status = MmWriteToSwapPage(SwapEntry, Page); - if (!NT_SUCCESS(Status)) - { - DPRINT1("MM: Failed to write to swap page (Status was 0x%.8X)\n", - Status); - MmSetDirtyAllRmaps(Page); - MiSetPageEvent(NULL, NULL); - return(STATUS_UNSUCCESSFUL); - } - - /* - * Otherwise we have succeeded. - */ - DPRINT("MM: Wrote section page 0x%.8X to swap!\n", Page << PAGE_SHIFT); - MiSetPageEvent(NULL, NULL); - return(STATUS_SUCCESS); -} - NTSTATUS NTAPI MmProtectSectionView(PMMSUPPORT AddressSpace, @@ -2560,48 +1959,6 @@ MmQuerySectionView(PMEMORY_AREA MemoryArea, return(STATUS_SUCCESS); } -VOID -NTAPI -MmpFreePageFileSegment(PMM_SECTION_SEGMENT Segment) -{ - ULONG Length; - LARGE_INTEGER Offset; - ULONG_PTR Entry; - SWAPENTRY SavedSwapEntry; - PFN_NUMBER Page; - - Page = 0; - - MmLockSectionSegment(Segment); - - Length = PAGE_ROUND_UP(Segment->Length.QuadPart); - for (Offset.QuadPart = 0; Offset.QuadPart < Length; Offset.QuadPart += PAGE_SIZE) - { - Entry = MmGetPageEntrySectionSegment(Segment, &Offset); - if (Entry) - { - MmSetPageEntrySectionSegment(Segment, &Offset, 0); - if (IS_SWAP_FROM_SSE(Entry)) - { - MmFreeSwapPage(SWAPENTRY_FROM_SSE(Entry)); - } - else - { - Page = PFN_FROM_SSE(Entry); - SavedSwapEntry = MmGetSavedSwapEntryPage(Page); - if (SavedSwapEntry != 0) - { - MmSetSavedSwapEntryPage(Page, 0); - MmFreeSwapPage(SavedSwapEntry); - } - MmReleasePageMemoryConsumer(MC_USER, Page); - } - } - } - - MmUnlockSectionSegment(Segment); -} - VOID NTAPI MmpDeleteSection(PVOID ObjectBody) { @@ -2617,10 +1974,7 @@ MmpDeleteSection(PVOID ObjectBody) DPRINT("MmpDeleteSection(ObjectBody %p)\n", ObjectBody); if (Section->u.Flags.Image) { - ULONG i; - ULONG NrSegments; - ULONG RefCount; - PMM_SECTION_SEGMENT SectionSegments; + PMM_IMAGE_SECTION_OBJECT ImageSectionObject = (PMM_IMAGE_SECTION_OBJECT)Section->Segment; /* * NOTE: Section->ImageSection can be NULL for short time @@ -2631,25 +1985,9 @@ MmpDeleteSection(PVOID ObjectBody) if (Section->Segment == NULL) return; - SectionSegments = ((PMM_IMAGE_SECTION_OBJECT)Section->Segment)->Segments; - NrSegments = ((PMM_IMAGE_SECTION_OBJECT)Section->Segment)->NrSegments; - - for (i = 0; i < NrSegments; i++) - { - if (SectionSegments[i].Image.Characteristics & IMAGE_SCN_MEM_SHARED) - { - MmLockSectionSegment(&SectionSegments[i]); - } - RefCount = InterlockedDecrementUL(&SectionSegments[i].ReferenceCount); - if (SectionSegments[i].Image.Characteristics & IMAGE_SCN_MEM_SHARED) - { - MmUnlockSectionSegment(&SectionSegments[i]); - if (RefCount == 0) - { - MmpFreePageFileSegment(&SectionSegments[i]); - } - } - } + /* We just dereference the first segment */ + ASSERT(ImageSectionObject->RefCount > 0); + MmDereferenceSegment(ImageSectionObject->Segments); } #ifdef NEWCC else if (Section->Segment && Section->Segment->Flags & MM_DATAFILE_SEGMENT) @@ -2672,25 +2010,17 @@ MmpDeleteSection(PVOID ObjectBody) #endif else { + PMM_SECTION_SEGMENT Segment = (PMM_SECTION_SEGMENT)Section->Segment; + /* * NOTE: Section->Segment can be NULL for short time * during the section creating. */ - if (Section->Segment == NULL) + if (Segment == NULL) return; - (void)InterlockedDecrementUL(&((PMM_SECTION_SEGMENT)Section->Segment)->ReferenceCount); - } - - if (Section->Segment) - { - PMM_SECTION_SEGMENT Segment = (PMM_SECTION_SEGMENT)Section->Segment; - if (Segment->FileObject != NULL) - { - #ifndef NEWCC - CcRosDereferenceCache(Segment->FileObject); - #endif - } + Segment->SectionCount--; + MmDereferenceSegment(Segment); } } @@ -2763,13 +2093,17 @@ MmCreatePhysicalMemorySection(VOID) } RtlZeroMemory(Segment, sizeof(MM_SECTION_SEGMENT)); PhysSection->Segment = (PSEGMENT)Segment; - Segment->ReferenceCount = 1; + Segment->RefCount = 1; + + Segment->ReferenceCount = &Segment->RefCount; + Segment->Flags = &Segment->SegFlags; + ExInitializeFastMutex(&Segment->Lock); Segment->Image.FileOffset = 0; Segment->Protection = PAGE_EXECUTE_READWRITE; Segment->RawLength = SectionSize; Segment->Length = SectionSize; - Segment->Flags = 0; + Segment->SegFlags = 0; Segment->WriteCopy = FALSE; Segment->Image.VirtualAddress = 0; Segment->Image.Characteristics = 0; @@ -2823,6 +2157,7 @@ MmInitSectionImplementation(VOID) return(STATUS_SUCCESS); } +static NTSTATUS NTAPI MmCreateDataFileSection(PSECTION *SectionObject, @@ -2831,7 +2166,8 @@ MmCreateDataFileSection(PSECTION *SectionObject, PLARGE_INTEGER UMaximumSize, ULONG SectionPageProtection, ULONG AllocationAttributes, - PFILE_OBJECT FileObject) + PFILE_OBJECT FileObject, + BOOLEAN GotFileHandle) /* * Create a section backed by a data file */ @@ -2840,8 +2176,7 @@ MmCreateDataFileSection(PSECTION *SectionObject, NTSTATUS Status; LARGE_INTEGER MaximumSize; PMM_SECTION_SEGMENT Segment; - FILE_STANDARD_INFORMATION FileInfo; - ULONG Length; + KIRQL OldIrql; /* * Create the section @@ -2869,63 +2204,63 @@ MmCreateDataFileSection(PSECTION *SectionObject, Section->u.Flags.filler = 1; Section->InitialPageProtection = SectionPageProtection; Section->u.Flags.File = 1; + if (AllocationAttributes & SEC_NO_CHANGE) Section->u.Flags.NoChange = 1; - /* - * FIXME: This is propably not entirely correct. We can't look into - * the standard FCB header because it might not be initialized yet - * (as in case of the EXT2FS driver by Manoj Paul Joseph where the - * standard file information is filled on first request). - */ - Status = IoQueryFileInformation(FileObject, - FileStandardInformation, - sizeof(FILE_STANDARD_INFORMATION), - &FileInfo, - &Length); - if (!NT_SUCCESS(Status)) - { - ObDereferenceObject(Section); - ObDereferenceObject(FileObject); - return Status; - } - - /* - * FIXME: Revise this once a locking order for file size changes is - * decided - */ - if ((UMaximumSize != NULL) && (UMaximumSize->QuadPart != 0)) + if (!GotFileHandle) { + ASSERT(UMaximumSize != NULL); + ASSERT(UMaximumSize->QuadPart != 0); MaximumSize = *UMaximumSize; } else { - MaximumSize = FileInfo.EndOfFile; - /* Mapping zero-sized files isn't allowed. */ - if (MaximumSize.QuadPart == 0) - { - ObDereferenceObject(Section); - ObDereferenceObject(FileObject); - return STATUS_MAPPED_FILE_SIZE_ZERO; - } - } - - if (MaximumSize.QuadPart > FileInfo.EndOfFile.QuadPart) - { - Status = IoSetInformation(FileObject, - FileEndOfFileInformation, - sizeof(LARGE_INTEGER), - &MaximumSize); + LARGE_INTEGER FileSize; + Status = FsRtlGetFileSize(FileObject, &FileSize); if (!NT_SUCCESS(Status)) { ObDereferenceObject(Section); ObDereferenceObject(FileObject); - return(STATUS_SECTION_NOT_EXTENDED); + return Status; + } + + /* + * FIXME: Revise this once a locking order for file size changes is + * decided + */ + if ((UMaximumSize != NULL) && (UMaximumSize->QuadPart != 0)) + { + MaximumSize = *UMaximumSize; + } + else + { + MaximumSize = FileSize; + /* Mapping zero-sized files isn't allowed. */ + if (MaximumSize.QuadPart == 0) + { + ObDereferenceObject(Section); + ObDereferenceObject(FileObject); + return STATUS_MAPPED_FILE_SIZE_ZERO; + } + } + + if (MaximumSize.QuadPart > FileSize.QuadPart) + { + Status = IoSetInformation(FileObject, + FileEndOfFileInformation, + sizeof(LARGE_INTEGER), + &MaximumSize); + if (!NT_SUCCESS(Status)) + { + ObDereferenceObject(Section); + ObDereferenceObject(FileObject); + return(STATUS_SECTION_NOT_EXTENDED); + } } } - if (FileObject->SectionObjectPointer == NULL || - FileObject->SectionObjectPointer->SharedCacheMap == NULL) + if (FileObject->SectionObjectPointer == NULL) { ObDereferenceObject(Section); ObDereferenceObject(FileObject); @@ -2943,34 +2278,61 @@ MmCreateDataFileSection(PSECTION *SectionObject, return(Status); } + /* Lock the PFN lock while messing with Section Object pointers */ + OldIrql = MiAcquirePfnLock(); + Segment = FileObject->SectionObjectPointer->DataSectionObject; + + while (Segment && (Segment->SegFlags & (MM_SEGMENT_INDELETE | MM_SEGMENT_INCREATE))) + { + LARGE_INTEGER ShortTime = {{-10 * 100 * 1000, -1}}; + + MiReleasePfnLock(OldIrql); + KeDelayExecutionThread(KernelMode, FALSE, &ShortTime); + OldIrql = MiAcquirePfnLock(); + Segment = FileObject->SectionObjectPointer->DataSectionObject; + } + /* * If this file hasn't been mapped as a data file before then allocate a * section segment to describe the data file mapping */ - if (FileObject->SectionObjectPointer->DataSectionObject == NULL) + if (Segment == NULL) { Segment = ExAllocatePoolWithTag(NonPagedPool, sizeof(MM_SECTION_SEGMENT), TAG_MM_SECTION_SEGMENT); if (Segment == NULL) { //KeSetEvent((PVOID)&FileObject->Lock, IO_NO_INCREMENT, FALSE); + MiReleasePfnLock(OldIrql); ObDereferenceObject(Section); ObDereferenceObject(FileObject); return(STATUS_NO_MEMORY); } + + /* We are creating it */ + RtlZeroMemory(Segment, sizeof(*Segment)); + Segment->SegFlags = MM_DATAFILE_SEGMENT | MM_SEGMENT_INCREATE; + Segment->RefCount = 1; + + FileObject->SectionObjectPointer->DataSectionObject = Segment; + + /* We're safe to release the lock now */ + MiReleasePfnLock(OldIrql); + Section->Segment = (PSEGMENT)Segment; - Segment->ReferenceCount = 1; + + /* Self-referencing segment */ + Segment->Flags = &Segment->SegFlags; + Segment->ReferenceCount = &Segment->RefCount; + + Segment->SectionCount = 1; + ExInitializeFastMutex(&Segment->Lock); Segment->FileObject = FileObject; - /* - * Set the lock before assigning the segment to the file object - */ - ExAcquireFastMutex(&Segment->Lock); - FileObject->SectionObjectPointer->DataSectionObject = (PVOID)Segment; Segment->Image.FileOffset = 0; Segment->Protection = SectionPageProtection; - Segment->Flags = MM_DATAFILE_SEGMENT; + Segment->Image.Characteristics = 0; Segment->WriteCopy = (SectionPageProtection & (PAGE_WRITECOPY | PAGE_EXECUTE_WRITECOPY)); if (AllocationAttributes & SEC_RESERVE) @@ -2983,20 +2345,21 @@ MmCreateDataFileSection(PSECTION *SectionObject, Segment->Length.QuadPart = PAGE_ROUND_UP(Segment->RawLength.QuadPart); } Segment->Image.VirtualAddress = 0; - Segment->Locked = TRUE; MiInitializeSectionPageTable(Segment); + + /* We're good to use it now */ + OldIrql = MiAcquirePfnLock(); + Segment->SegFlags &= ~MM_SEGMENT_INCREATE; + MiReleasePfnLock(OldIrql); } else { - /* - * If the file is already mapped as a data file then we may need - * to extend it - */ - Segment = - (PMM_SECTION_SEGMENT)FileObject->SectionObjectPointer-> - DataSectionObject; Section->Segment = (PSEGMENT)Segment; - (void)InterlockedIncrementUL(&Segment->ReferenceCount); + Segment->RefCount++; + InterlockedIncrementUL(&Segment->SectionCount); + + MiReleasePfnLock(OldIrql); + MmLockSectionSegment(Segment); if (MaximumSize.QuadPart > Segment->RawLength.QuadPart && @@ -3006,15 +2369,13 @@ MmCreateDataFileSection(PSECTION *SectionObject, Segment->Length.QuadPart = PAGE_ROUND_UP(Segment->RawLength.QuadPart); } - /* We let the segment reference the file object */ + MmUnlockSectionSegment(Segment); + + /* The segment already has a reference to a file object. Don't bother keeping one.*/ ObDereferenceObject(FileObject); - FileObject = Segment->FileObject; } - MmUnlockSectionSegment(Segment); Section->SizeOfSection = MaximumSize; -#ifndef NEWCC - CcRosReferenceCache(FileObject); -#endif + //KeSetEvent((PVOID)&FileObject->Lock, IO_NO_INCREMENT, FALSE); *SectionObject = Section; return(STATUS_SUCCESS); @@ -3541,7 +2902,6 @@ ExeFmtpCreateImageSection(PFILE_OBJECT FileObject, */ for (i = 0; i < RTL_NUMBER_OF(ExeFmtpLoaders); ++ i) { - RtlZeroMemory(ImageSectionObject, sizeof(*ImageSectionObject)); Flags = 0; Status = ExeFmtpLoaders[i](FileHeader, @@ -3580,6 +2940,7 @@ ExeFmtpCreateImageSection(PFILE_OBJECT FileObject, return Status; ASSERT(ImageSectionObject->Segments != NULL); + ASSERT(ImageSectionObject->RefCount > 0); /* * Some defaults @@ -3640,11 +3001,14 @@ ExeFmtpCreateImageSection(PFILE_OBJECT FileObject, for ( i = 0; i < ImageSectionObject->NrSegments; ++ i ) { ExInitializeFastMutex(&ImageSectionObject->Segments[i].Lock); - ImageSectionObject->Segments[i].ReferenceCount = 1; + ImageSectionObject->Segments[i].ReferenceCount = &ImageSectionObject->RefCount; + ImageSectionObject->Segments[i].Flags = &ImageSectionObject->SegFlags; MiInitializeSectionPageTable(&ImageSectionObject->Segments[i]); ImageSectionObject->Segments[i].FileObject = FileObject; } + ASSERT(ImageSectionObject->RefCount > 0); + ImageSectionObject->FileObject = FileObject; ASSERT(NT_SUCCESS(Status)); @@ -3662,20 +3026,18 @@ MmCreateImageSection(PSECTION *SectionObject, { PSECTION Section; NTSTATUS Status; - PMM_SECTION_SEGMENT SectionSegments; PMM_IMAGE_SECTION_OBJECT ImageSectionObject; - ULONG i; + KIRQL OldIrql; + if (FileObject == NULL) return STATUS_INVALID_FILE_FOR_SECTION; -#ifndef NEWCC - if (!CcIsFileCached(FileObject)) + if (FileObject->SectionObjectPointer == NULL) { DPRINT1("Denying section creation due to missing cache initialization\n"); return STATUS_INVALID_FILE_FOR_SECTION; } -#endif /* * Create the section @@ -3709,13 +3071,31 @@ MmCreateImageSection(PSECTION *SectionObject, if (AllocationAttributes & SEC_NO_CHANGE) Section->u.Flags.NoChange = 1; - if (FileObject->SectionObjectPointer->ImageSectionObject == NULL) + OldIrql = MiAcquirePfnLock(); + + /* Wait for it to be properly created or deleted */ + ImageSectionObject = FileObject->SectionObjectPointer->ImageSectionObject; + while(ImageSectionObject && (ImageSectionObject->SegFlags & (MM_SEGMENT_INDELETE | MM_SEGMENT_INCREATE))) + { + LARGE_INTEGER ShortTime; + + MiReleasePfnLock(OldIrql); + + ShortTime.QuadPart = - 10 * 100 * 1000; + KeDelayExecutionThread(KernelMode, FALSE, &ShortTime); + + OldIrql = MiAcquirePfnLock(); + ImageSectionObject = FileObject->SectionObjectPointer->ImageSectionObject; + } + + if (ImageSectionObject == NULL) { NTSTATUS StatusExeFmt; - ImageSectionObject = ExAllocatePoolWithTag(PagedPool, sizeof(MM_IMAGE_SECTION_OBJECT), TAG_MM_SECTION_SEGMENT); + ImageSectionObject = ExAllocatePoolWithTag(NonPagedPool, sizeof(MM_IMAGE_SECTION_OBJECT), TAG_MM_SECTION_SEGMENT); if (ImageSectionObject == NULL) { + MiReleasePfnLock(OldIrql); ObDereferenceObject(FileObject); ObDereferenceObject(Section); return(STATUS_NO_MEMORY); @@ -3723,10 +3103,21 @@ MmCreateImageSection(PSECTION *SectionObject, RtlZeroMemory(ImageSectionObject, sizeof(MM_IMAGE_SECTION_OBJECT)); + ImageSectionObject->SegFlags = MM_SEGMENT_INCREATE; + ImageSectionObject->RefCount = 1; + FileObject->SectionObjectPointer->ImageSectionObject = ImageSectionObject; + + MiReleasePfnLock(OldIrql); + StatusExeFmt = ExeFmtpCreateImageSection(FileObject, ImageSectionObject); if (!NT_SUCCESS(StatusExeFmt)) { + /* Unset */ + OldIrql = MiAcquirePfnLock(); + FileObject->SectionObjectPointer->ImageSectionObject = NULL; + MiReleasePfnLock(OldIrql); + if(ImageSectionObject->Segments != NULL) ExFreePool(ImageSectionObject->Segments); @@ -3747,6 +3138,7 @@ MmCreateImageSection(PSECTION *SectionObject, Section->Segment = (PSEGMENT)ImageSectionObject; ASSERT(ImageSectionObject->Segments); + ASSERT(ImageSectionObject->RefCount > 0); /* * Lock the file @@ -3754,6 +3146,11 @@ MmCreateImageSection(PSECTION *SectionObject, Status = MmspWaitForFileLock(FileObject); if (!NT_SUCCESS(Status)) { + /* Unset */ + OldIrql = MiAcquirePfnLock(); + FileObject->SectionObjectPointer->ImageSectionObject = NULL; + MiReleasePfnLock(OldIrql); + ExFreePool(ImageSectionObject->Segments); ExFreePool(ImageSectionObject); ObDereferenceObject(Section); @@ -3761,66 +3158,30 @@ MmCreateImageSection(PSECTION *SectionObject, return(Status); } - if (NULL != InterlockedCompareExchangePointer(&FileObject->SectionObjectPointer->ImageSectionObject, - ImageSectionObject, NULL)) - { - /* - * An other thread has initialized the same image in the background - */ - ExFreePool(ImageSectionObject->Segments); - ExFreePool(ImageSectionObject); - ImageSectionObject = FileObject->SectionObjectPointer->ImageSectionObject; - Section->Segment = (PSEGMENT)ImageSectionObject; - SectionSegments = ImageSectionObject->Segments; - - for (i = 0; i < ImageSectionObject->NrSegments; i++) - { - (void)InterlockedIncrementUL(&SectionSegments[i].ReferenceCount); - } - - /* We let the Image Section Object hold the reference */ - ObDereferenceObject(FileObject); - FileObject = ImageSectionObject->FileObject; - } + OldIrql = MiAcquirePfnLock(); + ImageSectionObject->SegFlags &= ~MM_SEGMENT_INCREATE; + MiReleasePfnLock(OldIrql); Status = StatusExeFmt; } else { - /* - * Lock the file - */ - Status = MmspWaitForFileLock(FileObject); - if (Status != STATUS_SUCCESS) - { - ObDereferenceObject(Section); - ObDereferenceObject(FileObject); - return(Status); - } + /* Take one ref */ + ImageSectionObject->RefCount++; + + MiReleasePfnLock(OldIrql); - ImageSectionObject = FileObject->SectionObjectPointer->ImageSectionObject; Section->Segment = (PSEGMENT)ImageSectionObject; - SectionSegments = ImageSectionObject->Segments; - - /* - * Otherwise just reference all the section segments - */ - for (i = 0; i < ImageSectionObject->NrSegments; i++) - { - (void)InterlockedIncrementUL(&SectionSegments[i].ReferenceCount); - } /* We let the Image Section Object hold the reference */ ObDereferenceObject(FileObject); - FileObject = ImageSectionObject->FileObject; Status = STATUS_SUCCESS; } -#ifndef NEWCC - CcRosReferenceCache(FileObject); -#endif //KeSetEvent((PVOID)&FileObject->Lock, IO_NO_INCREMENT, FALSE); *SectionObject = Section; + ASSERT(ImageSectionObject->RefCount > 0); + return(Status); } @@ -3908,10 +3269,6 @@ MmFreeSectionPage(PVOID Context, MEMORY_AREA* MemoryArea, PVOID Address, PFN_NUMBER Page, SWAPENTRY SwapEntry, BOOLEAN Dirty) { ULONG_PTR Entry; -#ifndef NEWCC - PFILE_OBJECT FileObject; - PROS_SHARED_CACHE_MAP SharedCacheMap; -#endif LARGE_INTEGER Offset; SWAPENTRY SavedSwapEntry; PMM_SECTION_SEGMENT Segment; @@ -3942,18 +3299,12 @@ MmFreeSectionPage(PVOID Context, MEMORY_AREA* MemoryArea, PVOID Address, } /* - * For a dirty, datafile, non-private page mark it as dirty in the - * cache manager. + * For a dirty, datafile, non-private page, there shoulkd be no swap entry */ - if (Segment->Flags & MM_DATAFILE_SEGMENT) + if (*Segment->Flags & MM_DATAFILE_SEGMENT) { if (Page == PFN_FROM_SSE(Entry) && Dirty) { -#ifndef NEWCC - FileObject = MemoryArea->SectionData.Segment->FileObject; - SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; - CcRosMarkDirtyFile(SharedCacheMap, Offset.QuadPart + Segment->Image.FileOffset); -#endif ASSERT(SwapEntry == 0); } } @@ -3970,6 +3321,8 @@ MmFreeSectionPage(PVOID Context, MEMORY_AREA* MemoryArea, PVOID Address, if (IS_SWAP_FROM_SSE(Entry) || Page != PFN_FROM_SSE(Entry)) { + ASSERT(Process != NULL); + /* * Just dereference private pages */ @@ -3984,7 +3337,8 @@ MmFreeSectionPage(PVOID Context, MEMORY_AREA* MemoryArea, PVOID Address, } else { - MmDeleteRmap(Page, Process, Address); + if (Process) + MmDeleteRmap(Page, Process, Address); MmUnsharePageEntrySectionSegment(MemoryArea, Segment, &Offset, Dirty, FALSE, NULL); } } @@ -4467,6 +3821,8 @@ MmMapViewOfSection(IN PVOID SectionObject, SectionSegments = ImageSectionObject->Segments; NrSegments = ImageSectionObject->NrSegments; + ASSERT(ImageSectionObject->RefCount > 0); + ImageBase = (ULONG_PTR)*BaseAddress; if (ImageBase == 0) { @@ -4550,6 +3906,8 @@ MmMapViewOfSection(IN PVOID SectionObject, PMM_SECTION_SEGMENT Segment = (PMM_SECTION_SEGMENT)Section->Segment; LONGLONG ViewOffset; + ASSERT(Segment->RefCount > 0); + /* check for write access */ if ((Protect & (PAGE_READWRITE|PAGE_EXECUTE_READWRITE)) && !(Section->InitialPageProtection & (PAGE_READWRITE|PAGE_EXECUTE_READWRITE))) @@ -4633,68 +3991,58 @@ BOOLEAN NTAPI MmCanFileBeTruncated (IN PSECTION_OBJECT_POINTERS SectionObjectPointer, IN PLARGE_INTEGER NewFileSize) { + KIRQL OldIrql = MiAcquirePfnLock(); + BOOLEAN Ret; + PMM_SECTION_SEGMENT Segment; + +CheckSectionPointer: /* Check whether an ImageSectionObject exists */ if (SectionObjectPointer->ImageSectionObject != NULL) { DPRINT1("ERROR: File can't be truncated because it has an image section\n"); + MiReleasePfnLock(OldIrql); + return FALSE; } - if (SectionObjectPointer->DataSectionObject != NULL) + Segment = (PMM_SECTION_SEGMENT)SectionObjectPointer->DataSectionObject; + /* Wait for it to be created/deleted properly */ + while (Segment && (Segment->SegFlags & (MM_SEGMENT_INCREATE | MM_SEGMENT_INDELETE))) { - PMM_SECTION_SEGMENT Segment; + LARGE_INTEGER ShortTime; - Segment = (PMM_SECTION_SEGMENT)SectionObjectPointer-> - DataSectionObject; + ShortTime.QuadPart = -10 * 100 * 1000; - if (Segment->ReferenceCount != 0) + /* Bad luck. Wait a bit for the operation to finish */ + MiReleasePfnLock(OldIrql); + KeDelayExecutionThread(KernelMode, FALSE, &ShortTime); + OldIrql = MiAcquirePfnLock(); + goto CheckSectionPointer; + } + + if (Segment) + { + if ((Segment->SectionCount == 1) && (SectionObjectPointer->SharedCacheMap != NULL)) { -#ifdef NEWCC - CC_FILE_SIZES FileSizes; - CcpLock(); - if (SectionObjectPointer->SharedCacheMap && (Segment->ReferenceCount > CcpCountCacheSections((PNOCC_CACHE_MAP)SectionObjectPointer->SharedCacheMap))) - { - CcpUnlock(); - /* Check size of file */ - if (SectionObjectPointer->SharedCacheMap) - { - if (!CcGetFileSizes(Segment->FileObject, &FileSizes)) - { - return FALSE; - } - - if (NewFileSize->QuadPart <= FileSizes.FileSize.QuadPart) - { - return FALSE; - } - } - } - else - CcpUnlock(); -#else - /* Check size of file */ - if (SectionObjectPointer->SharedCacheMap) - { - PROS_SHARED_CACHE_MAP SharedCacheMap = SectionObjectPointer->SharedCacheMap; - if (NewFileSize->QuadPart <= SharedCacheMap->FileSize.QuadPart) - { - return FALSE; - } - } -#endif + /* If the cache is the only one holding a reference to the segment, then it's fine to resize */ + Ret = TRUE; } else { - /* Something must gone wrong - * how can we have a Section but no - * reference? */ - DPRINT("ERROR: DataSectionObject without reference!\n"); + /* We can't shrink, but we can extend */ + Ret = NewFileSize->QuadPart > Segment->RawLength.QuadPart; } } + else + { + Ret = TRUE; + } + + MiReleasePfnLock(OldIrql); DPRINT("FIXME: didn't check for outstanding write probes\n"); - return TRUE; + return Ret; } @@ -4797,8 +4145,7 @@ MmMapViewInSystemSpaceEx ( MmLockAddressSpace(AddressSpace); - - if ((*ViewSize == 0) || ((SectionOffset->QuadPart + *ViewSize) > Section->SizeOfSection.QuadPart)) + if (*ViewSize == 0) { *ViewSize = MIN((Section->SizeOfSection.QuadPart - SectionOffset->QuadPart), SIZE_T_MAX); } @@ -4812,7 +4159,7 @@ MmMapViewInSystemSpaceEx ( *ViewSize, PAGE_READWRITE, SectionOffset->QuadPart, - 0); + SEC_RESERVE); MmUnlockSectionSegment(Segment); MmUnlockAddressSpace(AddressSpace); @@ -4946,13 +4293,8 @@ MmCreateSection (OUT PVOID * Section, return STATUS_INVALID_PARAMETER_6; } - /* Did the caller pass an object? */ - if (FileObject) - { - /* Reference the object directly */ - ObReferenceObject(FileObject); - } - else + /* Did the caller pass a handle? */ + if (FileHandle) { /* Reference the file handle to get the object */ Status = ObReferenceObjectByHandle(FileHandle, @@ -4967,6 +4309,11 @@ MmCreateSection (OUT PVOID * Section, return Status; } } + else + { + /* Reference the object directly */ + ObReferenceObject(FileObject); + } } else { @@ -4974,63 +4321,6 @@ MmCreateSection (OUT PVOID * Section, if (AllocationAttributes & SEC_IMAGE) return STATUS_INVALID_FILE_FOR_SECTION; } -#ifndef NEWCC // A hack for initializing caching. - // This is needed only in the old case. - if (FileHandle) - { - IO_STATUS_BLOCK Iosb; - NTSTATUS Status; - CHAR Buffer; - LARGE_INTEGER ByteOffset; - ByteOffset.QuadPart = 0; - Status = ZwReadFile(FileHandle, - NULL, - NULL, - NULL, - &Iosb, - &Buffer, - sizeof(Buffer), - &ByteOffset, - NULL); - if (!NT_SUCCESS(Status) && Status != STATUS_END_OF_FILE) - { - DPRINT1("CC failure: %lx\n", Status); - if (FileObject) - ObDereferenceObject(FileObject); - return Status; - } - // Caching is initialized... - - // Hack of the hack: actually, it might not be initialized if FSD init on effective right and if file is null-size - // In such case, force cache by initiating a write IRP - if (Status == STATUS_END_OF_FILE && !(AllocationAttributes & SEC_IMAGE) && FileObject != NULL && - (FileObject->SectionObjectPointer == NULL || FileObject->SectionObjectPointer->SharedCacheMap == NULL)) - { - Buffer = 0xdb; - Status = ZwWriteFile(FileHandle, - NULL, - NULL, - NULL, - &Iosb, - &Buffer, - sizeof(Buffer), - &ByteOffset, - NULL); - if (NT_SUCCESS(Status)) - { - LARGE_INTEGER Zero; - Zero.QuadPart = 0LL; - - Status = IoSetInformation(FileObject, - FileEndOfFileInformation, - sizeof(LARGE_INTEGER), - &Zero); - ASSERT(NT_SUCCESS(Status)); - } - } - } -#endif - if (AllocationAttributes & SEC_IMAGE) { Status = MmCreateImageSection(SectionObject, @@ -5042,7 +4332,7 @@ MmCreateSection (OUT PVOID * Section, FileObject); } #ifndef NEWCC - else if (FileHandle != NULL) + else if (FileObject != NULL) { Status = MmCreateDataFileSection(SectionObject, DesiredAccess, @@ -5050,7 +4340,8 @@ MmCreateSection (OUT PVOID * Section, MaximumSize, SectionPageProtection, AllocationAttributes, - FileObject); + FileObject, + FileHandle != NULL); } #else else if (FileHandle != NULL || FileObject != NULL) @@ -5066,7 +4357,7 @@ MmCreateSection (OUT PVOID * Section, #endif else { - /* All cases should be handled above, and the Physical Memorw section was created at initialization phase */ + /* All cases should be handled above */ ASSERT(FALSE); Status = STATUS_INVALID_PARAMETER; if (FileObject) @@ -5087,13 +4378,14 @@ MmArePagesResident( BOOLEAN Ret = TRUE; PMM_SECTION_SEGMENT Segment; LARGE_INTEGER SegmentOffset, RangeEnd; + PMMSUPPORT AddressSpace = Process ? &Process->Vm : MmGetKernelAddressSpace(); - MmLockAddressSpace(&Process->Vm); + MmLockAddressSpace(AddressSpace); - MemoryArea = MmLocateMemoryAreaByAddress(&Process->Vm, Address); + MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace, Address); if (MemoryArea == NULL) { - MmUnlockAddressSpace(&Process->Vm); + MmUnlockAddressSpace(AddressSpace); return FALSE; } @@ -5123,7 +4415,7 @@ MmArePagesResident( MmUnlockSectionSegment(Segment); - MmUnlockAddressSpace(&Process->Vm); + MmUnlockAddressSpace(AddressSpace); return Ret; } @@ -5137,14 +4429,15 @@ MmMakePagesResident( PMEMORY_AREA MemoryArea; PMM_SECTION_SEGMENT Segment; LARGE_INTEGER SegmentOffset, RangeEnd; + PMMSUPPORT AddressSpace = Process ? &Process->Vm : MmGetKernelAddressSpace(); - MmLockAddressSpace(&Process->Vm); + MmLockAddressSpace(AddressSpace); - MemoryArea = MmLocateMemoryAreaByAddress(&Process->Vm, Address); + MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace, Address); if (MemoryArea == NULL) { - MmUnlockAddressSpace(&Process->Vm); - return FALSE; + MmUnlockAddressSpace(AddressSpace); + return STATUS_NOT_MAPPED_VIEW; } /* Only supported in old Mm for now */ @@ -5160,6 +4453,8 @@ MmMakePagesResident( RangeEnd.QuadPart = PAGE_ROUND_UP((ULONG_PTR)Address + Length) - MA_GetStartingAddress(MemoryArea) + MemoryArea->SectionData.ViewOffset.QuadPart; + DPRINT("MmMakePagesResident: Segment %p, 0x%I64x -> 0x%I64x\n", Segment, SegmentOffset.QuadPart, RangeEnd.QuadPart); + while (SegmentOffset.QuadPart < RangeEnd.QuadPart) { ULONG_PTR Entry = MmGetPageEntrySectionSegment(Segment, &SegmentOffset); @@ -5168,9 +4463,9 @@ MmMakePagesResident( while (MM_IS_WAIT_PTE(Entry)) { MmUnlockSectionSegment(Segment); - MmUnlockAddressSpace(&Process->Vm); + MmUnlockAddressSpace(AddressSpace); MiWaitForPageEvent(NULL, NULL); - MmLockAddressSpace(&Process->Vm); + MmLockAddressSpace(AddressSpace); MmLockSectionSegment(Segment); Entry = MmGetPageEntrySectionSegment(Segment, &SegmentOffset); } @@ -5189,7 +4484,7 @@ MmMakePagesResident( */ MmSetPageEntrySectionSegment(Segment, &SegmentOffset, MAKE_SWAP_SSE(MM_WAIT_ENTRY)); MmUnlockSectionSegment(Segment); - MmUnlockAddressSpace(&Process->Vm); + MmUnlockAddressSpace(AddressSpace); /* FIXME: Read the whole range at once instead of one page at a time */ Status = MiReadPage(MemoryArea, SegmentOffset.QuadPart, &Page); @@ -5203,16 +4498,279 @@ MmMakePagesResident( return Status; } - MmLockAddressSpace(&Process->Vm); + MmLockAddressSpace(AddressSpace); MmLockSectionSegment(Segment); - MmSetPageEntrySectionSegment(Segment, &SegmentOffset, MAKE_SSE(Page << PAGE_SHIFT, 1)); + + /* We set it with 0 ref count, nobody maps this page yet. */ + MmSetPageEntrySectionSegment(Segment, &SegmentOffset, MAKE_SSE(Page << PAGE_SHIFT, 0)); + MiSetPageEvent(Process, Address); } SegmentOffset.QuadPart += PAGE_SIZE; } MmUnlockSectionSegment(Segment); - MmUnlockAddressSpace(&Process->Vm); + MmUnlockAddressSpace(AddressSpace); + return STATUS_SUCCESS; +} + +NTSTATUS +NTAPI +MmRosFlushVirtualMemory( + _In_ PEPROCESS Process, + _Inout_ PVOID* Address, + _Inout_ PSIZE_T Length, + _Out_ PIO_STATUS_BLOCK Iosb) +{ + PMEMORY_AREA MemoryArea; + PMM_SECTION_SEGMENT Segment; + LARGE_INTEGER SegmentOffset, RangeEnd; + PMMSUPPORT AddressSpace = Process ? &Process->Vm : MmGetKernelAddressSpace(); + PVOID CurrentAddress; + + PAGED_CODE(); + + MmLockAddressSpace(AddressSpace); + + DPRINT("Flushing Process %p at %p --> 0x%x", Process, *Address, *Length); + + MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace, *Address); + if ((MemoryArea == NULL) || (MemoryArea->Type != MEMORY_AREA_SECTION_VIEW) || + (MemoryArea->VadNode.u.VadFlags.VadType == VadImageMap)) + { + MmUnlockAddressSpace(AddressSpace); + return STATUS_NOT_MAPPED_VIEW; + } + + Segment = MemoryArea->SectionData.Segment; + + SegmentOffset.QuadPart = PAGE_ROUND_DOWN(*Address) - MA_GetStartingAddress(MemoryArea) + + MemoryArea->SectionData.ViewOffset.QuadPart; + RangeEnd.QuadPart = PAGE_ROUND_UP((ULONG_PTR)*Address + *Length) - MA_GetStartingAddress(MemoryArea) + + MemoryArea->SectionData.ViewOffset.QuadPart; + + CurrentAddress = *Address; + + MmUnlockAddressSpace(AddressSpace); + + MmLockSectionSegment(Segment); + + Iosb->Information = 0; + while (SegmentOffset.QuadPart < RangeEnd.QuadPart) + { + ULONG_PTR Entry = MmGetPageEntrySectionSegment(Segment, &SegmentOffset); + + /* Let any pending read proceed */ + while (MM_IS_WAIT_PTE(Entry)) + { + MmUnlockSectionSegment(Segment); + MiWaitForPageEvent(NULL, NULL); + MmLockSectionSegment(Segment); + Entry = MmGetPageEntrySectionSegment(Segment, &SegmentOffset); + } + + /* We are called from Cc, this can't be backed by the page files */ + ASSERT(!IS_SWAP_FROM_SSE(Entry)); + + /* At this point, there may be a valid page there */ + if (Entry != 0) + { + /* This will write the page to disk, if needed */ + MmCheckDirtySegment(Segment, &SegmentOffset, MmIsDirtyPage(Process, CurrentAddress), FALSE); + Iosb->Information += PAGE_SIZE; + } + SegmentOffset.QuadPart += PAGE_SIZE; + CurrentAddress = (PVOID)((ULONG_PTR)CurrentAddress + PAGE_SIZE); + } + + MmUnlockSectionSegment(Segment); + + return STATUS_SUCCESS; +} + +_Requires_exclusive_lock_held_(Segment->Lock) +BOOLEAN +NTAPI +MmCheckDirtySegment( + PMM_SECTION_SEGMENT Segment, + PLARGE_INTEGER Offset, + BOOLEAN ForceDirty, + BOOLEAN PageOut) +{ + ULONG_PTR Entry; + NTSTATUS Status; + PFN_NUMBER Page; + + ASSERT(Segment->Locked); + + DPRINT("Checking segment for file %wZ at offset 0x%I64X.\n", &Segment->FileObject->FileName, Offset->QuadPart); + + Entry = MmGetPageEntrySectionSegment(Segment, Offset); + if (Entry == 0) + return FALSE; + + Page = PFN_FROM_SSE(Entry); + if ((IS_DIRTY_SSE(Entry)) || ForceDirty) + { + BOOLEAN DirtyAgain; + + /* We got a dirty entry. Is this segment copy on write */ + ASSERT(!Segment->WriteCopy); + ASSERT(Segment->SegFlags & MM_DATAFILE_SEGMENT); + + /* Insert the cleaned entry back. Keep one ref to the page so nobody pages it out again behind us */ + MmSetPageEntrySectionSegment(Segment, Offset, + MAKE_SSE(Page << PAGE_SHIFT, SHARE_COUNT_FROM_SSE(Entry) + 1)); + + /* Tell the other users that we are clean again */ + MmSetCleanAllRmaps(Page); + + MmUnlockSectionSegment(Segment); + + /* Go ahead and write the page */ + Status = MiWritePage(Segment, Offset->QuadPart, Page); + + MmLockSectionSegment(Segment); + + /* Get the entry again */ + Entry = MmGetPageEntrySectionSegment(Segment, Offset); + ASSERT(PFN_FROM_SSE(Entry) == Page); + + if (!NT_SUCCESS(Status)) + { + /* Damn, this failed. Consider this page as still dirty */ + DPRINT1("MiWritePage FAILED: Status 0x%08x!\n", Status); + DirtyAgain = TRUE; + } + else + { + /* Check if someone dirtified this page while we were not looking */ + DirtyAgain = IS_DIRTY_SSE(Entry) || MmIsDirtyPageRmap(Page); + } + + /* Drop the reference we got */ + Entry = MAKE_SSE(Page << PAGE_SHIFT, SHARE_COUNT_FROM_SSE(Entry) - 1); + if (DirtyAgain) Entry = DIRTY_SSE(Entry); + MmSetPageEntrySectionSegment(Segment, Offset, Entry); + } + + /* Were this page hanging there just for the sake of being present ? */ + if (!IS_DIRTY_SSE(Entry) && (SHARE_COUNT_FROM_SSE(Entry) == 0) && PageOut) + { + /* Yes. Release it */ + MmSetPageEntrySectionSegment(Segment, Offset, 0); + MmReleasePageMemoryConsumer(MC_USER, Page); + /* Tell the caller we released the page */ + return TRUE; + } + + return FALSE; +} + +NTSTATUS +NTAPI +MmMakePagesDirty( + _In_ PEPROCESS Process, + _In_ PVOID Address, + _In_ ULONG Length) +{ + PMEMORY_AREA MemoryArea; + PMM_SECTION_SEGMENT Segment; + LARGE_INTEGER SegmentOffset, RangeEnd; + PMMSUPPORT AddressSpace = Process ? &Process->Vm : MmGetKernelAddressSpace(); + + MmLockAddressSpace(AddressSpace); + + MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace, Address); + if (MemoryArea == NULL) + { + MmUnlockAddressSpace(AddressSpace); + return STATUS_NOT_MAPPED_VIEW; + } + + /* Only supported in old Mm for now */ + ASSERT(MemoryArea->Type == MEMORY_AREA_SECTION_VIEW); + /* For file mappings */ + ASSERT(MemoryArea->VadNode.u.VadFlags.VadType != VadImageMap); + + Segment = MemoryArea->SectionData.Segment; + MmLockSectionSegment(Segment); + + SegmentOffset.QuadPart = PAGE_ROUND_DOWN(Address) - MA_GetStartingAddress(MemoryArea) + + MemoryArea->SectionData.ViewOffset.QuadPart; + RangeEnd.QuadPart = PAGE_ROUND_UP((ULONG_PTR)Address + Length) - MA_GetStartingAddress(MemoryArea) + + MemoryArea->SectionData.ViewOffset.QuadPart; + + DPRINT("MmMakePagesResident: Segment %p, 0x%I64x -> 0x%I64x\n", Segment, SegmentOffset.QuadPart, RangeEnd.QuadPart); + + while (SegmentOffset.QuadPart < RangeEnd.QuadPart) + { + ULONG_PTR Entry = MmGetPageEntrySectionSegment(Segment, &SegmentOffset); + + /* Let any pending read proceed */ + while (MM_IS_WAIT_PTE(Entry)) + { + MmUnlockSectionSegment(Segment); + MmUnlockAddressSpace(AddressSpace); + MiWaitForPageEvent(NULL, NULL); + MmLockAddressSpace(AddressSpace); + MmLockSectionSegment(Segment); + Entry = MmGetPageEntrySectionSegment(Segment, &SegmentOffset); + } + + /* We are called from Cc, this can't be backed by the page files */ + ASSERT(!IS_SWAP_FROM_SSE(Entry)); + + /* If there is no page there, there is nothing to make dirty */ + if (Entry != 0) + { + /* Dirtify the entry */ + MmSetPageEntrySectionSegment(Segment, &SegmentOffset, DIRTY_SSE(Entry)); + } + + SegmentOffset.QuadPart += PAGE_SIZE; + } + + MmUnlockSectionSegment(Segment); + + MmUnlockAddressSpace(AddressSpace); + return STATUS_SUCCESS; +} + +NTSTATUS +NTAPI +MmExtendSection( + _In_ PVOID _Section, + _Inout_ PLARGE_INTEGER NewSize) +{ + PSECTION Section = _Section; + + /* It makes no sense to extend an image mapping */ + if (Section->u.Flags.Image) + return STATUS_SECTION_NOT_EXTENDED; + + /* Nor is it possible to extend a page file mapping */ + if (!Section->u.Flags.File) + return STATUS_SECTION_NOT_EXTENDED; + + if (!MiIsRosSectionObject(Section)) + return STATUS_NOT_IMPLEMENTED; + + /* We just extend the sizes. Shrinking is a no-op ? */ + if (NewSize->QuadPart > Section->SizeOfSection.QuadPart) + { + PMM_SECTION_SEGMENT Segment = (PMM_SECTION_SEGMENT)Section->Segment; + Section->SizeOfSection = *NewSize; + + MmLockSectionSegment(Segment); + if (Segment->RawLength.QuadPart < NewSize->QuadPart) + { + Segment->RawLength = *NewSize; + Segment->Length.QuadPart = (NewSize->QuadPart + PAGE_SIZE - 1) & ~((LONGLONG)PAGE_SIZE); + } + MmUnlockSectionSegment(Segment); + } + return STATUS_SUCCESS; } diff --git a/ntoskrnl/po/power.c b/ntoskrnl/po/power.c index 9490da817ce..7d2f4d7114a 100644 --- a/ntoskrnl/po/power.c +++ b/ntoskrnl/po/power.c @@ -4,7 +4,7 @@ * FILE: ntoskrnl/po/power.c * PURPOSE: Power Manager * PROGRAMMERS: Casper S. Hornstrup (chorns@users.sourceforge.net) - * Hervé Poussineau (hpoussin@reactos.com) + * Herv� Poussineau (hpoussin@reactos.com) */ /* INCLUDES ******************************************************************/ @@ -1075,7 +1075,7 @@ NtSetSystemPowerState(IN POWER_ACTION SystemAction, #ifndef NEWCC /* Flush dirty cache pages */ /* XXX: Is that still mandatory? As now we'll wait on lazy writer to complete? */ - CcRosFlushDirtyPages(-1, &Dummy, FALSE, FALSE); //HACK: We really should wait here! + CcRosFlushDirtyPages(-1, &Dummy, TRUE, FALSE); //HACK: We really should wait here! #else Dummy = 0; #endif From d932bdb9505e270830e0eb177b523f39d32d1f10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Fri, 22 Jan 2021 09:30:55 +0100 Subject: [PATCH 22/99] [NTOS:MM] Fix locking when paging out --- ntoskrnl/mm/rmap.c | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/ntoskrnl/mm/rmap.c b/ntoskrnl/mm/rmap.c index d1cfb584dce..21b5b246457 100644 --- a/ntoskrnl/mm/rmap.c +++ b/ntoskrnl/mm/rmap.c @@ -70,7 +70,7 @@ MmPageOutPhysicalAddress(PFN_NUMBER Page) if (entry == NULL) { MiReleasePfnLock(OldIrql); - return STATUS_UNSUCCESSFUL; + goto WriteSegment; } Process = entry->Process; @@ -176,6 +176,12 @@ MmPageOutPhysicalAddress(PFN_NUMBER Page) MmSetDirtyPage(Process, Address); MmUnlockAddressSpace(AddressSpace); + if (Address < MmSystemRangeStart) + { + ExReleaseRundownProtection(&Process->RundownProtect); + ObDereferenceObject(Process); + } + return STATUS_UNSUCCESSFUL; } } @@ -199,6 +205,11 @@ MmPageOutPhysicalAddress(PFN_NUMBER Page) MmSetDirtyPage(Process, Address); MmUnlockAddressSpace(AddressSpace); + if (Address < MmSystemRangeStart) + { + ExReleaseRundownProtection(&Process->RundownProtect); + ObDereferenceObject(Process); + } return STATUS_UNSUCCESSFUL; } } @@ -256,12 +267,7 @@ MmPageOutPhysicalAddress(PFN_NUMBER Page) KeBugCheck(MEMORY_MANAGEMENT); } - if (Address < MmSystemRangeStart) - { - ExReleaseRundownProtection(&Process->RundownProtect); - ObDereferenceObject(Process); - } - +WriteSegment: /* Now write this page to file, if needed */ Segment = MmGetSectionAssociation(Page, &SegmentOffset); if (Segment) From ba49c390412450157275330268bcaab096aa6149 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Tue, 17 Nov 2020 17:38:53 +0100 Subject: [PATCH 23/99] [NTOS/MM] Assert there are no RMAP left before freeing the page --- ntoskrnl/mm/rmap.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ntoskrnl/mm/rmap.c b/ntoskrnl/mm/rmap.c index 21b5b246457..80576f4cc59 100644 --- a/ntoskrnl/mm/rmap.c +++ b/ntoskrnl/mm/rmap.c @@ -225,9 +225,8 @@ MmPageOutPhysicalAddress(PFN_NUMBER Page) /* We can finally let this page go */ MmDeleteRmap(Page, Process, Address); - MmReleasePageMemoryConsumer(MC_USER, Page); - ASSERT(MmGetRmapListHeadPage(Page) == NULL); + MmReleasePageMemoryConsumer(MC_USER, Page); if (Address < MmSystemRangeStart) { From b46173edaff285e3c414070eb1d1655f2873d998 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Wed, 18 Nov 2020 10:29:17 +0100 Subject: [PATCH 24/99] [NTOS/MM] DO not assert when being passed invalid parameters in MmCreateSection --- ntoskrnl/mm/section.c | 1 - 1 file changed, 1 deletion(-) diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index 39ae8bf75f5..6c521522f4f 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -4358,7 +4358,6 @@ MmCreateSection (OUT PVOID * Section, else { /* All cases should be handled above */ - ASSERT(FALSE); Status = STATUS_INVALID_PARAMETER; if (FileObject) ObDereferenceObject(FileObject); From d440434a8e120067abba5630214dd804ba06449f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Wed, 18 Nov 2020 12:14:41 +0100 Subject: [PATCH 25/99] [NTOS/MM] Do not error when FS returns STATUS_END_OF_FILE for paged read. Also remove a seemingly invalid ASSERT --- ntoskrnl/mm/section.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index 6c521522f4f..46bf5cb571d 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -1209,6 +1209,11 @@ MiReadPage(PMEMORY_AREA MemoryArea, MmUnmapLockedPages (Mdl->MappedSystemVa, Mdl); } + if (Status == STATUS_END_OF_FILE) + { + Status = STATUS_SUCCESS; + } + return Status; } @@ -2211,7 +2216,7 @@ MmCreateDataFileSection(PSECTION *SectionObject, if (!GotFileHandle) { ASSERT(UMaximumSize != NULL); - ASSERT(UMaximumSize->QuadPart != 0); + // ASSERT(UMaximumSize->QuadPart != 0); MaximumSize = *UMaximumSize; } else From 093333740468ad0af68280d53c928ed4bacfeea1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Wed, 18 Nov 2020 17:45:47 +0100 Subject: [PATCH 26/99] [NTOS/MM] Do not dirtify section pages for system-space maps Cc does it for itself. --- ntoskrnl/mm/section.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index 46bf5cb571d..f108533ed2e 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -3343,8 +3343,12 @@ MmFreeSectionPage(PVOID Context, MEMORY_AREA* MemoryArea, PVOID Address, else { if (Process) + { MmDeleteRmap(Page, Process, Address); - MmUnsharePageEntrySectionSegment(MemoryArea, Segment, &Offset, Dirty, FALSE, NULL); + } + + /* We don't dirtify for System Space Maps. We let Cc manage that */ + MmUnsharePageEntrySectionSegment(MemoryArea, Segment, &Offset, Process ? Dirty : FALSE, FALSE, NULL); } } } From 774a4c703fdf2413339064422f39ef2dbfd88fcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Tue, 24 Nov 2020 14:07:06 +0100 Subject: [PATCH 27/99] [NTOS/MM] Misc fixes Purge data section object before creating an image mapping Zero-out the tail of the page after reading from file Properly map page as read-only when paging-in a COW memory map. --- ntoskrnl/mm/section.c | 92 ++++++++++++++++++++++++++++++++----------- 1 file changed, 68 insertions(+), 24 deletions(-) diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index f108533ed2e..8df9b99cfb8 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -1214,6 +1214,18 @@ MiReadPage(PMEMORY_AREA MemoryArea, Status = STATUS_SUCCESS; } + if ((SegOffset + PAGE_SIZE) > MemoryArea->SectionData.Segment->RawLength.QuadPart) + { + KIRQL OldIrql; + PUCHAR PageMap; + + /* Zero out the end of it */ + PageMap = MiMapPageInHyperSpace(PsGetCurrentProcess(), *Page, &OldIrql); + RtlZeroMemory(PageMap + MemoryArea->SectionData.Segment->RawLength.QuadPart - SegOffset, + PAGE_SIZE - (MemoryArea->SectionData.Segment->RawLength.QuadPart - SegOffset)); + MiUnmapPageInHyperSpace(PsGetCurrentProcess(), PageMap, OldIrql); + } + return Status; } @@ -1448,32 +1460,24 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, { SWAPENTRY DummyEntry; - /* - * Is it a wait entry? - */ - if (HasSwapEntry) + MmGetPageFileMapping(Process, Address, &SwapEntry); + if (SwapEntry == MM_WAIT_ENTRY) { - MmGetPageFileMapping(Process, Address, &SwapEntry); - - if (SwapEntry == MM_WAIT_ENTRY) - { - MmUnlockSectionSegment(Segment); - MmUnlockAddressSpace(AddressSpace); - MiWaitForPageEvent(NULL, NULL); - MmLockAddressSpace(AddressSpace); - return STATUS_MM_RESTART_OPERATION; - } - - /* - * Must be private page we have swapped out. - */ - - /* - * Sanity check - */ - MmDeletePageFileMapping(Process, Address, &SwapEntry); + MmUnlockSectionSegment(Segment); + MmUnlockAddressSpace(AddressSpace); + MiWaitForPageEvent(NULL, NULL); + MmLockAddressSpace(AddressSpace); + return STATUS_MM_RESTART_OPERATION; } + /* + * Must be private page we have swapped out. + */ + + /* + * Sanity check + */ + MmDeletePageFileMapping(Process, Address, &SwapEntry); MmUnlockSectionSegment(Segment); /* Tell everyone else we are serving the fault. */ @@ -1709,7 +1713,7 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, /* Map the page into the process address space */ Status = MmCreateVirtualMapping(Process, PAddress, - Region->Protect, + Attributes, &Page, 1); if (!NT_SUCCESS(Status)) @@ -3096,6 +3100,7 @@ MmCreateImageSection(PSECTION *SectionObject, if (ImageSectionObject == NULL) { NTSTATUS StatusExeFmt; + PMM_SECTION_SEGMENT DataSectionObject; ImageSectionObject = ExAllocatePoolWithTag(NonPagedPool, sizeof(MM_IMAGE_SECTION_OBJECT), TAG_MM_SECTION_SEGMENT); if (ImageSectionObject == NULL) @@ -3112,8 +3117,47 @@ MmCreateImageSection(PSECTION *SectionObject, ImageSectionObject->RefCount = 1; FileObject->SectionObjectPointer->ImageSectionObject = ImageSectionObject; + /* Get a ref on the data section object */ + DataSectionObject = FileObject->SectionObjectPointer->DataSectionObject; + while (DataSectionObject && (DataSectionObject->SegFlags & (MM_SEGMENT_INDELETE | MM_SEGMENT_INCREATE))) + { + LARGE_INTEGER ShortTime; + + MiReleasePfnLock(OldIrql); + + ShortTime.QuadPart = - 10 * 100 * 1000; + KeDelayExecutionThread(KernelMode, FALSE, &ShortTime); + + OldIrql = MiAcquirePfnLock(); + DataSectionObject = FileObject->SectionObjectPointer->DataSectionObject; + ASSERT(DataSectionObject->SegFlags & MM_DATAFILE_SEGMENT); + } + + /* Get a ref on it. */ + if (DataSectionObject) + InterlockedIncrementUL(&DataSectionObject->RefCount); + MiReleasePfnLock(OldIrql); + if (DataSectionObject) + { + if ((DataSectionObject->SectionCount - (FileObject->SectionObjectPointer->SharedCacheMap != NULL)) > 0) + { + /* Someone's got a section opened. Deny creation */ + DPRINT1("Denying image creation for %wZ: Sections opened: %lu.\n", + &FileObject->FileName, DataSectionObject->SectionCount); + InterlockedExchangePointer(&FileObject->SectionObjectPointer->ImageSectionObject, NULL); + ExFreePoolWithTag(ImageSectionObject, TAG_MM_SECTION_SEGMENT); + MmDereferenceSegment(DataSectionObject); + ObDereferenceObject(FileObject); + ObDereferenceObject(Section); + return STATUS_ACCESS_DENIED; + } + + /* Purge the cache. */ + CcPurgeCacheSection(FileObject->SectionObjectPointer, NULL, 0, FALSE); + } + StatusExeFmt = ExeFmtpCreateImageSection(FileObject, ImageSectionObject); if (!NT_SUCCESS(StatusExeFmt)) From c05f3d8d1ce5641f1768ecd775a375a34e368cf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Wed, 2 Dec 2020 11:17:59 +0100 Subject: [PATCH 28/99] [FASTFAT_NEW] Enable the driver --- drivers/filesystems/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/filesystems/CMakeLists.txt b/drivers/filesystems/CMakeLists.txt index 7485d433b5a..3d5277dfa74 100644 --- a/drivers/filesystems/CMakeLists.txt +++ b/drivers/filesystems/CMakeLists.txt @@ -2,7 +2,8 @@ add_subdirectory(btrfs) add_subdirectory(cdfs) add_subdirectory(ext2) -add_subdirectory(fastfat) +#add_subdirectory(fastfat) +add_subdirectory(fastfat_new) add_subdirectory(ffs) add_subdirectory(fs_rec) add_subdirectory(msfs) From 91edefa11e4d7230155dcaafe052a6fd10d7dd57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Wed, 2 Dec 2020 11:40:25 +0100 Subject: [PATCH 29/99] [NTOS:MM] Do not limit ourselves in reading or writing the file --- ntoskrnl/mm/section.c | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index 8df9b99cfb8..f3b30dedaed 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -214,16 +214,9 @@ MiWritePage(PMM_SECTION_SEGMENT Segment, PMDL Mdl = (PMDL)MdlBase; PFILE_OBJECT FileObject = Segment->FileObject; LARGE_INTEGER FileOffset; - PFSRTL_COMMON_FCB_HEADER Fcb = FileObject->FsContext; FileOffset.QuadPart = Segment->Image.FileOffset + SegOffset; - /* Check if we are not writing off-limit */ - if (FileOffset.QuadPart >= Fcb->AllocationSize.QuadPart) - { - return STATUS_SUCCESS; - } - RtlZeroMemory(MdlBase, sizeof(MdlBase)); MmInitializeMdl(Mdl, NULL, PAGE_SIZE); MmBuildMdlFromPages(Mdl, &Page); @@ -1179,18 +1172,15 @@ MiReadPage(PMEMORY_AREA MemoryArea, PMDL Mdl = (PMDL)MdlBase; PFILE_OBJECT FileObject = MemoryArea->SectionData.Segment->FileObject; LARGE_INTEGER FileOffset; - PFSRTL_COMMON_FCB_HEADER Fcb = FileObject->FsContext; FileOffset.QuadPart = MemoryArea->SectionData.Segment->Image.FileOffset + SegOffset; + DPRINT("Reading file at offset %08x:%08x\n", FileOffset.HighPart, FileOffset.LowPart); + Status = MmRequestPageMemoryConsumer(MC_USER, FALSE, Page); if (!NT_SUCCESS(Status)) return Status; - /* Check if we are beyond the file */ - if (FileOffset.QuadPart > Fcb->FileSize.QuadPart) - return STATUS_SUCCESS; - RtlZeroMemory(MdlBase, sizeof(MdlBase)); MmInitializeMdl(Mdl, NULL, PAGE_SIZE); MmBuildMdlFromPages(Mdl, Page); @@ -1204,6 +1194,7 @@ MiReadPage(PMEMORY_AREA MemoryArea, KeWaitForSingleObject(&Event, WrPageIn, KernelMode, FALSE, NULL); Status = IoStatus.Status; } + if (Mdl->MdlFlags & MDL_MAPPED_TO_SYSTEM_VA) { MmUnmapLockedPages (Mdl->MappedSystemVa, Mdl); @@ -4539,6 +4530,7 @@ MmMakePagesResident( MmUnlockAddressSpace(AddressSpace); /* FIXME: Read the whole range at once instead of one page at a time */ + /* Ignore file size, as Cc already checked on its side. */ Status = MiReadPage(MemoryArea, SegmentOffset.QuadPart, &Page); if (!NT_SUCCESS(Status)) { From 7dab3583cc4c80304df61658fe4b1eeea8983f5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Wed, 2 Dec 2020 12:12:11 +0100 Subject: [PATCH 30/99] [NTOS/MM] Force flushing when the page is dirtified in user mode --- ntoskrnl/mm/section.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index f3b30dedaed..6dcf764d7a1 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -4620,7 +4620,7 @@ MmRosFlushVirtualMemory( if (Entry != 0) { /* This will write the page to disk, if needed */ - MmCheckDirtySegment(Segment, &SegmentOffset, MmIsDirtyPage(Process, CurrentAddress), FALSE); + MmCheckDirtySegment(Segment, &SegmentOffset, Process ? MmIsDirtyPage(Process, CurrentAddress) : FALSE, FALSE); Iosb->Information += PAGE_SIZE; } SegmentOffset.QuadPart += PAGE_SIZE; @@ -4694,7 +4694,10 @@ MmCheckDirtySegment( /* Drop the reference we got */ Entry = MAKE_SSE(Page << PAGE_SHIFT, SHARE_COUNT_FROM_SSE(Entry) - 1); - if (DirtyAgain) Entry = DIRTY_SSE(Entry); + if (DirtyAgain) + { + Entry = DIRTY_SSE(Entry); + } MmSetPageEntrySectionSegment(Segment, Offset, Entry); } From 36e18aab35049f5e4cb36d0751993f9af40e27cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Wed, 2 Dec 2020 12:14:49 +0100 Subject: [PATCH 31/99] [NTOS:CC] Remove unused functions --- ntoskrnl/cc/view.c | 59 ---------------------------------- ntoskrnl/include/internal/cc.h | 15 --------- 2 files changed, 74 deletions(-) diff --git a/ntoskrnl/cc/view.c b/ntoskrnl/cc/view.c index ddbe77e22af..a14635f2135 100644 --- a/ntoskrnl/cc/view.c +++ b/ntoskrnl/cc/view.c @@ -466,65 +466,6 @@ CcRosUnmarkDirtyVacb ( } } -NTSTATUS -NTAPI -CcRosMarkDirtyFile ( - PROS_SHARED_CACHE_MAP SharedCacheMap, - LONGLONG FileOffset) -{ - PROS_VACB Vacb; - - ASSERT(SharedCacheMap); - - DPRINT("CcRosMarkDirtyVacb(SharedCacheMap 0x%p, FileOffset %I64u)\n", - SharedCacheMap, FileOffset); - - Vacb = CcRosLookupVacb(SharedCacheMap, FileOffset); - if (Vacb == NULL) - { - KeBugCheck(CACHE_MANAGER); - } - - CcRosReleaseVacb(SharedCacheMap, Vacb, Vacb->Valid, TRUE, FALSE); - - return STATUS_SUCCESS; -} - -/* - * Note: this is not the contrary function of - * CcRosMapVacbInKernelSpace() - */ -NTSTATUS -NTAPI -CcRosUnmapVacb ( - PROS_SHARED_CACHE_MAP SharedCacheMap, - LONGLONG FileOffset, - BOOLEAN NowDirty) -{ - PROS_VACB Vacb; - - ASSERT(SharedCacheMap); - - DPRINT("CcRosUnmapVacb(SharedCacheMap 0x%p, FileOffset %I64u, NowDirty %u)\n", - SharedCacheMap, FileOffset, NowDirty); - - Vacb = CcRosLookupVacb(SharedCacheMap, FileOffset); - if (Vacb == NULL) - { - return STATUS_UNSUCCESSFUL; - } - - ASSERT(Vacb->MappedCount != 0); - if (InterlockedDecrement((PLONG)&Vacb->MappedCount) == 0) - { - CcRosVacbDecRefCount(Vacb); - } - - CcRosReleaseVacb(SharedCacheMap, Vacb, Vacb->Valid, NowDirty, FALSE); - - return STATUS_SUCCESS; -} - static BOOLEAN CcRosFreeUnusedVacb ( diff --git a/ntoskrnl/include/internal/cc.h b/ntoskrnl/include/internal/cc.h index 9e186a61d11..862a759ef7a 100644 --- a/ntoskrnl/include/internal/cc.h +++ b/ntoskrnl/include/internal/cc.h @@ -342,14 +342,6 @@ BOOLEAN NTAPI CcInitializeCacheManager(VOID); -NTSTATUS -NTAPI -CcRosUnmapVacb( - PROS_SHARED_CACHE_MAP SharedCacheMap, - LONGLONG FileOffset, - BOOLEAN NowDirty -); - PROS_VACB NTAPI CcRosLookupVacb( @@ -361,13 +353,6 @@ VOID NTAPI CcInitCacheZeroPage(VOID); -NTSTATUS -NTAPI -CcRosMarkDirtyFile( - PROS_SHARED_CACHE_MAP SharedCacheMap, - LONGLONG FileOffset -); - VOID NTAPI CcRosMarkDirtyVacb( From 7f47f032969cac0eaaf3246a3e0ae9da3008aeb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Wed, 2 Dec 2020 12:27:32 +0100 Subject: [PATCH 32/99] [NTOS:CC] Rewrite the non-cached path of CcZeroData --- ntoskrnl/cc/copy.c | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/ntoskrnl/cc/copy.c b/ntoskrnl/cc/copy.c index 5bb14cc210a..ced4271fc9b 100644 --- a/ntoskrnl/cc/copy.c +++ b/ntoskrnl/cc/copy.c @@ -759,25 +759,25 @@ CcZeroData ( PMDL Mdl; ULONG i; ULONG CurrentLength; + PPFN_NUMBER PfnArray; - Mdl = _alloca(MmSizeOfMdl(NULL, MAX_ZERO_LENGTH)); + /* Setup our Mdl */ + Mdl = IoAllocateMdl(NULL, min(Length, MAX_ZERO_LENGTH), FALSE, FALSE, NULL); + if (!Mdl) + ExRaiseStatus(STATUS_INSUFFICIENT_RESOURCES); + PfnArray = MmGetMdlPfnArray(Mdl); + for (i = 0; i < BYTES_TO_PAGES(Mdl->ByteCount); i++) + PfnArray[i] = CcZeroPage; + Mdl->MdlFlags |= MDL_PAGES_LOCKED; + + /* Perform the write sequencially */ while (Length > 0) { - if (Length + WriteOffset.QuadPart % PAGE_SIZE > MAX_ZERO_LENGTH) - { - CurrentLength = MAX_ZERO_LENGTH - WriteOffset.QuadPart % PAGE_SIZE; - } - else - { - CurrentLength = Length; - } - MmInitializeMdl(Mdl, (PVOID)(ULONG_PTR)WriteOffset.QuadPart, CurrentLength); - Mdl->MdlFlags |= MDL_PAGES_LOCKED; - for (i = 0; i < ((Mdl->Size - sizeof(MDL)) / sizeof(ULONG)); i++) - { - ((PPFN_NUMBER)(Mdl + 1))[i] = CcZeroPage; - } + CurrentLength = min(Length, MAX_ZERO_LENGTH); + + Mdl->ByteCount = CurrentLength; + KeInitializeEvent(&Event, NotificationEvent, FALSE); Status = IoSynchronousPageWrite(FileObject, Mdl, &WriteOffset, &Event, &Iosb); if (Status == STATUS_PENDING) @@ -791,12 +791,15 @@ CcZeroData ( } if (!NT_SUCCESS(Status)) { - return FALSE; + IoFreeMdl(Mdl); + ExRaiseStatus(Status); } WriteOffset.QuadPart += CurrentLength; Length -= CurrentLength; } + IoFreeMdl(Mdl); + return TRUE; } From c295d6b229fc679a38eaea2f0797b7ffb139478d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Wed, 2 Dec 2020 12:28:38 +0100 Subject: [PATCH 33/99] [NTOS:CC] Fix releasing VACB when an exception is raised in CcZeroData & CcCopyWrite --- ntoskrnl/cc/copy.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ntoskrnl/cc/copy.c b/ntoskrnl/cc/copy.c index ced4271fc9b..17b7a366d17 100644 --- a/ntoskrnl/cc/copy.c +++ b/ntoskrnl/cc/copy.c @@ -557,8 +557,9 @@ CcCopyWrite ( return FALSE; /* FIXME: Honor FileObject FO_WRITE_THROUGH flag */ + ASSERT((FileOffset->QuadPart + Length) <= SharedCacheMap->SectionSize.QuadPart); - ASSERT((FileOffset->QuadPart + Length) <= SharedCacheMap->FileSize.QuadPart); + ASSERT((FileObject->Flags & FO_WRITE_THROUGH) == 0); CurrentOffset = FileOffset->QuadPart; while(CurrentOffset < WriteEnd) @@ -577,7 +578,6 @@ CcCopyWrite ( { if (!CcRosEnsureVacbResident(Vacb, Wait, FALSE, VacbOffset, VacbLength)) { - CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); return FALSE; } @@ -589,7 +589,8 @@ CcCopyWrite ( } _SEH2_FINALLY { - CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, TRUE, FALSE); + /* Do not mark the VACB as dirty if an exception was raised */ + CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, !_SEH2_AbnormalTermination(), FALSE); } _SEH2_END; } @@ -821,7 +822,6 @@ CcZeroData ( { if (!CcRosEnsureVacbResident(Vacb, Wait, FALSE, VacbOffset, VacbLength)) { - CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); return FALSE; } @@ -832,7 +832,8 @@ CcZeroData ( } _SEH2_FINALLY { - CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, TRUE, FALSE); + /* Do not mark the VACB as dirty if an exception was raised */ + CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, !_SEH2_AbnormalTermination(), FALSE); } _SEH2_END; } From c74cbf0c0bed7ae9831c23527931907de55379cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Wed, 2 Dec 2020 12:30:48 +0100 Subject: [PATCH 34/99] [NTOS/CC] Be more precise when notifying Mm about dirty pages --- ntoskrnl/cc/copy.c | 10 ++++++++++ ntoskrnl/cc/pin.c | 5 +++++ ntoskrnl/cc/view.c | 3 --- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/ntoskrnl/cc/copy.c b/ntoskrnl/cc/copy.c index 17b7a366d17..f92040aa0b2 100644 --- a/ntoskrnl/cc/copy.c +++ b/ntoskrnl/cc/copy.c @@ -586,6 +586,11 @@ CcCopyWrite ( Buffer = (PVOID)((ULONG_PTR)Buffer + VacbLength); CurrentOffset += VacbLength; Length -= VacbLength; + + /* Tell Mm */ + Status = MmMakePagesDirty(NULL, Add2Ptr(Vacb->BaseAddress, VacbOffset), VacbLength); + if (!NT_SUCCESS(Status)) + ExRaiseStatus(Status); } _SEH2_FINALLY { @@ -829,6 +834,11 @@ CcZeroData ( WriteOffset.QuadPart += VacbLength; Length -= VacbLength; + + /* Tell Mm */ + Status = MmMakePagesDirty(NULL, Add2Ptr(Vacb->BaseAddress, VacbOffset), VacbLength); + if (!NT_SUCCESS(Status)) + ExRaiseStatus(Status); } _SEH2_FINALLY { diff --git a/ntoskrnl/cc/pin.c b/ntoskrnl/cc/pin.c index 189ba480305..0d5642b9932 100644 --- a/ntoskrnl/cc/pin.c +++ b/ntoskrnl/cc/pin.c @@ -540,6 +540,11 @@ CcSetDirtyPinnedData ( CCTRACE(CC_API_DEBUG, "Bcb=%p Lsn=%p\n", Bcb, Lsn); + /* Tell Mm */ + MmMakePagesDirty(NULL, + Add2Ptr(iBcb->Vacb->BaseAddress, iBcb->PFCB.MappedFileOffset.QuadPart - iBcb->Vacb->FileOffset.QuadPart), + iBcb->PFCB.MappedLength); + if (!iBcb->Vacb->Dirty) { CcRosMarkDirtyVacb(iBcb->Vacb); diff --git a/ntoskrnl/cc/view.c b/ntoskrnl/cc/view.c index a14635f2135..0e07eb7f1b6 100644 --- a/ntoskrnl/cc/view.c +++ b/ntoskrnl/cc/view.c @@ -420,9 +420,6 @@ CcRosMarkDirtyVacb ( CcScheduleLazyWriteScan(FALSE); } KeReleaseQueuedSpinLock(LockQueueMasterLock, oldIrql); - - /* Tell Mm */ - MmMakePagesDirty(NULL, Vacb->BaseAddress, Length); } VOID From 70c62aa2c9e5b4bdabb05ac5b92f6c56ee166fac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Wed, 2 Dec 2020 12:39:58 +0100 Subject: [PATCH 35/99] [NTOS:CC] Fix Vacb size usage & check --- ntoskrnl/cc/view.c | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/ntoskrnl/cc/view.c b/ntoskrnl/cc/view.c index 0e07eb7f1b6..de4a75ea8f7 100644 --- a/ntoskrnl/cc/view.c +++ b/ntoskrnl/cc/view.c @@ -388,7 +388,6 @@ CcRosMarkDirtyVacb ( { KIRQL oldIrql; PROS_SHARED_CACHE_MAP SharedCacheMap; - ULONG Length = VACB_MAPPING_GRANULARITY; SharedCacheMap = Vacb->SharedCacheMap; @@ -398,12 +397,9 @@ CcRosMarkDirtyVacb ( ASSERT(!Vacb->Dirty); InsertTailList(&DirtyVacbListHead, &Vacb->DirtyVacbListEntry); -#if 0 - if (Vacb->FileOffset.QuadPart + Length > SharedCacheMap->SectionSize.QuadPart) - Length = SharedCacheMap->SectionSize.QuadPart - Vacb->FileOffset.QuadPart; -#endif - CcTotalDirtyPages += PAGE_ROUND_UP(Length) / PAGE_SIZE; - Vacb->SharedCacheMap->DirtyPages += PAGE_ROUND_UP(Length) / PAGE_SIZE; + /* FIXME: There is no reason to account for the whole VACB. */ + CcTotalDirtyPages += VACB_MAPPING_GRANULARITY / PAGE_SIZE; + Vacb->SharedCacheMap->DirtyPages += VACB_MAPPING_GRANULARITY / PAGE_SIZE; CcRosVacbIncRefCount(Vacb); /* Move to the tail of the LRU list */ @@ -430,7 +426,6 @@ CcRosUnmarkDirtyVacb ( { KIRQL oldIrql; PROS_SHARED_CACHE_MAP SharedCacheMap; - ULONG Length = VACB_MAPPING_GRANULARITY; SharedCacheMap = Vacb->SharedCacheMap; @@ -447,12 +442,8 @@ CcRosUnmarkDirtyVacb ( RemoveEntryList(&Vacb->DirtyVacbListEntry); InitializeListHead(&Vacb->DirtyVacbListEntry); -#if 0 - if (Vacb->FileOffset.QuadPart + Length > SharedCacheMap->SectionSize.QuadPart) - Length = SharedCacheMap->SectionSize.QuadPart - Vacb->FileOffset.QuadPart; -#endif - CcTotalDirtyPages -= PAGE_ROUND_UP(Length) / PAGE_SIZE; - Vacb->SharedCacheMap->DirtyPages -= PAGE_ROUND_UP(Length) / PAGE_SIZE; + CcTotalDirtyPages -= VACB_MAPPING_GRANULARITY / PAGE_SIZE; + Vacb->SharedCacheMap->DirtyPages -= VACB_MAPPING_GRANULARITY / PAGE_SIZE; CcRosVacbDecRefCount(Vacb); @@ -718,8 +709,11 @@ CcRosEnsureVacbResident( ASSERT((Offset + Length) <= VACB_MAPPING_GRANULARITY); - if ((Vacb->FileOffset.QuadPart + Offset) > Vacb->SharedCacheMap->FileSize.QuadPart) + if ((Vacb->FileOffset.QuadPart + Offset) > Vacb->SharedCacheMap->SectionSize.QuadPart) + { + DPRINT1("Vacb read beyond the file size!\n"); return FALSE; + } BaseAddress = (PVOID)((ULONG_PTR)Vacb->BaseAddress + Offset); From d2fa434cb7849442c9813ca3daac83882114cecd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Wed, 2 Dec 2020 15:17:28 +0100 Subject: [PATCH 36/99] [NTOSKRNL] Address issues raised in PR 3361 review --- ntoskrnl/cache/section/data.c | 4 ---- ntoskrnl/mm/ARM3/section.c | 8 +++++++ ntoskrnl/mm/section.c | 39 ++++++++++++++--------------------- 3 files changed, 24 insertions(+), 27 deletions(-) diff --git a/ntoskrnl/cache/section/data.c b/ntoskrnl/cache/section/data.c index 1ec6a3326d9..fa83080eddb 100644 --- a/ntoskrnl/cache/section/data.c +++ b/ntoskrnl/cache/section/data.c @@ -274,8 +274,6 @@ MmFinalizeSegment(PMM_SECTION_SEGMENT Segment) DPRINT("Finalize segment %p\n", Segment); - __debugbreak(); - MmLockSectionSegment(Segment); RemoveEntryList(&Segment->ListOfSegments); if (Segment->Flags & MM_DATAFILE_SEGMENT) { @@ -622,8 +620,6 @@ MiFreeSegmentPage(PMM_SECTION_SEGMENT Segment, ULONG_PTR Entry; PFILE_OBJECT FileObject = Segment->FileObject; - __debugbreak(); - Entry = MmGetPageEntrySectionSegment(Segment, FileOffset); DPRINTC("MiFreeSegmentPage(%p:%I64x -> Entry %Ix\n", Segment, diff --git a/ntoskrnl/mm/ARM3/section.c b/ntoskrnl/mm/ARM3/section.c index 9ea36637a22..dde3b2d8cc4 100644 --- a/ntoskrnl/mm/ARM3/section.c +++ b/ntoskrnl/mm/ARM3/section.c @@ -1094,6 +1094,14 @@ MiMapViewInSystemSpace(IN PVOID Section, *ViewSize = SectionSize - SectionOffset->QuadPart; } + /* Check overflow */ + if ((SectionOffset->QuadPart + *ViewSize) < SectionOffset->QuadPart) + { + DPRINT1("Integer overflow between size & offset!\n"); + MiDereferenceControlArea(ControlArea); + return STATUS_INVALID_VIEW_SIZE; + } + /* Check if the caller wanted a larger section than the view */ if (SectionOffset->QuadPart + *ViewSize > SectionSize) { diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index 6dcf764d7a1..e39f82d0f22 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -2050,7 +2050,7 @@ MmCreatePhysicalMemorySection(VOID) /* * Create the section mapping physical memory */ - SectionSize.QuadPart = ~((ULONG_PTR)0); + SectionSize.QuadPart = MmNumberOfPhysicalPages * PAGE_SIZE; InitializeObjectAttributes(&Obj, &Name, OBJ_PERMANENT | OBJ_KERNEL_EXCLUSIVE, @@ -2192,7 +2192,6 @@ MmCreateDataFileSection(PSECTION *SectionObject, (PVOID*)&Section); if (!NT_SUCCESS(Status)) { - ObDereferenceObject(FileObject); return(Status); } /* @@ -2240,7 +2239,6 @@ MmCreateDataFileSection(PSECTION *SectionObject, if (MaximumSize.QuadPart == 0) { ObDereferenceObject(Section); - ObDereferenceObject(FileObject); return STATUS_MAPPED_FILE_SIZE_ZERO; } } @@ -2254,7 +2252,6 @@ MmCreateDataFileSection(PSECTION *SectionObject, if (!NT_SUCCESS(Status)) { ObDereferenceObject(Section); - ObDereferenceObject(FileObject); return(STATUS_SECTION_NOT_EXTENDED); } } @@ -2263,7 +2260,6 @@ MmCreateDataFileSection(PSECTION *SectionObject, if (FileObject->SectionObjectPointer == NULL) { ObDereferenceObject(Section); - ObDereferenceObject(FileObject); return STATUS_INVALID_FILE_FOR_SECTION; } @@ -2274,7 +2270,6 @@ MmCreateDataFileSection(PSECTION *SectionObject, if (Status != STATUS_SUCCESS) { ObDereferenceObject(Section); - ObDereferenceObject(FileObject); return(Status); } @@ -2305,7 +2300,6 @@ MmCreateDataFileSection(PSECTION *SectionObject, //KeSetEvent((PVOID)&FileObject->Lock, IO_NO_INCREMENT, FALSE); MiReleasePfnLock(OldIrql); ObDereferenceObject(Section); - ObDereferenceObject(FileObject); return(STATUS_NO_MEMORY); } @@ -2329,6 +2323,7 @@ MmCreateDataFileSection(PSECTION *SectionObject, ExInitializeFastMutex(&Segment->Lock); Segment->FileObject = FileObject; + ObReferenceObject(FileObject); Segment->Image.FileOffset = 0; Segment->Protection = SectionPageProtection; @@ -2370,9 +2365,6 @@ MmCreateDataFileSection(PSECTION *SectionObject, } MmUnlockSectionSegment(Segment); - - /* The segment already has a reference to a file object. Don't bother keeping one.*/ - ObDereferenceObject(FileObject); } Section->SizeOfSection = MaximumSize; @@ -3053,7 +3045,6 @@ MmCreateImageSection(PSECTION *SectionObject, (PVOID*)(PVOID)&Section); if (!NT_SUCCESS(Status)) { - ObDereferenceObject(FileObject); return(Status); } @@ -3093,17 +3084,14 @@ MmCreateImageSection(PSECTION *SectionObject, NTSTATUS StatusExeFmt; PMM_SECTION_SEGMENT DataSectionObject; - ImageSectionObject = ExAllocatePoolWithTag(NonPagedPool, sizeof(MM_IMAGE_SECTION_OBJECT), TAG_MM_SECTION_SEGMENT); + ImageSectionObject = ExAllocatePoolZero(NonPagedPool, sizeof(MM_IMAGE_SECTION_OBJECT), TAG_MM_SECTION_SEGMENT); if (ImageSectionObject == NULL) { MiReleasePfnLock(OldIrql); - ObDereferenceObject(FileObject); ObDereferenceObject(Section); return(STATUS_NO_MEMORY); } - RtlZeroMemory(ImageSectionObject, sizeof(MM_IMAGE_SECTION_OBJECT)); - ImageSectionObject->SegFlags = MM_SEGMENT_INCREATE; ImageSectionObject->RefCount = 1; FileObject->SectionObjectPointer->ImageSectionObject = ImageSectionObject; @@ -3172,7 +3160,6 @@ MmCreateImageSection(PSECTION *SectionObject, ExFreePoolWithTag(ImageSectionObject, TAG_MM_SECTION_SEGMENT); ObDereferenceObject(Section); - ObDereferenceObject(FileObject); return(Status); } @@ -3194,12 +3181,15 @@ MmCreateImageSection(PSECTION *SectionObject, ExFreePool(ImageSectionObject->Segments); ExFreePool(ImageSectionObject); ObDereferenceObject(Section); - ObDereferenceObject(FileObject); return(Status); } OldIrql = MiAcquirePfnLock(); ImageSectionObject->SegFlags &= ~MM_SEGMENT_INCREATE; + + /* Take a ref on the file on behalf of the newly created structure */ + ObReferenceObject(FileObject); + MiReleasePfnLock(OldIrql); Status = StatusExeFmt; @@ -3213,9 +3203,6 @@ MmCreateImageSection(PSECTION *SectionObject, Section->Segment = (PSEGMENT)ImageSectionObject; - /* We let the Image Section Object hold the reference */ - ObDereferenceObject(FileObject); - Status = STATUS_SUCCESS; } //KeSetEvent((PVOID)&FileObject->Lock, IO_NO_INCREMENT, FALSE); @@ -3693,8 +3680,10 @@ NtQuerySection( _SEH2_TRY { *((SECTION_BASIC_INFORMATION*)SectionInformation) = Sbi; - if (ResultLength) + if (ResultLength != NULL) + { *ResultLength = sizeof(Sbi); + } } _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER) { @@ -3718,7 +3707,9 @@ NtQuerySection( PSECTION_IMAGE_INFORMATION Sii = (PSECTION_IMAGE_INFORMATION)SectionInformation; *Sii = ImageSectionObject->ImageInformation; if (ResultLength != NULL) + { *ResultLength = sizeof(*Sii); + } } _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER) { @@ -4374,6 +4365,7 @@ MmCreateSection (OUT PVOID * Section, SectionPageProtection, AllocationAttributes, FileObject); + ObDereferenceObject(FileObject); } #ifndef NEWCC else if (FileObject != NULL) @@ -4386,6 +4378,7 @@ MmCreateSection (OUT PVOID * Section, AllocationAttributes, FileObject, FileHandle != NULL); + ObDereferenceObject(FileObject); } #else else if (FileHandle != NULL || FileObject != NULL) @@ -4393,8 +4386,8 @@ MmCreateSection (OUT PVOID * Section, Status = MmCreateCacheSection(SectionObject, DesiredAccess, ObjectAttributes, - SizeOfSection, - InitialPageProtection, + MaximumSize, + SectionPageProtection, AllocationAttributes, FileObject); } From 76ac898bc55a6c4c40a4f1b503bcb0aac0d51dd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Thu, 3 Dec 2020 17:13:38 +0100 Subject: [PATCH 37/99] [SETUPLIB] Lock the volume before writing MBR boot code --- base/setup/lib/bootsup.c | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/base/setup/lib/bootsup.c b/base/setup/lib/bootsup.c index 37f7bdb43ab..b1e11d108d4 100644 --- a/base/setup/lib/bootsup.c +++ b/base/setup/lib/bootsup.c @@ -555,7 +555,7 @@ InstallBootCodeToDisk( IN PCWSTR RootPath, IN PFS_INSTALL_BOOTCODE InstallBootCode) { - NTSTATUS Status; + NTSTATUS Status, LockStatus; UNICODE_STRING Name; OBJECT_ATTRIBUTES ObjectAttributes; IO_STATUS_BLOCK IoStatusBlock; @@ -584,9 +584,32 @@ InstallBootCodeToDisk( if (!NT_SUCCESS(Status)) return Status; + /* Lock the volume */ + LockStatus = NtFsControlFile(PartitionHandle, NULL, NULL, NULL, &IoStatusBlock, FSCTL_LOCK_VOLUME, NULL, 0, NULL, 0); + if (!NT_SUCCESS(LockStatus)) + { + DPRINT1("Unable to lock the volume before installing boot code. Status 0x%08x. Expect problems.\n", LockStatus); + } + /* Install the bootcode (MBR, VBR) */ Status = InstallBootCode(SrcPath, PartitionHandle, PartitionHandle); + /* dismount & Unlock the volume */ + if (NT_SUCCESS(LockStatus)) + { + LockStatus = NtFsControlFile(PartitionHandle, NULL, NULL, NULL, &IoStatusBlock, FSCTL_DISMOUNT_VOLUME, NULL, 0, NULL, 0); + if (!NT_SUCCESS(LockStatus)) + { + DPRINT1("Unable to unlock the volume after installing boot code. Status 0x%08x. Expect problems.\n", LockStatus); + } + + LockStatus = NtFsControlFile(PartitionHandle, NULL, NULL, NULL, &IoStatusBlock, FSCTL_UNLOCK_VOLUME, NULL, 0, NULL, 0); + if (!NT_SUCCESS(LockStatus)) + { + DPRINT1("Unable to unlock the volume after installing boot code. Status 0x%08x. Expect problems.\n", LockStatus); + } + } + /* Close the partition */ NtClose(PartitionHandle); @@ -1076,6 +1099,7 @@ InstallFatBootcodeToPartition( DPRINT1("Install FAT32 bootcode: %S ==> %S\n", SrcPath, SystemRootPath->Buffer); Status = InstallBootCodeToDisk(SrcPath, SystemRootPath->Buffer, InstallFat32BootCode); + DPRINT1("Status: 0x%08X\n", Status); if (!NT_SUCCESS(Status)) { DPRINT1("InstallBootCodeToDisk(FAT32) failed (Status %lx)\n", Status); From 7d70af61e8fff18b24d9baa3bb63c37c96aa21c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Thu, 3 Dec 2020 17:18:29 +0100 Subject: [PATCH 38/99] [NTOS:MM] Acquire file lock when creating section --- ntoskrnl/mm/section.c | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index e39f82d0f22..08e68a20c33 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -4290,6 +4290,7 @@ MmCreateSection (OUT PVOID * Section, NTSTATUS Status; ULONG Protection; PSECTION *SectionObject = (PSECTION *)Section; + BOOLEAN FileLock = FALSE; /* Check if an ARM3 section is being created instead */ if (!(AllocationAttributes & (SEC_IMAGE | SEC_PHYSICALMEMORY))) @@ -4343,6 +4344,24 @@ MmCreateSection (OUT PVOID * Section, DPRINT1("Failed to get a handle to the FO: %lx\n", Status); return Status; } + + /* Lock the file */ + Status = FsRtlAcquireToCreateMappedSection(FileObject, SectionPageProtection); + if (!NT_SUCCESS(Status)) + { + ObDereferenceObject(FileObject); + return Status; + } + + FileLock = TRUE; + + /* Deny access if there are writes on the file */ + if ((AllocationAttributes & SEC_IMAGE) && (Status == STATUS_FILE_LOCKED_WITH_WRITERS)) + { + DPRINT1("Cannot create image maps with writers open on the file!\n"); + Status = STATUS_ACCESS_DENIED; + goto Quit; + } } else { @@ -4365,7 +4384,6 @@ MmCreateSection (OUT PVOID * Section, SectionPageProtection, AllocationAttributes, FileObject); - ObDereferenceObject(FileObject); } #ifndef NEWCC else if (FileObject != NULL) @@ -4378,7 +4396,6 @@ MmCreateSection (OUT PVOID * Section, AllocationAttributes, FileObject, FileHandle != NULL); - ObDereferenceObject(FileObject); } #else else if (FileHandle != NULL || FileObject != NULL) @@ -4396,10 +4413,14 @@ MmCreateSection (OUT PVOID * Section, { /* All cases should be handled above */ Status = STATUS_INVALID_PARAMETER; - if (FileObject) - ObDereferenceObject(FileObject); } +Quit: + if (FileLock) + FsRtlReleaseFile(FileObject); + if (FileObject) + ObDereferenceObject(FileObject); + return Status; } From f114209782a0f44fa6b80cc3aa1e7d583d5dacea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Fri, 4 Dec 2020 09:10:13 +0100 Subject: [PATCH 39/99] [NTOS/MM] Do not zero out the tail of the segment if the mapping is not an image --- ntoskrnl/mm/section.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index 08e68a20c33..6b6a6cb94be 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -1205,7 +1205,7 @@ MiReadPage(PMEMORY_AREA MemoryArea, Status = STATUS_SUCCESS; } - if ((SegOffset + PAGE_SIZE) > MemoryArea->SectionData.Segment->RawLength.QuadPart) + if ((MemoryArea->VadNode.u.VadFlags.VadType == VadImageMap) && ((SegOffset + PAGE_SIZE) > MemoryArea->SectionData.Segment->RawLength.QuadPart)) { KIRQL OldIrql; PUCHAR PageMap; From e39fc52d25a577b3c5abd9a9c9b5d20508c42d88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Fri, 4 Dec 2020 09:10:59 +0100 Subject: [PATCH 40/99] [NTOS/MM] Set FSRTL_MOD_WRITE_TOP_LEVEL_IRP top-level IRP when paging-out --- ntoskrnl/mm/section.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index 6b6a6cb94be..866b90810e0 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -4685,9 +4685,16 @@ MmCheckDirtySegment( MmUnlockSectionSegment(Segment); + /* Tell the FS driver who we are */ + if (PageOut) + IoSetTopLevelIrp((PIRP)FSRTL_MOD_WRITE_TOP_LEVEL_IRP); + /* Go ahead and write the page */ Status = MiWritePage(Segment, Offset->QuadPart, Page); + if (PageOut) + IoSetTopLevelIrp(NULL); + MmLockSectionSegment(Segment); /* Get the entry again */ From cbfc04157d9553fe4e7f2171db64561be2eaf243 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Fri, 4 Dec 2020 09:12:18 +0100 Subject: [PATCH 41/99] [NTOS:CC] File size check fix --- ntoskrnl/cc/copy.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ntoskrnl/cc/copy.c b/ntoskrnl/cc/copy.c index f92040aa0b2..d228a981ea2 100644 --- a/ntoskrnl/cc/copy.c +++ b/ntoskrnl/cc/copy.c @@ -809,7 +809,7 @@ CcZeroData ( return TRUE; } - ASSERT(EndOffset->QuadPart <= SharedCacheMap->FileSize.QuadPart); + ASSERT(EndOffset->QuadPart <= SharedCacheMap->SectionSize.QuadPart); while(WriteOffset.QuadPart < EndOffset->QuadPart) { From 2d3454a809238331dae3fd6f25185eb5b92cf73f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Fri, 4 Dec 2020 09:13:01 +0100 Subject: [PATCH 42/99] [NTOS:IO] VFAT sucks less now --- ntoskrnl/io/iomgr/iofunc.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/ntoskrnl/io/iomgr/iofunc.c b/ntoskrnl/io/iomgr/iofunc.c index 80814a70ba3..7d0a08c5395 100644 --- a/ntoskrnl/io/iomgr/iofunc.c +++ b/ntoskrnl/io/iomgr/iofunc.c @@ -3014,10 +3014,8 @@ NtReadFile(IN HANDLE FileHandle, /* Now set the deferred read flags */ Irp->Flags |= (IRP_READ_OPERATION | IRP_DEFER_IO_COMPLETION); -#if 0 - /* FIXME: VFAT SUCKS */ + if (FileObject->Flags & FO_NO_INTERMEDIATE_BUFFERING) Irp->Flags |= IRP_NOCACHE; -#endif /* Perform the call */ return IopPerformSynchronousRequest(DeviceObject, @@ -4082,10 +4080,8 @@ NtWriteFile(IN HANDLE FileHandle, /* Now set the deferred read flags */ Irp->Flags |= (IRP_WRITE_OPERATION | IRP_DEFER_IO_COMPLETION); -#if 0 - /* FIXME: VFAT SUCKS */ + if (FileObject->Flags & FO_NO_INTERMEDIATE_BUFFERING) Irp->Flags |= IRP_NOCACHE; -#endif /* Perform the call */ return IopPerformSynchronousRequest(DeviceObject, From d9f7459bb1dcb0893adbec74939962e7a4d95d9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Fri, 4 Dec 2020 09:17:21 +0100 Subject: [PATCH 43/99] [NTOS:MM] Fix file object bad referencing --- ntoskrnl/mm/section.c | 1 - 1 file changed, 1 deletion(-) diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index 866b90810e0..b38dc0bfc4a 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -3128,7 +3128,6 @@ MmCreateImageSection(PSECTION *SectionObject, InterlockedExchangePointer(&FileObject->SectionObjectPointer->ImageSectionObject, NULL); ExFreePoolWithTag(ImageSectionObject, TAG_MM_SECTION_SEGMENT); MmDereferenceSegment(DataSectionObject); - ObDereferenceObject(FileObject); ObDereferenceObject(Section); return STATUS_ACCESS_DENIED; } From f8aa14ce4e651ca57f11904b482972fb0f749295 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Fri, 4 Dec 2020 10:18:32 +0100 Subject: [PATCH 44/99] [NTOS:CC] Acquire file for flush when flushing if necessary --- ntoskrnl/cc/view.c | 16 ++++++++++++++++ ntoskrnl/include/internal/fsrtl.h | 8 ++++++++ 2 files changed, 24 insertions(+) diff --git a/ntoskrnl/cc/view.c b/ntoskrnl/cc/view.c index de4a75ea8f7..ee0ec497d40 100644 --- a/ntoskrnl/cc/view.c +++ b/ntoskrnl/cc/view.c @@ -172,11 +172,27 @@ CcRosFlushVacb ( SIZE_T FlushSize = min(VACB_MAPPING_GRANULARITY, Vacb->SharedCacheMap->SectionSize.QuadPart - Vacb->FileOffset.QuadPart); NTSTATUS Status; + BOOLEAN HaveLock = FALSE; CcRosUnmarkDirtyVacb(Vacb, TRUE); + /* Lock for flush, if we are not already the top-level */ + if (IoGetTopLevelIrp() != (PIRP)FSRTL_CACHE_TOP_LEVEL_IRP) + { + Status = FsRtlAcquireFileForCcFlushEx(Vacb->SharedCacheMap->FileObject); + if (!NT_SUCCESS(Status)) + goto quit; + HaveLock = TRUE; + } + Status = MmFlushVirtualMemory(NULL, &Vacb->BaseAddress, &FlushSize, &Iosb); + if (HaveLock) + { + FsRtlReleaseFileForCcFlush(Vacb->SharedCacheMap->FileObject); + } + +quit: if (!NT_SUCCESS(Status)) CcRosMarkDirtyVacb(Vacb); diff --git a/ntoskrnl/include/internal/fsrtl.h b/ntoskrnl/include/internal/fsrtl.h index 79cc3b6cfbc..228bb7a1cf8 100644 --- a/ntoskrnl/include/internal/fsrtl.h +++ b/ntoskrnl/include/internal/fsrtl.h @@ -148,3 +148,11 @@ NTSTATUS NTAPI FsRtlAcquireToCreateMappedSection(_In_ PFILE_OBJECT FileObject, _In_ ULONG SectionPageProtection); + +VOID +NTAPI +FsRtlReleaseFileForCcFlush(IN PFILE_OBJECT FileObject); + +NTSTATUS +NTAPI +FsRtlAcquireFileForCcFlushEx(IN PFILE_OBJECT FileObject); From cd23d6e19aef6daf321f0d6a5e021e58a09982ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Fri, 4 Dec 2020 15:45:04 +0100 Subject: [PATCH 45/99] [NTOS:CC] Honor FO_WRITE_THROUGH flag in CcCopyWrite and fix it in CcZeroData --- ntoskrnl/cc/copy.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/ntoskrnl/cc/copy.c b/ntoskrnl/cc/copy.c index d228a981ea2..357e98162e6 100644 --- a/ntoskrnl/cc/copy.c +++ b/ntoskrnl/cc/copy.c @@ -556,16 +556,13 @@ CcCopyWrite ( if (!SharedCacheMap) return FALSE; - /* FIXME: Honor FileObject FO_WRITE_THROUGH flag */ ASSERT((FileOffset->QuadPart + Length) <= SharedCacheMap->SectionSize.QuadPart); - ASSERT((FileObject->Flags & FO_WRITE_THROUGH) == 0); - CurrentOffset = FileOffset->QuadPart; while(CurrentOffset < WriteEnd) { ULONG VacbOffset = CurrentOffset % VACB_MAPPING_GRANULARITY; - ULONG VacbLength = min(Length, VACB_MAPPING_GRANULARITY - VacbOffset); + ULONG VacbLength = min(WriteEnd - CurrentOffset, VACB_MAPPING_GRANULARITY - VacbOffset); Status = CcRosGetVacb(SharedCacheMap, CurrentOffset, &Vacb); if (!NT_SUCCESS(Status)) @@ -585,7 +582,6 @@ CcCopyWrite ( Buffer = (PVOID)((ULONG_PTR)Buffer + VacbLength); CurrentOffset += VacbLength; - Length -= VacbLength; /* Tell Mm */ Status = MmMakePagesDirty(NULL, Add2Ptr(Vacb->BaseAddress, VacbOffset), VacbLength); @@ -600,6 +596,10 @@ CcCopyWrite ( _SEH2_END; } + /* Flush if needed */ + if (FileObject->Flags & FO_WRITE_THROUGH) + CcFlushCache(FileObject->SectionObjectPointer, FileOffset, Length, NULL); + return TRUE; } @@ -757,7 +757,7 @@ CcZeroData ( Length = EndOffset->QuadPart - StartOffset->QuadPart; WriteOffset.QuadPart = StartOffset->QuadPart; - if (!SharedCacheMap || (FileObject->Flags & FO_WRITE_THROUGH)) + if (!SharedCacheMap) { /* Make this a non-cached write */ IO_STATUS_BLOCK Iosb; @@ -848,5 +848,9 @@ CcZeroData ( _SEH2_END; } + /* Flush if needed */ + if (FileObject->Flags & FO_WRITE_THROUGH) + CcFlushCache(FileObject->SectionObjectPointer, StartOffset, EndOffset->QuadPart - StartOffset->QuadPart, NULL); + return TRUE; } From fb07802956b5d10363b43d0d0e2ea4427764779f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Mon, 7 Dec 2020 09:55:12 +0100 Subject: [PATCH 46/99] [NTOS:MM] Make the situation about dirty pages in image map a bit clearer --- ntoskrnl/mm/section.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index b38dc0bfc4a..d71bc490934 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -1082,7 +1082,7 @@ MmUnsharePageEntrySectionSegment(PMEMORY_AREA MemoryArea, return FALSE; } - if (IS_DIRTY_SSE(Entry) && !(Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) + if (Dirty && (MemoryArea->VadNode.u.VadFlags.VadType != VadImageMap)) { ASSERT(!Segment->WriteCopy); ASSERT(MmGetSavedSwapEntryPage(Page) == 0); @@ -1092,6 +1092,9 @@ MmUnsharePageEntrySectionSegment(PMEMORY_AREA MemoryArea, return FALSE; } + /* Only valid case for shared dirty pages is shared image section */ + ASSERT(!Dirty || (Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)); + SwapEntry = MmGetSavedSwapEntryPage(Page); if (Dirty && !SwapEntry) { From aab24ed1b1a3269de23dbaa334a67741434bc4c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Mon, 7 Dec 2020 11:42:47 +0100 Subject: [PATCH 47/99] [NTOS:MM] Have the legacy Mm be less greedy about the number of pages it wants --- ntoskrnl/mm/balance.c | 33 ++++++++------------------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/ntoskrnl/mm/balance.c b/ntoskrnl/mm/balance.c index 29c0dfc6f47..34dddf5dbfa 100644 --- a/ntoskrnl/mm/balance.c +++ b/ntoskrnl/mm/balance.c @@ -28,7 +28,6 @@ MM_ALLOCATION_REQUEST, *PMM_ALLOCATION_REQUEST; MM_MEMORY_CONSUMER MiMemoryConsumers[MC_MAXIMUM]; static ULONG MiMinimumAvailablePages; -static ULONG MiNrTotalPages; static LIST_ENTRY AllocationListHead; static KSPIN_LOCK AllocationListLock; static ULONG MiMinimumPagesPerRun; @@ -49,12 +48,10 @@ MmInitializeBalancer(ULONG NrAvailablePages, ULONG NrSystemPages) InitializeListHead(&AllocationListHead); KeInitializeSpinLock(&AllocationListLock); - MiNrTotalPages = NrAvailablePages; - /* Set up targets. */ MiMinimumAvailablePages = 256; MiMinimumPagesPerRun = 256; - MiMemoryConsumers[MC_USER].PagesTarget = NrAvailablePages - MiMinimumAvailablePages; + MiMemoryConsumers[MC_USER].PagesTarget = NrAvailablePages / 2; } CODE_SEG("INIT") @@ -120,15 +117,12 @@ MiTrimMemoryConsumer(ULONG Consumer, ULONG InitialTarget) Target = (ULONG)max(Target, MiMinimumAvailablePages - MmAvailablePages); } + /* Don't be too greedy if we're not in a hurry */ + if (MmAvailablePages > MiMinimumAvailablePages) + Target = min(Target, 256); + if (Target) { - if (!InitialTarget) - { - /* If there was no initial target, - * swap at least MiMinimumPagesPerRun */ - Target = max(Target, MiMinimumPagesPerRun); - } - /* Now swap the pages out */ Status = MiMemoryConsumers[Consumer].Trim(Target, 0, &NrFreedPages); @@ -138,21 +132,10 @@ MiTrimMemoryConsumer(ULONG Consumer, ULONG InitialTarget) { KeBugCheck(MEMORY_MANAGEMENT); } - - /* Update the target */ - if (NrFreedPages < Target) - Target -= NrFreedPages; - else - Target = 0; - - /* Return the remaining pages needed to meet the target */ - return Target; - } - else - { - /* Initial target is zero and we don't have anything else to add */ - return 0; } + + /* Return the page count needed to be freed to meet the initial target */ + return (InitialTarget > NrFreedPages) ? (InitialTarget - NrFreedPages) : 0; } NTSTATUS From 4441b1cd44bbf0f16b49383759bf50a35828da6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Tue, 8 Dec 2020 10:28:52 +0100 Subject: [PATCH 48/99] [NTOS:CC] Fix some tests for CcCopyRead and CcCopyWrite Most importantly: raise the right status when provided an invalid buffer. --- ntoskrnl/cc/copy.c | 51 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 45 insertions(+), 6 deletions(-) diff --git a/ntoskrnl/cc/copy.c b/ntoskrnl/cc/copy.c index 357e98162e6..315e9f1f1b0 100644 --- a/ntoskrnl/cc/copy.c +++ b/ntoskrnl/cc/copy.c @@ -447,6 +447,26 @@ CcCanIWrite ( return TRUE; } +static +int +CcpCheckInvalidUserBuffer(PEXCEPTION_POINTERS Except, PVOID Buffer, ULONG Length) +{ + ULONG_PTR ExceptionAddress; + ULONG_PTR BeginAddress = (ULONG_PTR)Buffer; + ULONG_PTR EndAddress = (ULONG_PTR)Buffer + Length; + + if (Except->ExceptionRecord->ExceptionCode != STATUS_ACCESS_VIOLATION) + return EXCEPTION_CONTINUE_SEARCH; + if (Except->ExceptionRecord->NumberParameters < 2) + return EXCEPTION_CONTINUE_SEARCH; + + ExceptionAddress = Except->ExceptionRecord->ExceptionInformation[1]; + if ((ExceptionAddress >= BeginAddress) && (ExceptionAddress < EndAddress)) + return EXCEPTION_EXECUTE_HANDLER; + + return EXCEPTION_CONTINUE_SEARCH; +} + /* * @implemented */ @@ -465,6 +485,7 @@ CcCopyRead ( NTSTATUS Status; LONGLONG CurrentOffset; LONGLONG ReadEnd = FileOffset->QuadPart + Length; + ULONG ReadLength = 0; CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%I64d Length=%lu Wait=%d\n", FileObject, FileOffset->QuadPart, Length, Wait); @@ -480,9 +501,6 @@ CcCopyRead ( /* Documented to ASSERT, but KMTests test this case... */ // ASSERT((FileOffset->QuadPart + Length) <= SharedCacheMap->FileSize.QuadPart); - IoStatus->Status = STATUS_SUCCESS; - IoStatus->Information = 0; - CurrentOffset = FileOffset->QuadPart; while(CurrentOffset < ReadEnd) { @@ -506,13 +524,23 @@ CcCopyRead ( if (CurrentOffset + VacbLength > SharedCacheMap->SectionSize.QuadPart) CopyLength = SharedCacheMap->SectionSize.QuadPart - CurrentOffset; if (CopyLength != 0) - RtlCopyMemory(Buffer, (PUCHAR)Vacb->BaseAddress + VacbOffset, CopyLength); + { + _SEH2_TRY + { + RtlCopyMemory(Buffer, (PUCHAR)Vacb->BaseAddress + VacbOffset, CopyLength); + } + _SEH2_EXCEPT(CcpCheckInvalidUserBuffer(_SEH2_GetExceptionInformation(), Buffer, VacbLength)) + { + ExRaiseStatus(STATUS_INVALID_USER_BUFFER); + } + _SEH2_END; + } /* Zero-out the buffer tail if needed */ if (CopyLength < VacbLength) RtlZeroMemory((PUCHAR)Buffer + CopyLength, VacbLength - CopyLength); - IoStatus->Information += VacbLength; + ReadLength += VacbLength; Buffer = (PVOID)((ULONG_PTR)Buffer + VacbLength); CurrentOffset += VacbLength; @@ -525,6 +553,9 @@ CcCopyRead ( _SEH2_END; } + IoStatus->Status = STATUS_SUCCESS; + IoStatus->Information = ReadLength; + return TRUE; } @@ -578,7 +609,15 @@ CcCopyWrite ( return FALSE; } - RtlCopyMemory((PVOID)((ULONG_PTR)Vacb->BaseAddress + VacbOffset), Buffer, VacbLength); + _SEH2_TRY + { + RtlCopyMemory((PVOID)((ULONG_PTR)Vacb->BaseAddress + VacbOffset), Buffer, VacbLength); + } + _SEH2_EXCEPT(CcpCheckInvalidUserBuffer(_SEH2_GetExceptionInformation(), Buffer, VacbLength)) + { + ExRaiseStatus(STATUS_INVALID_USER_BUFFER); + } + _SEH2_END; Buffer = (PVOID)((ULONG_PTR)Buffer + VacbLength); CurrentOffset += VacbLength; From a52bc6d1799385726097ecf701f833b7a327a08e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Fri, 11 Dec 2020 14:31:08 +0100 Subject: [PATCH 49/99] [NTOS:CC] Restore read-ahead behaviour in CcCopyRead --- ntoskrnl/cc/copy.c | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/ntoskrnl/cc/copy.c b/ntoskrnl/cc/copy.c index 315e9f1f1b0..3dd172fa9db 100644 --- a/ntoskrnl/cc/copy.c +++ b/ntoskrnl/cc/copy.c @@ -556,6 +556,27 @@ CcCopyRead ( IoStatus->Status = STATUS_SUCCESS; IoStatus->Information = ReadLength; + /* If that was a successful sync read operation, let's handle read ahead */ + if (Length == 0 && Wait) + { + PPRIVATE_CACHE_MAP PrivateCacheMap = FileObject->PrivateCacheMap; + + /* If file isn't random access and next read may get us cross VACB boundary, + * schedule next read + */ + if (!BooleanFlagOn(FileObject->Flags, FO_RANDOM_ACCESS) && + (CurrentOffset - 1) / VACB_MAPPING_GRANULARITY != (CurrentOffset + ReadLength - 1) / VACB_MAPPING_GRANULARITY) + { + CcScheduleReadAhead(FileObject, FileOffset, ReadLength); + } + + /* And update read history in private cache map */ + PrivateCacheMap->FileOffset1.QuadPart = PrivateCacheMap->FileOffset2.QuadPart; + PrivateCacheMap->BeyondLastByte1.QuadPart = PrivateCacheMap->BeyondLastByte2.QuadPart; + PrivateCacheMap->FileOffset2.QuadPart = FileOffset->QuadPart; + PrivateCacheMap->BeyondLastByte2.QuadPart = FileOffset->QuadPart + ReadLength; + } + return TRUE; } From 30f71c7fc0f6404cb298841a9d2123eaca1877be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Fri, 11 Dec 2020 15:29:35 +0100 Subject: [PATCH 50/99] [NTOS] Zero data unconditionally after segment end, unless section is created with SEC_RESERVE Use a SEC_RESERVE section in Cc --- ntoskrnl/cc/view.c | 2 +- ntoskrnl/mm/section.c | 18 ++++++++++++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/ntoskrnl/cc/view.c b/ntoskrnl/cc/view.c index ee0ec497d40..ae6464f2b58 100644 --- a/ntoskrnl/cc/view.c +++ b/ntoskrnl/cc/view.c @@ -1286,7 +1286,7 @@ CcRosInitializeFileCache ( NULL, &SharedCacheMap->SectionSize, PAGE_READWRITE, - 0, + SEC_RESERVE, NULL, FileObject); diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index d71bc490934..8e9669d9ab9 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -1208,7 +1208,8 @@ MiReadPage(PMEMORY_AREA MemoryArea, Status = STATUS_SUCCESS; } - if ((MemoryArea->VadNode.u.VadFlags.VadType == VadImageMap) && ((SegOffset + PAGE_SIZE) > MemoryArea->SectionData.Segment->RawLength.QuadPart)) + if (!MemoryArea->SectionData.Section->u.Flags.Reserve + && ((SegOffset + PAGE_SIZE) > MemoryArea->SectionData.Segment->RawLength.QuadPart)) { KIRQL OldIrql; PUCHAR PageMap; @@ -2209,6 +2210,8 @@ MmCreateDataFileSection(PSECTION *SectionObject, if (AllocationAttributes & SEC_NO_CHANGE) Section->u.Flags.NoChange = 1; + if (AllocationAttributes & SEC_RESERVE) + Section->u.Flags.Reserve = 1; if (!GotFileHandle) { @@ -4832,13 +4835,16 @@ MmExtendSection( PMM_SECTION_SEGMENT Segment = (PMM_SECTION_SEGMENT)Section->Segment; Section->SizeOfSection = *NewSize; - MmLockSectionSegment(Segment); - if (Segment->RawLength.QuadPart < NewSize->QuadPart) + if (!Section->u.Flags.Reserve) { - Segment->RawLength = *NewSize; - Segment->Length.QuadPart = (NewSize->QuadPart + PAGE_SIZE - 1) & ~((LONGLONG)PAGE_SIZE); + MmLockSectionSegment(Segment); + if (Segment->RawLength.QuadPart < NewSize->QuadPart) + { + Segment->RawLength = *NewSize; + Segment->Length.QuadPart = (NewSize->QuadPart + PAGE_SIZE - 1) & ~((LONGLONG)PAGE_SIZE); + } + MmUnlockSectionSegment(Segment); } - MmUnlockSectionSegment(Segment); } return STATUS_SUCCESS; From 3141668724e37d0dac6cee831db1fe4eda6bf84f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Fri, 11 Dec 2020 18:53:32 +0100 Subject: [PATCH 51/99] [NTOS:MM] Various fixes. Fix MmFlushImageSection behaviour Remove an invalid ASSERT --- ntoskrnl/mm/section.c | 48 +++++++++++++------------------------------ 1 file changed, 14 insertions(+), 34 deletions(-) diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index 8e9669d9ab9..8acc368a884 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -4093,46 +4093,26 @@ CheckSectionPointer: */ BOOLEAN NTAPI MmFlushImageSection (IN PSECTION_OBJECT_POINTERS SectionObjectPointer, - IN MMFLUSH_TYPE FlushType) + IN MMFLUSH_TYPE FlushType) { - BOOLEAN Result = TRUE; -#ifdef NEWCC - PMM_SECTION_SEGMENT Segment; -#endif - switch(FlushType) { - case MmFlushForDelete: - if (SectionObjectPointer->ImageSectionObject || - SectionObjectPointer->DataSectionObject) + case MmFlushForDelete: + case MmFlushForWrite: { - return FALSE; - } -#ifndef NEWCC - CcRosRemoveIfClosed(SectionObjectPointer); -#endif - return TRUE; - case MmFlushForWrite: - { - DPRINT("MmFlushImageSection(%d)\n", FlushType); -#ifdef NEWCC - Segment = (PMM_SECTION_SEGMENT)SectionObjectPointer->DataSectionObject; -#endif + BOOLEAN Ret = TRUE; + KIRQL OldIrql = MiAcquirePfnLock(); - if (SectionObjectPointer->ImageSectionObject) - { - DPRINT1("SectionObject has ImageSection\n"); - return FALSE; - } + if (SectionObjectPointer->ImageSectionObject) + { + PMM_IMAGE_SECTION_OBJECT ImageSectionObject = SectionObjectPointer->ImageSectionObject; + if (!(ImageSectionObject->SegFlags & MM_SEGMENT_INDELETE)) + Ret = FALSE; + } -#ifdef NEWCC - CcpLock(); - Result = !SectionObjectPointer->SharedCacheMap || (Segment->ReferenceCount == CcpCountCacheSections((PNOCC_CACHE_MAP)SectionObjectPointer->SharedCacheMap)); - CcpUnlock(); - DPRINT("Result %d\n", Result); -#endif - return Result; - } + MiReleasePfnLock(OldIrql); + return Ret; + } } return FALSE; } From 3c2b1bf59b0771a92f7d2aa8c96cad1dd58dc31d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Mon, 14 Dec 2020 15:48:27 +0100 Subject: [PATCH 52/99] [NTOS:MM] Save the dirty bit when sharing a segment page --- ntoskrnl/mm/section.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index 8acc368a884..a3efe4f0b6b 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -1020,6 +1020,7 @@ MmSharePageEntrySectionSegment(PMM_SECTION_SEGMENT Segment, PLARGE_INTEGER Offset) { ULONG_PTR Entry; + BOOLEAN Dirty; Entry = MmGetPageEntrySectionSegment(Segment, Offset); if (Entry == 0) @@ -1036,7 +1037,10 @@ MmSharePageEntrySectionSegment(PMM_SECTION_SEGMENT Segment, { KeBugCheck(MEMORY_MANAGEMENT); } + Dirty = IS_DIRTY_SSE(Entry); Entry = MAKE_SSE(PAGE_FROM_SSE(Entry), SHARE_COUNT_FROM_SSE(Entry) + 1); + if (Dirty) + Entry = DIRTY_SSE(Entry); MmSetPageEntrySectionSegment(Segment, Offset, Entry); } From 1f796267bc885ad0b8154352bf714477ea2a0b58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Mon, 14 Dec 2020 15:55:08 +0100 Subject: [PATCH 53/99] [NTOS:MM] Do not reference the section when mapping it. Referencing the segment is enough. --- ntoskrnl/include/internal/mm.h | 2 +- ntoskrnl/mm/section.c | 51 +++++++++++++++++++++------------- 2 files changed, 33 insertions(+), 20 deletions(-) diff --git a/ntoskrnl/include/internal/mm.h b/ntoskrnl/include/internal/mm.h index 55e9fcfc29c..05f89c50cb9 100644 --- a/ntoskrnl/include/internal/mm.h +++ b/ntoskrnl/include/internal/mm.h @@ -202,6 +202,7 @@ typedef struct _MM_IMAGE_SECTION_OBJECT PMM_SECTION_SEGMENT Segments; } MM_IMAGE_SECTION_OBJECT, *PMM_IMAGE_SECTION_OBJECT; +#define MM_PHYSICALMEMORY_SEGMENT (0x1) #define MM_DATAFILE_SEGMENT (0x2) #define MM_SEGMENT_INDELETE (0x4) #define MM_SEGMENT_INCREATE (0x8) @@ -222,7 +223,6 @@ typedef struct _MEMORY_AREA struct { - PSECTION Section; LARGE_INTEGER ViewOffset; PMM_SECTION_SEGMENT Segment; LIST_ENTRY RegionListHead; diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index a3efe4f0b6b..c20c506c2b8 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -83,6 +83,14 @@ _MmUnlockSectionSegment(PMM_SECTION_SEGMENT Segment, const char *file, int line) } #endif +/* Somewhat grotesque, but eh... */ +PMM_IMAGE_SECTION_OBJECT ImageSectionObjectFromSegment(PMM_SECTION_SEGMENT Segment) +{ + ASSERT((Segment->SegFlags & MM_DATAFILE_SEGMENT) == 0); + + return CONTAINING_RECORD(Segment->ReferenceCount, MM_IMAGE_SECTION_OBJECT, RefCount); +} + NTSTATUS NTAPI MiMapViewInSystemSpace(IN PVOID Section, @@ -1209,15 +1217,18 @@ MiReadPage(PMEMORY_AREA MemoryArea, if (Status == STATUS_END_OF_FILE) { + DPRINT1("Got STATUS_END_OF_FILE at offset %I64d for file %wZ.\n", SegOffset, &FileObject->FileName); Status = STATUS_SUCCESS; } - if (!MemoryArea->SectionData.Section->u.Flags.Reserve + if ((MemoryArea->VadNode.u.VadFlags.VadType == VadImageMap) && ((SegOffset + PAGE_SIZE) > MemoryArea->SectionData.Segment->RawLength.QuadPart)) { KIRQL OldIrql; PUCHAR PageMap; + DPRINT("Zeroing at offset %I64d for file %wZ.\n", SegOffset, &FileObject->FileName); + /* Zero out the end of it */ PageMap = MiMapPageInHyperSpace(PsGetCurrentProcess(), *Page, &OldIrql); RtlZeroMemory(PageMap + MemoryArea->SectionData.Segment->RawLength.QuadPart - SegOffset, @@ -1352,7 +1363,6 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, LARGE_INTEGER Offset; PFN_NUMBER Page; NTSTATUS Status; - PSECTION Section; PMM_SECTION_SEGMENT Segment; ULONG_PTR Entry; ULONG_PTR Entry1; @@ -1391,7 +1401,6 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, + MemoryArea->SectionData.ViewOffset.QuadPart; Segment = MemoryArea->SectionData.Segment; - Section = MemoryArea->SectionData.Section; Region = MmFindRegion((PVOID)MA_GetStartingAddress(MemoryArea), &MemoryArea->SectionData.RegionListHead, Address, NULL); @@ -1537,7 +1546,7 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, /* * Satisfying a page fault on a map of /Device/PhysicalMemory is easy */ - if (Section->u.Flags.PhysicalMemory) + if ((*Segment->Flags) & MM_PHYSICALMEMORY_SEGMENT) { MmUnlockSectionSegment(Segment); /* @@ -1600,6 +1609,7 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, } else { + DPRINT("Getting fresh page for file %wZ at offset %I64d.\n", &Segment->FileObject->FileName, Offset.QuadPart); Status = MiReadPage(MemoryArea, Offset.QuadPart, &Page); if (!NT_SUCCESS(Status)) { @@ -2111,7 +2121,7 @@ MmCreatePhysicalMemorySection(VOID) Segment->Protection = PAGE_EXECUTE_READWRITE; Segment->RawLength = SectionSize; Segment->Length = SectionSize; - Segment->SegFlags = 0; + Segment->SegFlags = MM_PHYSICALMEMORY_SEGMENT; Segment->WriteCopy = FALSE; Segment->Image.VirtualAddress = 0; Segment->Image.Characteristics = 0; @@ -2230,7 +2240,6 @@ MmCreateDataFileSection(PSECTION *SectionObject, if (!NT_SUCCESS(Status)) { ObDereferenceObject(Section); - ObDereferenceObject(FileObject); return Status; } @@ -3283,10 +3292,9 @@ MmMapViewOfSegment(PMMSUPPORT AddressSpace, return(Status); } - ObReferenceObject((PVOID)Section); + InterlockedIncrementUL(Segment->ReferenceCount); MArea->SectionData.Segment = Segment; - MArea->SectionData.Section = Section; MArea->SectionData.ViewOffset.QuadPart = ViewOffset; if (Section->u.Flags.Image) { @@ -3390,7 +3398,6 @@ MmUnmapViewOfSegment(PMMSUPPORT AddressSpace, { NTSTATUS Status; PMEMORY_AREA MemoryArea; - PSECTION Section; PMM_SECTION_SEGMENT Segment; PLIST_ENTRY CurrentEntry; PMM_REGION CurrentRegion; @@ -3403,7 +3410,6 @@ MmUnmapViewOfSegment(PMMSUPPORT AddressSpace, return(STATUS_UNSUCCESSFUL); } - Section = MemoryArea->SectionData.Section; Segment = MemoryArea->SectionData.Segment; #ifdef NEWCC @@ -3429,7 +3435,7 @@ MmUnmapViewOfSegment(PMMSUPPORT AddressSpace, ExFreePoolWithTag(CurrentRegion, TAG_MM_REGION); } - if (Section->u.Flags.PhysicalMemory) + if ((*Segment->Flags) & MM_PHYSICALMEMORY_SEGMENT) { Status = MmFreeMemoryArea(AddressSpace, MemoryArea, @@ -3444,7 +3450,7 @@ MmUnmapViewOfSegment(PMMSUPPORT AddressSpace, AddressSpace); } MmUnlockSectionSegment(Segment); - ObDereferenceObject(Section); + MmDereferenceSegment(Segment); return(Status); } @@ -3457,7 +3463,6 @@ MiRosUnmapViewOfSection(IN PEPROCESS Process, NTSTATUS Status; PMEMORY_AREA MemoryArea; PMMSUPPORT AddressSpace; - PSECTION Section; PVOID ImageBaseAddress = 0; DPRINT("Opening memory area Process %p BaseAddress %p\n", @@ -3484,9 +3489,7 @@ MiRosUnmapViewOfSection(IN PEPROCESS Process, return STATUS_NOT_MAPPED_VIEW; } - Section = MemoryArea->SectionData.Section; - - if ((Section != NULL) && Section->u.Flags.Image) + if (MemoryArea->VadNode.u.VadFlags.VadType == VadImageMap) { ULONG i; ULONG NrSegments; @@ -3495,7 +3498,7 @@ MiRosUnmapViewOfSection(IN PEPROCESS Process, PMM_SECTION_SEGMENT Segment; Segment = MemoryArea->SectionData.Segment; - ImageSectionObject = ((PMM_IMAGE_SECTION_OBJECT)Section->Segment); + ImageSectionObject = ImageSectionObjectFromSegment(Segment); SectionSegments = ImageSectionObject->Segments; NrSegments = ImageSectionObject->NrSegments; @@ -3937,6 +3940,15 @@ MmMapViewOfSection(IN PVOID SectionObject, MmUnlockSectionSegment(&SectionSegments[i]); if (!NT_SUCCESS(Status)) { + /* roll-back */ + while (i--) + { + SBaseAddress = ((char*)ImageBase + (ULONG_PTR)SectionSegments[i].Image.VirtualAddress); + MmLockSectionSegment(&SectionSegments[i]); + MmUnmapViewOfSegment(AddressSpace, SBaseAddress); + MmUnlockSectionSegment(&SectionSegments[i]); + } + MmUnlockAddressSpace(AddressSpace); return(Status); } @@ -4018,7 +4030,6 @@ MmMapViewOfSection(IN PVOID SectionObject, } MmUnlockAddressSpace(AddressSpace); - ASSERT(*BaseAddress == ALIGN_DOWN_POINTER_BY(*BaseAddress, MM_VIRTMEM_GRANULARITY)); if (NotAtBase) Status = STATUS_IMAGE_NOT_AT_BASE; @@ -4074,7 +4085,7 @@ CheckSectionPointer: else { /* We can't shrink, but we can extend */ - Ret = NewFileSize->QuadPart > Segment->RawLength.QuadPart; + Ret = NewFileSize->QuadPart >= Segment->RawLength.QuadPart; } } else @@ -4679,6 +4690,8 @@ MmCheckDirtySegment( IoSetTopLevelIrp((PIRP)FSRTL_MOD_WRITE_TOP_LEVEL_IRP); /* Go ahead and write the page */ + DPRINT("Writing page at offset %I64d for file %wZ, Pageout: %s\n", + Offset->QuadPart, &Segment->FileObject->FileName, PageOut ? "TRUE" : "FALSE"); Status = MiWritePage(Segment, Offset->QuadPart, Page); if (PageOut) From 674ab3dce3da08ca41b9685cbec5bb3d48fdcece Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Thu, 17 Dec 2020 18:09:56 +0100 Subject: [PATCH 54/99] [NTOS:MM] Remove a check for now --- ntoskrnl/mm/section.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index c20c506c2b8..b709a50cd8d 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -4356,12 +4356,17 @@ MmCreateSection (OUT PVOID * Section, FileLock = TRUE; /* Deny access if there are writes on the file */ +#if 0 if ((AllocationAttributes & SEC_IMAGE) && (Status == STATUS_FILE_LOCKED_WITH_WRITERS)) { DPRINT1("Cannot create image maps with writers open on the file!\n"); Status = STATUS_ACCESS_DENIED; goto Quit; } +#else + if ((AllocationAttributes & SEC_IMAGE) && (Status == STATUS_FILE_LOCKED_WITH_WRITERS)) + DPRINT1("Creating image map with writers open on the file!\n"); +#endif } else { @@ -4415,7 +4420,6 @@ MmCreateSection (OUT PVOID * Section, Status = STATUS_INVALID_PARAMETER; } -Quit: if (FileLock) FsRtlReleaseFile(FileObject); if (FileObject) From 6d97d8d2e15d9db634a2188d0f17983d74ea156c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Fri, 18 Dec 2020 15:24:09 +0100 Subject: [PATCH 55/99] [NTOS:CC] Fix some tests, complain where the current implementation won't let us do the right thing --- ntoskrnl/cc/pin.c | 9 ++++++++- ntoskrnl/cc/view.c | 6 ------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/ntoskrnl/cc/pin.c b/ntoskrnl/cc/pin.c index 0d5642b9932..597468f6fd7 100644 --- a/ntoskrnl/cc/pin.c +++ b/ntoskrnl/cc/pin.c @@ -225,9 +225,13 @@ CcpPinData( BOOLEAN Result; VacbOffset = (ULONG)(FileOffset->QuadPart % VACB_MAPPING_GRANULARITY); - /* This seems to be valid, according to KMTests */ + if ((VacbOffset + Length) > VACB_MAPPING_GRANULARITY) + { + /* Complain loudly, we shoud pin the whole range */ + DPRINT1("TRUNCATING DATA PIN FROM %lu to %lu!\n", Length, VACB_MAPPING_GRANULARITY - VacbOffset); Length = VACB_MAPPING_GRANULARITY - VacbOffset; + } KeAcquireSpinLock(&SharedCacheMap->BcbSpinLock, &OldIrql); NewBcb = CcpFindBcb(SharedCacheMap, FileOffset, Length, TRUE); @@ -353,7 +357,10 @@ CcMapData ( VacbOffset = (ULONG)(FileOffset->QuadPart % VACB_MAPPING_GRANULARITY); /* KMTests seem to show that it is allowed to call accross mapping granularity */ if ((VacbOffset + Length) > VACB_MAPPING_GRANULARITY) + { + DPRINT1("TRUNCATING DATA MAP FROM %lu to %lu!\n", Length, VACB_MAPPING_GRANULARITY - VacbOffset); Length = VACB_MAPPING_GRANULARITY - VacbOffset; + } KeAcquireSpinLock(&SharedCacheMap->BcbSpinLock, &OldIrql); iBcb = CcpFindBcb(SharedCacheMap, FileOffset, Length, FALSE); diff --git a/ntoskrnl/cc/view.c b/ntoskrnl/cc/view.c index ae6464f2b58..3952986dc44 100644 --- a/ntoskrnl/cc/view.c +++ b/ntoskrnl/cc/view.c @@ -572,12 +572,6 @@ CcRosCreateVacb ( DPRINT("CcRosCreateVacb()\n"); - if (FileOffset >= SharedCacheMap->SectionSize.QuadPart) - { - *Vacb = NULL; - return STATUS_INVALID_PARAMETER; - } - current = ExAllocateFromNPagedLookasideList(&VacbLookasideList); current->BaseAddress = NULL; current->Valid = FALSE; From 91f6fa0259cc2a98371b8644310d47d7fdf79e12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Fri, 18 Dec 2020 16:19:45 +0100 Subject: [PATCH 56/99] Disable CcScheduleReadAhead call --- ntoskrnl/cc/copy.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ntoskrnl/cc/copy.c b/ntoskrnl/cc/copy.c index 3dd172fa9db..502dfb03ee8 100644 --- a/ntoskrnl/cc/copy.c +++ b/ntoskrnl/cc/copy.c @@ -556,6 +556,7 @@ CcCopyRead ( IoStatus->Status = STATUS_SUCCESS; IoStatus->Information = ReadLength; +#if 0 /* If that was a successful sync read operation, let's handle read ahead */ if (Length == 0 && Wait) { @@ -576,6 +577,7 @@ CcCopyRead ( PrivateCacheMap->FileOffset2.QuadPart = FileOffset->QuadPart; PrivateCacheMap->BeyondLastByte2.QuadPart = FileOffset->QuadPart + ReadLength; } +#endif return TRUE; } From 543112008aafc40b34fd68b43baa04645aeaaaf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Fri, 18 Dec 2020 16:53:39 +0100 Subject: [PATCH 57/99] Fix physical memory section creation --- ntoskrnl/mm/section.c | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index b709a50cd8d..b9242c6eade 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -2068,7 +2068,7 @@ MmCreatePhysicalMemorySection(VOID) /* * Create the section mapping physical memory */ - SectionSize.QuadPart = MmNumberOfPhysicalPages * PAGE_SIZE; + SectionSize.QuadPart = MmHighestPhysicalPage * PAGE_SIZE; InitializeObjectAttributes(&Obj, &Name, OBJ_PERMANENT | OBJ_KERNEL_EXCLUSIVE, @@ -2136,6 +2136,7 @@ MmCreatePhysicalMemorySection(VOID) if (!NT_SUCCESS(Status)) { ObDereferenceObject(PhysSection); + return Status; } ObCloseHandle(Handle, KernelMode); @@ -3654,16 +3655,17 @@ NtQuerySection( Sbi.BaseAddress = (PVOID)Section->Address.StartingVpn; Sbi.Attributes = 0; - if (Section->u.Flags.Commit) - Sbi.Attributes |= SEC_COMMIT; - if (Section->u.Flags.Reserve) - Sbi.Attributes |= SEC_RESERVE; if (Section->u.Flags.File) Sbi.Attributes |= SEC_FILE; if (Section->u.Flags.Image) Sbi.Attributes |= SEC_IMAGE; - /* FIXME : Complete/test the list of flags passed back from NtCreateSection */ + /* Those are not set ************* + if (Section->u.Flags.Commit) + Sbi.Attributes |= SEC_COMMIT; + if (Section->u.Flags.Reserve) + Sbi.Attributes |= SEC_RESERVE; + **********************************/ if (Section->u.Flags.Image) { @@ -4005,8 +4007,9 @@ MmMapViewOfSection(IN PVOID SectionObject, { (*ViewSize) = Section->SizeOfSection.QuadPart - ViewOffset; } - else if (((*ViewSize)+ViewOffset) > Section->SizeOfSection.QuadPart) + else if ((ExGetPreviousMode() == UserMode) && (((*ViewSize)+ViewOffset) > Section->SizeOfSection.QuadPart)) { + /* Dubious */ (*ViewSize) = MIN(Section->SizeOfSection.QuadPart - ViewOffset, SIZE_T_MAX - PAGE_SIZE); } @@ -4329,8 +4332,16 @@ MmCreateSection (OUT PVOID * Section, return STATUS_INVALID_PARAMETER_6; } - /* Did the caller pass a handle? */ - if (FileHandle) + /* Did the caller pass a file object ? */ + if (FileObject) + { + /* Reference the object directly */ + ObReferenceObject(FileObject); + + /* We don't create image mappings with file objects */ + AllocationAttributes &= ~SEC_IMAGE; + } + else { /* Reference the file handle to get the object */ Status = ObReferenceObjectByHandle(FileHandle, @@ -4368,11 +4379,6 @@ MmCreateSection (OUT PVOID * Section, DPRINT1("Creating image map with writers open on the file!\n"); #endif } - else - { - /* Reference the object directly */ - ObReferenceObject(FileObject); - } } else { From addf824d8739e937098e10fc3cd05ca094016d6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Mon, 21 Dec 2020 13:54:29 +0100 Subject: [PATCH 58/99] [NTOS:MM] Keep image maps & file maps coherent at the time of creating the image map --- ntoskrnl/mm/section.c | 134 +++++++++++++++++++++--------------------- 1 file changed, 68 insertions(+), 66 deletions(-) diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index b9242c6eade..4756875a661 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -2448,6 +2448,19 @@ ExeFmtpAllocateSegments(IN ULONG NrSegments) return Segments; } +static NTSTATUS +MmMapViewOfSegment(PMMSUPPORT AddressSpace, + BOOLEAN AsImage, + PMM_SECTION_SEGMENT Segment, + PVOID* BaseAddress, + SIZE_T ViewSize, + ULONG Protect, + LONGLONG ViewOffset, + ULONG AllocationType); +static NTSTATUS +MmUnmapViewOfSegment(PMMSUPPORT AddressSpace, + PVOID BaseAddress); + static NTSTATUS NTAPI @@ -2490,36 +2503,57 @@ ExeFmtpReadFile(IN PVOID File, BufferSize = Length + OffsetAdjustment; BufferSize = PAGE_ROUND_UP(BufferSize); - /* Flush data since we're about to perform a non-cached read */ - CcFlushCache(FileObject->SectionObjectPointer, - &FileOffset, - BufferSize, - &Iosb); - /* * It's ok to use paged pool, because this is a temporary buffer only used in * the loading of executables. The assumption is that MmCreateSection is * always called at low IRQLs and that these buffers don't survive a brief * initialization phase */ - Buffer = ExAllocatePoolWithTag(PagedPool, - BufferSize, - 'rXmM'); + Buffer = ExAllocatePoolWithTag(PagedPool, BufferSize, 'rXmM'); if (!Buffer) { return STATUS_INSUFFICIENT_RESOURCES; } - UsedSize = 0; - - Status = MiSimpleRead(FileObject, &FileOffset, Buffer, BufferSize, TRUE, &Iosb); - - UsedSize = (ULONG)Iosb.Information; - - if(NT_SUCCESS(Status) && UsedSize < OffsetAdjustment) + if (FileObject->SectionObjectPointer->DataSectionObject) { - Status = STATUS_IN_PAGE_ERROR; - ASSERT(!NT_SUCCESS(Status)); + PVOID SegmentMap = NULL; + + /* Get the data from the file mapping instead */ + MmLockAddressSpace(MmGetKernelAddressSpace()); + Status = MmMapViewOfSegment(MmGetKernelAddressSpace(), + FALSE, + FileObject->SectionObjectPointer->DataSectionObject, + &SegmentMap, + BufferSize, + PAGE_READONLY, + FileOffset.QuadPart, + 0); + MmUnlockAddressSpace(MmGetKernelAddressSpace()); + + if (!NT_SUCCESS(Status)) + return Status; + + RtlCopyMemory(Buffer, SegmentMap, BufferSize); + UsedSize = BufferSize; + + MmLockAddressSpace(MmGetKernelAddressSpace()); + + MmUnmapViewOfSegment(MmGetKernelAddressSpace(), SegmentMap); + + MmUnlockAddressSpace(MmGetKernelAddressSpace()); + } + else + { + Status = MiSimpleRead(FileObject, &FileOffset, Buffer, BufferSize, TRUE, &Iosb); + + UsedSize = (ULONG)Iosb.Information; + + if(NT_SUCCESS(Status) && UsedSize < OffsetAdjustment) + { + Status = STATUS_IN_PAGE_ERROR; + ASSERT(!NT_SUCCESS(Status)); + } } if(NT_SUCCESS(Status)) @@ -3102,7 +3136,6 @@ MmCreateImageSection(PSECTION *SectionObject, if (ImageSectionObject == NULL) { NTSTATUS StatusExeFmt; - PMM_SECTION_SEGMENT DataSectionObject; ImageSectionObject = ExAllocatePoolZero(NonPagedPool, sizeof(MM_IMAGE_SECTION_OBJECT), TAG_MM_SECTION_SEGMENT); if (ImageSectionObject == NULL) @@ -3116,44 +3149,12 @@ MmCreateImageSection(PSECTION *SectionObject, ImageSectionObject->RefCount = 1; FileObject->SectionObjectPointer->ImageSectionObject = ImageSectionObject; - /* Get a ref on the data section object */ - DataSectionObject = FileObject->SectionObjectPointer->DataSectionObject; - while (DataSectionObject && (DataSectionObject->SegFlags & (MM_SEGMENT_INDELETE | MM_SEGMENT_INCREATE))) - { - LARGE_INTEGER ShortTime; - - MiReleasePfnLock(OldIrql); - - ShortTime.QuadPart = - 10 * 100 * 1000; - KeDelayExecutionThread(KernelMode, FALSE, &ShortTime); - - OldIrql = MiAcquirePfnLock(); - DataSectionObject = FileObject->SectionObjectPointer->DataSectionObject; - ASSERT(DataSectionObject->SegFlags & MM_DATAFILE_SEGMENT); - } - - /* Get a ref on it. */ - if (DataSectionObject) - InterlockedIncrementUL(&DataSectionObject->RefCount); - MiReleasePfnLock(OldIrql); - if (DataSectionObject) + /* Purge the cache */ + if (CcIsFileCached(FileObject)) { - if ((DataSectionObject->SectionCount - (FileObject->SectionObjectPointer->SharedCacheMap != NULL)) > 0) - { - /* Someone's got a section opened. Deny creation */ - DPRINT1("Denying image creation for %wZ: Sections opened: %lu.\n", - &FileObject->FileName, DataSectionObject->SectionCount); - InterlockedExchangePointer(&FileObject->SectionObjectPointer->ImageSectionObject, NULL); - ExFreePoolWithTag(ImageSectionObject, TAG_MM_SECTION_SEGMENT); - MmDereferenceSegment(DataSectionObject); - ObDereferenceObject(Section); - return STATUS_ACCESS_DENIED; - } - - /* Purge the cache. */ - CcPurgeCacheSection(FileObject->SectionObjectPointer, NULL, 0, FALSE); + CcFlushCache(FileObject->SectionObjectPointer, NULL, 0, NULL); } StatusExeFmt = ExeFmtpCreateImageSection(FileObject, ImageSectionObject); @@ -3234,14 +3235,15 @@ MmCreateImageSection(PSECTION *SectionObject, static NTSTATUS -MmMapViewOfSegment(PMMSUPPORT AddressSpace, - PSECTION Section, - PMM_SECTION_SEGMENT Segment, - PVOID* BaseAddress, - SIZE_T ViewSize, - ULONG Protect, - LONGLONG ViewOffset, - ULONG AllocationType) +MmMapViewOfSegment( + PMMSUPPORT AddressSpace, + BOOLEAN AsImage, + PMM_SECTION_SEGMENT Segment, + PVOID* BaseAddress, + SIZE_T ViewSize, + ULONG Protect, + LONGLONG ViewOffset, + ULONG AllocationType) { PMEMORY_AREA MArea; NTSTATUS Status; @@ -3297,7 +3299,7 @@ MmMapViewOfSegment(PMMSUPPORT AddressSpace, MArea->SectionData.Segment = Segment; MArea->SectionData.ViewOffset.QuadPart = ViewOffset; - if (Section->u.Flags.Image) + if (AsImage) { MArea->VadNode.u.VadFlags.VadType = VadImageMap; } @@ -3932,7 +3934,7 @@ MmMapViewOfSection(IN PVOID SectionObject, ((char*)ImageBase + (ULONG_PTR)SectionSegments[i].Image.VirtualAddress); MmLockSectionSegment(&SectionSegments[i]); Status = MmMapViewOfSegment(AddressSpace, - Section, + TRUE, &SectionSegments[i], &SBaseAddress, SectionSegments[i].Length.QuadPart, @@ -4017,7 +4019,7 @@ MmMapViewOfSection(IN PVOID SectionObject, MmLockSectionSegment(Segment); Status = MmMapViewOfSegment(AddressSpace, - Section, + FALSE, Segment, BaseAddress, *ViewSize, @@ -4191,7 +4193,7 @@ MmMapViewInSystemSpaceEx ( MmLockSectionSegment(Segment); Status = MmMapViewOfSegment(AddressSpace, - Section, + Section->u.Flags.Image, Segment, MappedBase, *ViewSize, From 347a4f146b8245dd6753e39365aafe492d37ca60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Mon, 21 Dec 2020 18:47:32 +0100 Subject: [PATCH 59/99] [NTOS] Loop again and again until the whole cache is empty when sutting down --- ntoskrnl/cc/view.c | 11 ++++++++++- ntoskrnl/po/power.c | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/ntoskrnl/cc/view.c b/ntoskrnl/cc/view.c index 3952986dc44..35d0df31545 100644 --- a/ntoskrnl/cc/view.c +++ b/ntoskrnl/cc/view.c @@ -212,6 +212,7 @@ CcRosFlushDirtyPages ( BOOLEAN Locked; NTSTATUS Status; KIRQL OldIrql; + BOOLEAN FlushAll = (Target == MAXULONG); DPRINT("CcRosFlushDirtyPages(Target %lu)\n", Target); @@ -226,8 +227,16 @@ CcRosFlushDirtyPages ( DPRINT("No Dirty pages\n"); } - while ((current_entry != &DirtyVacbListHead) && (Target > 0)) + while (((current_entry != &DirtyVacbListHead) && (Target > 0)) || FlushAll) { + if (current_entry == &DirtyVacbListHead) + { + ASSERT(FlushAll); + if (IsListEmpty(&DirtyVacbListHead)) + break; + current_entry = DirtyVacbListHead.Flink; + } + current = CONTAINING_RECORD(current_entry, ROS_VACB, DirtyVacbListEntry); diff --git a/ntoskrnl/po/power.c b/ntoskrnl/po/power.c index 7d2f4d7114a..6dee7054c02 100644 --- a/ntoskrnl/po/power.c +++ b/ntoskrnl/po/power.c @@ -1075,7 +1075,7 @@ NtSetSystemPowerState(IN POWER_ACTION SystemAction, #ifndef NEWCC /* Flush dirty cache pages */ /* XXX: Is that still mandatory? As now we'll wait on lazy writer to complete? */ - CcRosFlushDirtyPages(-1, &Dummy, TRUE, FALSE); //HACK: We really should wait here! + CcRosFlushDirtyPages(MAXULONG, &Dummy, TRUE, FALSE); //HACK: We really should wait here! #else Dummy = 0; #endif From 1505abbc095f9f9414e15847ce840d2a8fe592d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Tue, 22 Dec 2020 11:31:51 +0100 Subject: [PATCH 60/99] [NTOS:CC] Do not write behind concurrently the same file --- ntoskrnl/cc/view.c | 12 ++++++++++++ ntoskrnl/include/internal/cc.h | 1 + 2 files changed, 13 insertions(+) diff --git a/ntoskrnl/cc/view.c b/ntoskrnl/cc/view.c index 35d0df31545..e6ac23c2b6e 100644 --- a/ntoskrnl/cc/view.c +++ b/ntoskrnl/cc/view.c @@ -262,6 +262,15 @@ CcRosFlushDirtyPages ( ASSERT(current->Dirty); + /* Do not lazy-write the same file concurrently. Fastfat ASSERTS on that */ + if (current->SharedCacheMap->Flags & SHARED_CACHE_MAP_IN_LAZYWRITE) + { + CcRosVacbDecRefCount(current); + continue; + } + + current->SharedCacheMap->Flags |= SHARED_CACHE_MAP_IN_LAZYWRITE; + KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); Locked = current->SharedCacheMap->Callbacks->AcquireForLazyWrite( @@ -272,6 +281,7 @@ CcRosFlushDirtyPages ( ASSERT(!Wait); OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); CcRosVacbDecRefCount(current); + current->SharedCacheMap->Flags &= ~SHARED_CACHE_MAP_IN_LAZYWRITE; continue; } @@ -287,6 +297,8 @@ CcRosFlushDirtyPages ( CcRosVacbDecRefCount(current); OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); + current->SharedCacheMap->Flags &= ~SHARED_CACHE_MAP_IN_LAZYWRITE; + if (!NT_SUCCESS(Status) && (Status != STATUS_END_OF_FILE) && (Status != STATUS_MEDIA_WRITE_PROTECTED)) { diff --git a/ntoskrnl/include/internal/cc.h b/ntoskrnl/include/internal/cc.h index 862a759ef7a..65fd3fe6ffa 100644 --- a/ntoskrnl/include/internal/cc.h +++ b/ntoskrnl/include/internal/cc.h @@ -200,6 +200,7 @@ typedef struct _ROS_SHARED_CACHE_MAP #define READAHEAD_DISABLED 0x1 #define WRITEBEHIND_DISABLED 0x2 #define SHARED_CACHE_MAP_IN_CREATION 0x4 +#define SHARED_CACHE_MAP_IN_LAZYWRITE 0x8 typedef struct _ROS_VACB { From 358d06c47ca31d8854c8ba2ffa16ab577f798e82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Tue, 29 Dec 2020 10:07:57 +0100 Subject: [PATCH 61/99] [NTOS:CC] Always honor WriteThrough parameter in CcUnpinRepinnedBcb --- ntoskrnl/cc/pin.c | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/ntoskrnl/cc/pin.c b/ntoskrnl/cc/pin.c index 597468f6fd7..8a79ee45031 100644 --- a/ntoskrnl/cc/pin.c +++ b/ntoskrnl/cc/pin.c @@ -629,29 +629,25 @@ CcUnpinRepinnedBcb ( SharedCacheMap = iBcb->Vacb->SharedCacheMap; IoStatus->Status = STATUS_SUCCESS; + if (WriteThrough) + { + CcFlushCache(iBcb->Vacb->SharedCacheMap->FileObject->SectionObjectPointer, + &iBcb->PFCB.MappedFileOffset, + iBcb->PFCB.MappedLength, + IoStatus); + } + else + { + IoStatus->Status = STATUS_SUCCESS; + IoStatus->Information = 0; + } + KeAcquireSpinLock(&SharedCacheMap->BcbSpinLock, &OldIrql); if (--iBcb->RefCount == 0) { RemoveEntryList(&iBcb->BcbEntry); KeReleaseSpinLock(&SharedCacheMap->BcbSpinLock, OldIrql); - IoStatus->Information = 0; - if (WriteThrough) - { - if (iBcb->Vacb->Dirty) - { - IoStatus->Status = CcRosFlushVacb(iBcb->Vacb); - } - else - { - IoStatus->Status = STATUS_SUCCESS; - } - } - else - { - IoStatus->Status = STATUS_SUCCESS; - } - if (iBcb->PinCount != 0) { ExReleaseResourceLite(&iBcb->Lock); From 8287a098b94d5cf60a9604e711fba19d3a758be5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Tue, 29 Dec 2020 11:26:25 +0100 Subject: [PATCH 62/99] [NTOS:CC] Fix potnetial use-after-free --- ntoskrnl/cc/view.c | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/ntoskrnl/cc/view.c b/ntoskrnl/cc/view.c index e6ac23c2b6e..6f0490c31d0 100644 --- a/ntoskrnl/cc/view.c +++ b/ntoskrnl/cc/view.c @@ -208,8 +208,6 @@ CcRosFlushDirtyPages ( BOOLEAN CalledFromLazy) { PLIST_ENTRY current_entry; - PROS_VACB current; - BOOLEAN Locked; NTSTATUS Status; KIRQL OldIrql; BOOLEAN FlushAll = (Target == MAXULONG); @@ -229,6 +227,10 @@ CcRosFlushDirtyPages ( while (((current_entry != &DirtyVacbListHead) && (Target > 0)) || FlushAll) { + PROS_SHARED_CACHE_MAP SharedCacheMap; + PROS_VACB current; + BOOLEAN Locked; + if (current_entry == &DirtyVacbListHead) { ASSERT(FlushAll); @@ -244,17 +246,17 @@ CcRosFlushDirtyPages ( CcRosVacbIncRefCount(current); + SharedCacheMap = current->SharedCacheMap; + /* When performing lazy write, don't handle temporary files */ - if (CalledFromLazy && - BooleanFlagOn(current->SharedCacheMap->FileObject->Flags, FO_TEMPORARY_FILE)) + if (CalledFromLazy && BooleanFlagOn(SharedCacheMap->FileObject->Flags, FO_TEMPORARY_FILE)) { CcRosVacbDecRefCount(current); continue; } /* Don't attempt to lazy write the files that asked not to */ - if (CalledFromLazy && - BooleanFlagOn(current->SharedCacheMap->Flags, WRITEBEHIND_DISABLED)) + if (CalledFromLazy && BooleanFlagOn(SharedCacheMap->Flags, WRITEBEHIND_DISABLED)) { CcRosVacbDecRefCount(current); continue; @@ -263,32 +265,30 @@ CcRosFlushDirtyPages ( ASSERT(current->Dirty); /* Do not lazy-write the same file concurrently. Fastfat ASSERTS on that */ - if (current->SharedCacheMap->Flags & SHARED_CACHE_MAP_IN_LAZYWRITE) + if (SharedCacheMap->Flags & SHARED_CACHE_MAP_IN_LAZYWRITE) { CcRosVacbDecRefCount(current); continue; } - current->SharedCacheMap->Flags |= SHARED_CACHE_MAP_IN_LAZYWRITE; + SharedCacheMap->Flags |= SHARED_CACHE_MAP_IN_LAZYWRITE; KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); - Locked = current->SharedCacheMap->Callbacks->AcquireForLazyWrite( - current->SharedCacheMap->LazyWriteContext, Wait); + Locked = SharedCacheMap->Callbacks->AcquireForLazyWrite(SharedCacheMap->LazyWriteContext, Wait); if (!Locked) { DPRINT("Not locked!"); ASSERT(!Wait); - OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); CcRosVacbDecRefCount(current); - current->SharedCacheMap->Flags &= ~SHARED_CACHE_MAP_IN_LAZYWRITE; + OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); + SharedCacheMap->Flags &= ~SHARED_CACHE_MAP_IN_LAZYWRITE; continue; } Status = CcRosFlushVacb(current); - current->SharedCacheMap->Callbacks->ReleaseFromLazyWrite( - current->SharedCacheMap->LazyWriteContext); + SharedCacheMap->Callbacks->ReleaseFromLazyWrite(SharedCacheMap->LazyWriteContext); /* We release the VACB before acquiring the lock again, because * CcRosVacbDecRefCount might free the VACB, as CcRosFlushVacb dropped a @@ -297,7 +297,7 @@ CcRosFlushDirtyPages ( CcRosVacbDecRefCount(current); OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); - current->SharedCacheMap->Flags &= ~SHARED_CACHE_MAP_IN_LAZYWRITE; + SharedCacheMap->Flags &= ~SHARED_CACHE_MAP_IN_LAZYWRITE; if (!NT_SUCCESS(Status) && (Status != STATUS_END_OF_FILE) && (Status != STATUS_MEDIA_WRITE_PROTECTED)) From cff3c399c63a36bd5f05aa2389d54b0824a567ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Tue, 29 Dec 2020 16:43:03 +0100 Subject: [PATCH 63/99] [NTOS:MM] Fix input validation/correction in MmMapViewInSystemSpace --- ntoskrnl/include/ntoskrnl.h | 2 ++ ntoskrnl/mm/section.c | 56 +++++++++++++++++++++++++++++++++---- 2 files changed, 52 insertions(+), 6 deletions(-) diff --git a/ntoskrnl/include/ntoskrnl.h b/ntoskrnl/include/ntoskrnl.h index 7685c36bcf1..353efeb44c3 100644 --- a/ntoskrnl/include/ntoskrnl.h +++ b/ntoskrnl/include/ntoskrnl.h @@ -63,7 +63,9 @@ #include #include #include +#define ENABLE_INTSAFE_SIGNED_FUNCTIONS #include +#undef ENABLE_INTSAFE_SIGNED_FUNCTIONS /* C Headers */ #include diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index 4756875a661..17762f8628e 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -3249,6 +3249,8 @@ MmMapViewOfSegment( NTSTATUS Status; ULONG Granularity; + ASSERT(ViewSize != 0); + if (Segment->WriteCopy) { /* We have to do this because the not present fault @@ -3488,6 +3490,8 @@ MiRosUnmapViewOfSection(IN PEPROCESS Process, { if (MemoryArea) ASSERT(MemoryArea->Type != MEMORY_AREA_OWNED_BY_ARM3); + + DPRINT1("Unable to find memory area at address %p.\n", BaseAddress); MmUnlockAddressSpace(AddressSpace); return STATUS_NOT_MAPPED_VIEW; } @@ -4009,7 +4013,9 @@ MmMapViewOfSection(IN PVOID SectionObject, { (*ViewSize) = Section->SizeOfSection.QuadPart - ViewOffset; } - else if ((ExGetPreviousMode() == UserMode) && (((*ViewSize)+ViewOffset) > Section->SizeOfSection.QuadPart)) + else if ((ExGetPreviousMode() == UserMode) && + (((*ViewSize)+ViewOffset) > Section->SizeOfSection.QuadPart) && + (!Section->u.Flags.Reserve)) { /* Dubious */ (*ViewSize) = MIN(Section->SizeOfSection.QuadPart - ViewOffset, SIZE_T_MAX - PAGE_SIZE); @@ -4165,6 +4171,7 @@ MmMapViewInSystemSpaceEx ( PMM_SECTION_SEGMENT Segment; PMMSUPPORT AddressSpace; NTSTATUS Status; + PAGED_CODE(); if (MiIsRosSectionObject(SectionObject) == FALSE) @@ -4181,15 +4188,49 @@ MmMapViewInSystemSpaceEx ( Section = SectionObject; Segment = (PMM_SECTION_SEGMENT)Section->Segment; + if (*ViewSize == 0) + { + LONGLONG MapSizeLL; + + /* Page-align the mapping */ + SectionOffset->LowPart = PAGE_ROUND_DOWN(SectionOffset->LowPart); + + if (!NT_SUCCESS(RtlLongLongSub(Section->SizeOfSection.QuadPart, SectionOffset->QuadPart, &MapSizeLL))) + return STATUS_INVALID_VIEW_SIZE; + + if (!NT_SUCCESS(RtlLongLongToSIZET(MapSizeLL, ViewSize))) + return STATUS_INVALID_VIEW_SIZE; + } + else + { + LONGLONG HelperLL; + + /* Get the map end */ + if (!NT_SUCCESS(RtlLongLongAdd(SectionOffset->QuadPart, *ViewSize, &HelperLL))) + return STATUS_INVALID_VIEW_SIZE; + + /* Round it up, if needed */ + if (HelperLL % PAGE_SIZE) + { + if (!NT_SUCCESS(RtlLongLongAdd(HelperLL, PAGE_SIZE - (HelperLL % PAGE_SIZE), &HelperLL))) + return STATUS_INVALID_VIEW_SIZE; + } + + /* Now that we have the mapping end, we can align down its start */ + SectionOffset->LowPart = PAGE_ROUND_DOWN(SectionOffset->LowPart); + + /* Get the new size */ + if (!NT_SUCCESS(RtlLongLongSub(HelperLL, SectionOffset->QuadPart, &HelperLL))) + return STATUS_INVALID_VIEW_SIZE; + + if (!NT_SUCCESS(RtlLongLongToSIZET(HelperLL, ViewSize))) + return STATUS_INVALID_VIEW_SIZE; + } + AddressSpace = MmGetKernelAddressSpace(); MmLockAddressSpace(AddressSpace); - if (*ViewSize == 0) - { - *ViewSize = MIN((Section->SizeOfSection.QuadPart - SectionOffset->QuadPart), SIZE_T_MAX); - } - MmLockSectionSegment(Segment); Status = MmMapViewOfSegment(AddressSpace, @@ -4505,6 +4546,7 @@ MmMakePagesResident( MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace, Address); if (MemoryArea == NULL) { + DPRINT1("Unable to find memory area at address %p.\n", Address); MmUnlockAddressSpace(AddressSpace); return STATUS_NOT_MAPPED_VIEW; } @@ -4608,6 +4650,7 @@ MmRosFlushVirtualMemory( if ((MemoryArea == NULL) || (MemoryArea->Type != MEMORY_AREA_SECTION_VIEW) || (MemoryArea->VadNode.u.VadFlags.VadType == VadImageMap)) { + DPRINT1("Unable to find memory area at address %p.\n", Address); MmUnlockAddressSpace(AddressSpace); return STATUS_NOT_MAPPED_VIEW; } @@ -4766,6 +4809,7 @@ MmMakePagesDirty( MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace, Address); if (MemoryArea == NULL) { + DPRINT1("Unable to find memory area at address %p.\n", Address); MmUnlockAddressSpace(AddressSpace); return STATUS_NOT_MAPPED_VIEW; } From 57ee31ee33e2fdf7ef70172cd78690a0ddb4b92c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Tue, 29 Dec 2020 16:55:19 +0100 Subject: [PATCH 64/99] [NTOS:CC] Perform sanity checks before doing anything else --- ntoskrnl/cc/pin.c | 4 +--- ntoskrnl/cc/view.c | 21 ++++++++++++--------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/ntoskrnl/cc/pin.c b/ntoskrnl/cc/pin.c index 8a79ee45031..30cba0a248a 100644 --- a/ntoskrnl/cc/pin.c +++ b/ntoskrnl/cc/pin.c @@ -581,7 +581,6 @@ CcUnpinDataForThread ( IN ERESOURCE_THREAD ResourceThreadId) { PINTERNAL_BCB iBcb = Bcb; - PROS_SHARED_CACHE_MAP SharedCacheMap; CCTRACE(CC_API_DEBUG, "Bcb=%p ResourceThreadId=%lu\n", Bcb, ResourceThreadId); @@ -591,8 +590,7 @@ CcUnpinDataForThread ( iBcb->PinCount--; } - SharedCacheMap = iBcb->Vacb->SharedCacheMap; - CcpDereferenceBcb(SharedCacheMap, iBcb); + CcpDereferenceBcb(iBcb->Vacb->SharedCacheMap, iBcb); } /* diff --git a/ntoskrnl/cc/view.c b/ntoskrnl/cc/view.c index 6f0490c31d0..4866db4f849 100644 --- a/ntoskrnl/cc/view.c +++ b/ntoskrnl/cc/view.c @@ -740,11 +740,13 @@ CcRosEnsureVacbResident( ASSERT((Offset + Length) <= VACB_MAPPING_GRANULARITY); +#if 0 if ((Vacb->FileOffset.QuadPart + Offset) > Vacb->SharedCacheMap->SectionSize.QuadPart) { DPRINT1("Vacb read beyond the file size!\n"); return FALSE; } +#endif BaseAddress = (PVOID)((ULONG_PTR)Vacb->BaseAddress + Offset); @@ -862,15 +864,6 @@ CcRosInternalFreeVacb ( } #endif - /* Delete the mapping */ - Status = MmUnmapViewInSystemSpace(Vacb->BaseAddress); - if (!NT_SUCCESS(Status)) - { - DPRINT1("Failed to unmap VACB from System address space! Status 0x%08X\n", Status); - ASSERT(FALSE); - /* Proceed with the deĺetion anyway */ - } - if (Vacb->ReferenceCount != 0) { DPRINT1("Invalid free: %ld\n", Vacb->ReferenceCount); @@ -884,6 +877,16 @@ CcRosInternalFreeVacb ( ASSERT(IsListEmpty(&Vacb->CacheMapVacbListEntry)); ASSERT(IsListEmpty(&Vacb->DirtyVacbListEntry)); ASSERT(IsListEmpty(&Vacb->VacbLruListEntry)); + + /* Delete the mapping */ + Status = MmUnmapViewInSystemSpace(Vacb->BaseAddress); + if (!NT_SUCCESS(Status)) + { + DPRINT1("Failed to unmap VACB from System address space! Status 0x%08X\n", Status); + ASSERT(FALSE); + /* Proceed with the deĺetion anyway */ + } + RtlFillMemory(Vacb, sizeof(*Vacb), 0xfd); ExFreeToNPagedLookasideList(&VacbLookasideList, Vacb); return STATUS_SUCCESS; From caf89b95829ca8cf851ed20103b70a645e8c5a83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Tue, 29 Dec 2020 19:50:00 +0100 Subject: [PATCH 65/99] [NTOS:MM] Fix a race condition when unmapping sections views --- ntoskrnl/mm/ARM3/section.c | 18 ++++++++++++------ ntoskrnl/mm/marea.c | 6 ------ ntoskrnl/mm/section.c | 19 +++---------------- 3 files changed, 15 insertions(+), 28 deletions(-) diff --git a/ntoskrnl/mm/ARM3/section.c b/ntoskrnl/mm/ARM3/section.c index dde3b2d8cc4..fddaf752031 100644 --- a/ntoskrnl/mm/ARM3/section.c +++ b/ntoskrnl/mm/ARM3/section.c @@ -833,12 +833,17 @@ MiUnmapViewOfSection(IN PEPROCESS Process, PEPROCESS CurrentProcess = PsGetCurrentProcess(); PAGED_CODE(); + /* Check if we need to lock the address space */ + if (!Flags) MmLockAddressSpace(&Process->Vm); + /* Check for Mm Region */ MemoryArea = MmLocateMemoryAreaByAddress(&Process->Vm, BaseAddress); if ((MemoryArea) && (MemoryArea->Type != MEMORY_AREA_OWNED_BY_ARM3)) { /* Call Mm API */ - return MiRosUnmapViewOfSection(Process, BaseAddress, Process->ProcessExiting); + NTSTATUS Status = MiRosUnmapViewOfSection(Process, BaseAddress, Process->ProcessExiting); + if (!Flags) MmUnlockAddressSpace(&Process->Vm); + return Status; } /* Check if we should attach to the process */ @@ -849,10 +854,7 @@ MiUnmapViewOfSection(IN PEPROCESS Process, Attached = TRUE; } - /* Check if we need to lock the address space */ - if (!Flags) MmLockAddressSpace(&Process->Vm); - - /* Check if the process is already daed */ + /* Check if the process is already dead */ if (Process->VmDeleted) { /* Fail the call */ @@ -3116,11 +3118,15 @@ MmUnmapViewInSystemSpace(IN PVOID MappedBase) PAGED_CODE(); /* Was this mapped by RosMm? */ + MmLockAddressSpace(MmGetKernelAddressSpace()); MemoryArea = MmLocateMemoryAreaByAddress(MmGetKernelAddressSpace(), MappedBase); if ((MemoryArea) && (MemoryArea->Type != MEMORY_AREA_OWNED_BY_ARM3)) { - return MiRosUnmapViewInSystemSpace(MappedBase); + NTSTATUS Status = MiRosUnmapViewInSystemSpace(MappedBase); + MmUnlockAddressSpace(MmGetKernelAddressSpace()); + return Status; } + MmUnlockAddressSpace(MmGetKernelAddressSpace()); /* It was not, call the ARM3 routine */ return MiUnmapViewInSystemSpace(&MmSession, MappedBase); diff --git a/ntoskrnl/mm/marea.c b/ntoskrnl/mm/marea.c index cd7b9594f1d..6e4226b6c35 100644 --- a/ntoskrnl/mm/marea.c +++ b/ntoskrnl/mm/marea.c @@ -543,9 +543,6 @@ MiRosCleanupMemoryArea( (Process->ActiveThreads == 1)) || (Process->ActiveThreads == 0)); - /* We are in cleanup, we don't need to synchronize */ - MmUnlockAddressSpace(&Process->Vm); - MemoryArea = (PMEMORY_AREA)Vad; BaseAddress = (PVOID)MA_GetStartingAddress(MemoryArea); @@ -567,9 +564,6 @@ MiRosCleanupMemoryArea( /* Make sure this worked! */ ASSERT(NT_SUCCESS(Status)); - - /* Lock the address space again */ - MmLockAddressSpace(&Process->Vm); } VOID diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index 17762f8628e..1c7fbdbefc6 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -3459,6 +3459,7 @@ MmUnmapViewOfSegment(PMMSUPPORT AddressSpace, return(Status); } +/* This functions must be called with a locked address space */ NTSTATUS NTAPI MiRosUnmapViewOfSection(IN PEPROCESS Process, @@ -3477,7 +3478,6 @@ MiRosUnmapViewOfSection(IN PEPROCESS Process, AddressSpace = Process ? &Process->Vm : MmGetKernelAddressSpace(); - MmLockAddressSpace(AddressSpace); MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace, BaseAddress); if (MemoryArea == NULL || @@ -3492,7 +3492,6 @@ MiRosUnmapViewOfSection(IN PEPROCESS Process, if (MemoryArea) ASSERT(MemoryArea->Type != MEMORY_AREA_OWNED_BY_ARM3); DPRINT1("Unable to find memory area at address %p.\n", BaseAddress); - MmUnlockAddressSpace(AddressSpace); return STATUS_NOT_MAPPED_VIEW; } @@ -3551,8 +3550,6 @@ MiRosUnmapViewOfSection(IN PEPROCESS Process, } } - MmUnlockAddressSpace(AddressSpace); - /* Notify debugger */ if (ImageBaseAddress && !SkipDebuggerNotify) DbgkUnMapViewOfSection(ImageBaseAddress); @@ -4248,24 +4245,14 @@ MmMapViewInSystemSpaceEx ( return Status; } +/* This function must be called with adress space lock held */ NTSTATUS NTAPI MiRosUnmapViewInSystemSpace(IN PVOID MappedBase) { - PMMSUPPORT AddressSpace; - NTSTATUS Status; - DPRINT("MmUnmapViewInSystemSpace() called\n"); - AddressSpace = MmGetKernelAddressSpace(); - - MmLockAddressSpace(AddressSpace); - - Status = MmUnmapViewOfSegment(AddressSpace, MappedBase); - - MmUnlockAddressSpace(AddressSpace); - - return Status; + return MmUnmapViewOfSegment(MmGetKernelAddressSpace(), MappedBase); } /********************************************************************** From 11eee4eeec8771c399988b68e4c7bb829d2405ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Tue, 29 Dec 2020 19:50:59 +0100 Subject: [PATCH 66/99] [NTOS:MM] Check that we don't add rmap for NULL process --- ntoskrnl/mm/rmap.c | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/ntoskrnl/mm/rmap.c b/ntoskrnl/mm/rmap.c index 80576f4cc59..c000953cca1 100644 --- a/ntoskrnl/mm/rmap.c +++ b/ntoskrnl/mm/rmap.c @@ -410,17 +410,11 @@ MmInsertRmap(PFN_NUMBER Page, PEPROCESS Process, if (!RMAP_IS_SEGMENT(Address)) { - if (Process == NULL) + ASSERT(Process != NULL); + PrevSize = InterlockedExchangeAddUL(&Process->Vm.WorkingSetSize, PAGE_SIZE); + if (PrevSize >= Process->Vm.PeakWorkingSetSize) { - Process = PsInitialSystemProcess; - } - if (Process) - { - PrevSize = InterlockedExchangeAddUL(&Process->Vm.WorkingSetSize, PAGE_SIZE); - if (PrevSize >= Process->Vm.PeakWorkingSetSize) - { - Process->Vm.PeakWorkingSetSize = PrevSize + PAGE_SIZE; - } + Process->Vm.PeakWorkingSetSize = PrevSize + PAGE_SIZE; } } } @@ -455,14 +449,8 @@ MmDeleteRmap(PFN_NUMBER Page, PEPROCESS Process, ExFreeToNPagedLookasideList(&RmapLookasideList, current_entry); if (!RMAP_IS_SEGMENT(Address)) { - if (Process == NULL) - { - Process = PsInitialSystemProcess; - } - if (Process) - { - (void)InterlockedExchangeAddUL(&Process->Vm.WorkingSetSize, -PAGE_SIZE); - } + ASSERT(Process != NULL); + (void)InterlockedExchangeAddUL(&Process->Vm.WorkingSetSize, -PAGE_SIZE); } return; } From 8ed15a49a7540a91afc07d8eafe26a952b6d0ae6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Wed, 30 Dec 2020 09:43:55 +0100 Subject: [PATCH 67/99] [NTOS:MM] Fix a race --- ntoskrnl/mm/freelist.c | 7 ++++++- ntoskrnl/mm/rmap.c | 27 ++++++++++++++++++++++++++- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/ntoskrnl/mm/freelist.c b/ntoskrnl/mm/freelist.c index 93d9bbd9f59..ba16362f546 100644 --- a/ntoskrnl/mm/freelist.c +++ b/ntoskrnl/mm/freelist.c @@ -436,7 +436,12 @@ MmGetRmapListHeadPage(PFN_NUMBER Pfn) /* Get the entry */ Pfn1 = MiGetPfnEntry(Pfn); ASSERT(Pfn1); - ASSERT_IS_ROS_PFN(Pfn1); + + if (!MI_IS_ROS_PFN(Pfn1)) + { + MiReleasePfnLock(oldIrql); + return NULL; + } /* Get the list head */ ListHead = Pfn1->RmapListHead; diff --git a/ntoskrnl/mm/rmap.c b/ntoskrnl/mm/rmap.c index c000953cca1..785539ea58d 100644 --- a/ntoskrnl/mm/rmap.c +++ b/ntoskrnl/mm/rmap.c @@ -149,7 +149,32 @@ MmPageOutPhysicalAddress(PFN_NUMBER Page) /* Delete this virtual mapping in the process */ MmDeleteVirtualMapping(Process, Address, &Dirty, &MapPage); - ASSERT(MapPage == Page); + + /* There is a window betwwen the start of this function and now, + * where it's possible that the process changed its memory layout, + * because of copy-on-write, unmapping memory, or whatsoever. + * Just go away if that is the case */ + if (MapPage != Page) + { + PMM_REGION Region = MmFindRegion((PVOID)MA_GetStartingAddress(MemoryArea), + &MemoryArea->SectionData.RegionListHead, + Address, NULL); + /* Restore the mapping */ + MmCreateVirtualMapping(Process, Address, Region->Protect, &MapPage, 1); + if (Dirty) + MmSetDirtyPage(Process, Address); + + MmUnlockSectionSegment(Segment); + MmUnlockAddressSpace(AddressSpace); + if (Address < MmSystemRangeStart) + { + ExReleaseRundownProtection(&Process->RundownProtect); + ObDereferenceObject(Process); + } + + /* We can still try to flush it to disk, though */ + goto WriteSegment; + } if (Page != PFN_FROM_SSE(Entry)) { From a9193b5cc2891ae888afe059c4833985159efcb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Wed, 30 Dec 2020 09:45:06 +0100 Subject: [PATCH 68/99] [NTOS:CC] Remove dead code --- ntoskrnl/cc/view.c | 18 ------------------ ntoskrnl/include/internal/cc.h | 4 ---- 2 files changed, 22 deletions(-) diff --git a/ntoskrnl/cc/view.c b/ntoskrnl/cc/view.c index 4866db4f849..40413a91601 100644 --- a/ntoskrnl/cc/view.c +++ b/ntoskrnl/cc/view.c @@ -1092,24 +1092,6 @@ CcRosReferenceCache ( KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); } -VOID -NTAPI -CcRosRemoveIfClosed ( - PSECTION_OBJECT_POINTERS SectionObjectPointer) -{ - PROS_SHARED_CACHE_MAP SharedCacheMap; - KIRQL OldIrql; - - DPRINT("CcRosRemoveIfClosed()\n"); - OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); - SharedCacheMap = SectionObjectPointer->SharedCacheMap; - if (SharedCacheMap && SharedCacheMap->OpenCount == 0) - { - CcRosDeleteFileCache(SharedCacheMap->FileObject, SharedCacheMap, &OldIrql); - } - KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); -} - NTSTATUS NTAPI CcRosReleaseFileCache ( diff --git a/ntoskrnl/include/internal/cc.h b/ntoskrnl/include/internal/cc.h index 65fd3fe6ffa..32692c01db0 100644 --- a/ntoskrnl/include/internal/cc.h +++ b/ntoskrnl/include/internal/cc.h @@ -382,10 +382,6 @@ VOID NTAPI CcRosReferenceCache(PFILE_OBJECT FileObject); -VOID -NTAPI -CcRosRemoveIfClosed(PSECTION_OBJECT_POINTERS SectionObjectPointer); - NTSTATUS NTAPI CcRosReleaseVacb( From 33cde2831225c26f4b7afed69dc1f56e4a6ab32b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Wed, 30 Dec 2020 10:53:31 +0100 Subject: [PATCH 69/99] [NTOS:CC] Simplify CcRosDeleteFileCache --- ntoskrnl/cc/view.c | 151 +++++++++++++++++++++------------------------ 1 file changed, 70 insertions(+), 81 deletions(-) diff --git a/ntoskrnl/cc/view.c b/ntoskrnl/cc/view.c index 40413a91601..b6b7ec57560 100644 --- a/ntoskrnl/cc/view.c +++ b/ntoskrnl/cc/view.c @@ -974,105 +974,94 @@ CcRosDeleteFileCache ( */ { PLIST_ENTRY current_entry; - PROS_VACB current; - LIST_ENTRY FreeList; ASSERT(SharedCacheMap); + ASSERT(SharedCacheMap == FileObject->SectionObjectPointer->SharedCacheMap); + ASSERT(SharedCacheMap->OpenCount == 0); - SharedCacheMap->OpenCount++; + /* Remove all VACBs from the global lists */ + KeAcquireSpinLockAtDpcLevel(&SharedCacheMap->CacheMapLock); + current_entry = SharedCacheMap->CacheMapVacbListHead.Flink; + while (current_entry != &SharedCacheMap->CacheMapVacbListHead) + { + PROS_VACB Vacb = CONTAINING_RECORD(current_entry, ROS_VACB, CacheMapVacbListEntry); + + RemoveEntryList(&Vacb->VacbLruListEntry); + InitializeListHead(&Vacb->VacbLruListEntry); + + if (Vacb->Dirty) + { + CcRosUnmarkDirtyVacb(Vacb, FALSE); + /* Mark it as dirty again so we know that we have to flush before freeing it */ + Vacb->Dirty = TRUE; + } + + current_entry = current_entry->Flink; + } + + /* Make sure there is no trace anymore of this map */ + FileObject->SectionObjectPointer->SharedCacheMap = NULL; + RemoveEntryList(&SharedCacheMap->SharedCacheMapLinks); + + KeReleaseSpinLockFromDpcLevel(&SharedCacheMap->CacheMapLock); KeReleaseQueuedSpinLock(LockQueueMasterLock, *OldIrql); - CcFlushCache(FileObject->SectionObjectPointer, NULL, 0, NULL); - - *OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); - SharedCacheMap->OpenCount--; - if (SharedCacheMap->OpenCount == 0) + /* Now that we're out of the locks, free everything for real */ + while (!IsListEmpty(&SharedCacheMap->CacheMapVacbListHead)) { - FileObject->SectionObjectPointer->SharedCacheMap = NULL; + PROS_VACB Vacb = CONTAINING_RECORD(RemoveHeadList(&SharedCacheMap->CacheMapVacbListHead), ROS_VACB, CacheMapVacbListEntry); + ULONG RefCount; - /* - * Release all VACBs - */ - InitializeListHead(&FreeList); - KeAcquireSpinLockAtDpcLevel(&SharedCacheMap->CacheMapLock); - while (!IsListEmpty(&SharedCacheMap->CacheMapVacbListHead)) + InitializeListHead(&Vacb->CacheMapVacbListEntry); + + /* Flush to disk, if needed */ + if (Vacb->Dirty) { - current_entry = RemoveTailList(&SharedCacheMap->CacheMapVacbListHead); - KeReleaseSpinLockFromDpcLevel(&SharedCacheMap->CacheMapLock); + SIZE_T FlushSize = min(VACB_MAPPING_GRANULARITY, Vacb->SharedCacheMap->SectionSize.QuadPart - Vacb->FileOffset.QuadPart); + IO_STATUS_BLOCK Iosb; + NTSTATUS Status; - current = CONTAINING_RECORD(current_entry, ROS_VACB, CacheMapVacbListEntry); - RemoveEntryList(¤t->VacbLruListEntry); - InitializeListHead(¤t->VacbLruListEntry); - if (current->Dirty) + Status = MmFlushVirtualMemory(NULL, &Vacb->BaseAddress, &FlushSize, &Iosb); + if (!NT_SUCCESS(Status)) { - KeAcquireSpinLockAtDpcLevel(&SharedCacheMap->CacheMapLock); - CcRosUnmarkDirtyVacb(current, FALSE); - KeReleaseSpinLockFromDpcLevel(&SharedCacheMap->CacheMapLock); - DPRINT1("Freeing dirty VACB\n"); + /* Complain. There's not much we can do */ + DPRINT1("Failed to flush VACB to disk while deleting the cache entry. Status: 0x%08x\n", Status); } - if (current->MappedCount != 0) - { - current->MappedCount = 0; - NT_VERIFY(CcRosVacbDecRefCount(current) > 0); - DPRINT1("Freeing mapped VACB\n"); - } - InsertHeadList(&FreeList, ¤t->CacheMapVacbListEntry); - - KeAcquireSpinLockAtDpcLevel(&SharedCacheMap->CacheMapLock); + Vacb->Dirty = FALSE; } -#if DBG - SharedCacheMap->Trace = FALSE; -#endif - KeReleaseSpinLockFromDpcLevel(&SharedCacheMap->CacheMapLock); - KeReleaseQueuedSpinLock(LockQueueMasterLock, *OldIrql); - if(SharedCacheMap->Section) - ObDereferenceObject(SharedCacheMap->Section); - ObDereferenceObject(SharedCacheMap->FileObject); - - while (!IsListEmpty(&FreeList)) - { - ULONG Refs; - - current_entry = RemoveTailList(&FreeList); - current = CONTAINING_RECORD(current_entry, ROS_VACB, CacheMapVacbListEntry); - InitializeListHead(¤t->CacheMapVacbListEntry); - Refs = CcRosVacbDecRefCount(current); + RefCount = CcRosVacbDecRefCount(Vacb); #if DBG // CORE-14578 - if (Refs != 0) + if (RefCount != 0) + { + DPRINT1("Leaking VACB %p attached to %p (%I64d)\n", Vacb, FileObject, Vacb->FileOffset.QuadPart); + DPRINT1("There are: %d references left\n", RefCount); + DPRINT1("Map: %d\n", Vacb->MappedCount); + DPRINT1("Dirty: %d\n", Vacb->Dirty); + if (FileObject->FileName.Length != 0) { - DPRINT1("Leaking VACB %p attached to %p (%I64d)\n", current, FileObject, current->FileOffset.QuadPart); - DPRINT1("There are: %d references left\n", Refs); - DPRINT1("Map: %d\n", current->MappedCount); - DPRINT1("Dirty: %d\n", current->Dirty); - if (FileObject->FileName.Length != 0) - { - DPRINT1("File was: %wZ\n", &FileObject->FileName); - } - else if (FileObject->FsContext != NULL && - ((PFSRTL_COMMON_FCB_HEADER)(FileObject->FsContext))->NodeTypeCode == 0x0502 && - ((PFSRTL_COMMON_FCB_HEADER)(FileObject->FsContext))->NodeByteSize == 0x1F8 && - ((PUNICODE_STRING)(((PUCHAR)FileObject->FsContext) + 0x100))->Length != 0) - { - DPRINT1("File was: %wZ (FastFAT)\n", (PUNICODE_STRING)(((PUCHAR)FileObject->FsContext) + 0x100)); - } - else - { - DPRINT1("No name for the file\n"); - } + DPRINT1("File was: %wZ\n", &FileObject->FileName); + } + else + { + DPRINT1("No name for the file\n"); } -#else - ASSERT(Refs == 0); -#endif } - - *OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); - RemoveEntryList(&SharedCacheMap->SharedCacheMapLinks); - KeReleaseQueuedSpinLock(LockQueueMasterLock, *OldIrql); - - ExFreeToNPagedLookasideList(&SharedCacheMapLookasideList, SharedCacheMap); - *OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); +#else + (void)RefCount; +#endif } + + /* Release the references we own */ + if(SharedCacheMap->Section) + ObDereferenceObject(SharedCacheMap->Section); + ObDereferenceObject(SharedCacheMap->FileObject); + + ExFreeToNPagedLookasideList(&SharedCacheMapLookasideList, SharedCacheMap); + + /* Acquire the lock again for our caller */ + *OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); + return STATUS_SUCCESS; } From 02167f2fb3f144d259f4279003ede2e01e2105b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Wed, 30 Dec 2020 18:25:02 +0100 Subject: [PATCH 70/99] [NTOS:CC] Properly return the public BCBs --- ntoskrnl/cc/cacheman.c | 2 +- ntoskrnl/cc/fs.c | 2 +- ntoskrnl/cc/pin.c | 24 ++++++++++++------------ 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/ntoskrnl/cc/cacheman.c b/ntoskrnl/cc/cacheman.c index 6086360db25..fa519d43e7a 100644 --- a/ntoskrnl/cc/cacheman.c +++ b/ntoskrnl/cc/cacheman.c @@ -308,7 +308,7 @@ CcSetBcbOwnerPointer ( IN PVOID Owner ) { - PINTERNAL_BCB iBcb = Bcb; + PINTERNAL_BCB iBcb = CONTAINING_RECORD(Bcb, INTERNAL_BCB, PFCB); CCTRACE(CC_API_DEBUG, "Bcb=%p Owner=%p\n", Bcb, Owner); diff --git a/ntoskrnl/cc/fs.c b/ntoskrnl/cc/fs.c index 5a74ecee48c..7a4c8cf4df2 100644 --- a/ntoskrnl/cc/fs.c +++ b/ntoskrnl/cc/fs.c @@ -45,7 +45,7 @@ NTAPI CcGetFileObjectFromBcb ( IN PVOID Bcb) { - PINTERNAL_BCB iBcb = (PINTERNAL_BCB)Bcb; + PINTERNAL_BCB iBcb = CONTAINING_RECORD(Bcb, INTERNAL_BCB, PFCB); CCTRACE(CC_API_DEBUG, "Bcb=%p\n", Bcb); diff --git a/ntoskrnl/cc/pin.c b/ntoskrnl/cc/pin.c index 30cba0a248a..12b5f87018b 100644 --- a/ntoskrnl/cc/pin.c +++ b/ntoskrnl/cc/pin.c @@ -126,8 +126,8 @@ CcpGetAppropriateBcb( } RtlZeroMemory(iBcb, sizeof(*iBcb)); - iBcb->PFCB.NodeTypeCode = 0xDE45; /* Undocumented (CAPTIVE_PUBLIC_BCB_NODETYPECODE) */ - iBcb->PFCB.NodeByteSize = sizeof(PUBLIC_BCB); + iBcb->PFCB.NodeTypeCode = 0x2FD; /* As per KMTests */ + iBcb->PFCB.NodeByteSize = 0; iBcb->PFCB.MappedLength = Length; iBcb->PFCB.MappedFileOffset = *FileOffset; iBcb->Vacb = Vacb; @@ -308,8 +308,9 @@ CcpPinData( } _SEH2_END; - *Bcb = NewBcb; + *Bcb = &NewBcb->PFCB; *Buffer = (PVOID)((ULONG_PTR)NewBcb->Vacb->BaseAddress + VacbOffset); + return TRUE; } @@ -411,7 +412,7 @@ CcMapData ( } _SEH2_END; - *pBcb = iBcb; + *pBcb = &iBcb->PFCB; *pBuffer = (PVOID)((ULONG_PTR)iBcb->Vacb->BaseAddress + VacbOffset); CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> TRUE Bcb=%p, Buffer %p\n", @@ -451,14 +452,14 @@ CcPinMappedData ( return FALSE; } - iBcb = *Bcb; + iBcb = *Bcb ? CONTAINING_RECORD(*Bcb, INTERNAL_BCB, PFCB) : NULL; ++CcPinMappedDataCount; Result = CcpPinData(SharedCacheMap, FileOffset, Length, Flags, Bcb, &Buffer); if (Result) { - CcUnpinData(iBcb); + CcUnpinData(&iBcb->PFCB); } return Result; @@ -542,10 +543,9 @@ CcSetDirtyPinnedData ( IN PVOID Bcb, IN PLARGE_INTEGER Lsn) { - PINTERNAL_BCB iBcb = Bcb; + PINTERNAL_BCB iBcb = CONTAINING_RECORD(Bcb, INTERNAL_BCB, PFCB); - CCTRACE(CC_API_DEBUG, "Bcb=%p Lsn=%p\n", - Bcb, Lsn); + CCTRACE(CC_API_DEBUG, "Bcb=%p Lsn=%p\n", Bcb, Lsn); /* Tell Mm */ MmMakePagesDirty(NULL, @@ -580,7 +580,7 @@ CcUnpinDataForThread ( IN PVOID Bcb, IN ERESOURCE_THREAD ResourceThreadId) { - PINTERNAL_BCB iBcb = Bcb; + PINTERNAL_BCB iBcb = CONTAINING_RECORD(Bcb, INTERNAL_BCB, PFCB); CCTRACE(CC_API_DEBUG, "Bcb=%p ResourceThreadId=%lu\n", Bcb, ResourceThreadId); @@ -601,7 +601,7 @@ NTAPI CcRepinBcb ( IN PVOID Bcb) { - PINTERNAL_BCB iBcb = Bcb; + PINTERNAL_BCB iBcb = CONTAINING_RECORD(Bcb, INTERNAL_BCB, PFCB); CCTRACE(CC_API_DEBUG, "Bcb=%p\n", Bcb); @@ -618,7 +618,7 @@ CcUnpinRepinnedBcb ( IN BOOLEAN WriteThrough, IN PIO_STATUS_BLOCK IoStatus) { - PINTERNAL_BCB iBcb = Bcb; + PINTERNAL_BCB iBcb = CONTAINING_RECORD(Bcb, INTERNAL_BCB, PFCB); KIRQL OldIrql; PROS_SHARED_CACHE_MAP SharedCacheMap; From e46f1a9af37ffd0fd9a99293c66f23acb0990856 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Mon, 4 Jan 2021 18:26:18 +0100 Subject: [PATCH 71/99] [NTOS:MM] Disable APC while performing Paged Read --- ntoskrnl/mm/section.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index 1c7fbdbefc6..b30462094c7 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -1187,6 +1187,7 @@ MiReadPage(PMEMORY_AREA MemoryArea, PMDL Mdl = (PMDL)MdlBase; PFILE_OBJECT FileObject = MemoryArea->SectionData.Segment->FileObject; LARGE_INTEGER FileOffset; + KIRQL OldIrql; FileOffset.QuadPart = MemoryArea->SectionData.Segment->Image.FileOffset + SegOffset; @@ -1203,6 +1204,9 @@ MiReadPage(PMEMORY_AREA MemoryArea, KeInitializeEvent(&Event, NotificationEvent, FALSE); + /* Disable APCs */ + KeRaiseIrql(APC_LEVEL, &OldIrql); + Status = IoPageRead(FileObject, Mdl, &FileOffset, &Event, &IoStatus); if (Status == STATUS_PENDING) { @@ -1215,6 +1219,8 @@ MiReadPage(PMEMORY_AREA MemoryArea, MmUnmapLockedPages (Mdl->MappedSystemVa, Mdl); } + KeLowerIrql(OldIrql); + if (Status == STATUS_END_OF_FILE) { DPRINT1("Got STATUS_END_OF_FILE at offset %I64d for file %wZ.\n", SegOffset, &FileObject->FileName); From 9b6240ee03eef58f1113bc6897269d379f68141d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Tue, 5 Jan 2021 10:41:41 +0100 Subject: [PATCH 72/99] [NTOS:CC] Get rid of ROS_VACB:Valid --- ntoskrnl/cc/copy.c | 14 +++++++------- ntoskrnl/cc/pin.c | 10 ++++------ ntoskrnl/cc/view.c | 23 ++++++----------------- ntoskrnl/include/internal/cc.h | 3 --- 4 files changed, 17 insertions(+), 33 deletions(-) diff --git a/ntoskrnl/cc/copy.c b/ntoskrnl/cc/copy.c index 502dfb03ee8..677ab00dd2f 100644 --- a/ntoskrnl/cc/copy.c +++ b/ntoskrnl/cc/copy.c @@ -227,12 +227,12 @@ CcPerformReadAhead( CurrentOffset % VACB_MAPPING_GRANULARITY, PartialLength); if (!NT_SUCCESS(Status)) { - CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE, FALSE); + CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE); DPRINT1("Failed to read data: %lx!\n", Status); goto Clear; } - CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); + CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE); Length -= PartialLength; CurrentOffset += PartialLength; @@ -254,12 +254,12 @@ CcPerformReadAhead( Status = CcRosEnsureVacbResident(Vacb, TRUE, FALSE, 0, PartialLength); if (!NT_SUCCESS(Status)) { - CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE, FALSE); + CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE); DPRINT1("Failed to read data: %lx!\n", Status); goto Clear; } - CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); + CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE); Length -= PartialLength; CurrentOffset += PartialLength; @@ -548,7 +548,7 @@ CcCopyRead ( } _SEH2_FINALLY { - CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); + CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE); } _SEH2_END; } @@ -653,7 +653,7 @@ CcCopyWrite ( _SEH2_FINALLY { /* Do not mark the VACB as dirty if an exception was raised */ - CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, !_SEH2_AbnormalTermination(), FALSE); + CcRosReleaseVacb(SharedCacheMap, Vacb, !_SEH2_AbnormalTermination(), FALSE); } _SEH2_END; } @@ -905,7 +905,7 @@ CcZeroData ( _SEH2_FINALLY { /* Do not mark the VACB as dirty if an exception was raised */ - CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, !_SEH2_AbnormalTermination(), FALSE); + CcRosReleaseVacb(SharedCacheMap, Vacb, !_SEH2_AbnormalTermination(), FALSE); } _SEH2_END; } diff --git a/ntoskrnl/cc/pin.c b/ntoskrnl/cc/pin.c index 12b5f87018b..ae62d2da0bc 100644 --- a/ntoskrnl/cc/pin.c +++ b/ntoskrnl/cc/pin.c @@ -91,7 +91,6 @@ CcpDereferenceBcb( */ CcRosReleaseVacb(SharedCacheMap, Bcb->Vacb, - TRUE, FALSE, FALSE); @@ -121,7 +120,7 @@ CcpGetAppropriateBcb( iBcb = ExAllocateFromNPagedLookasideList(&iBcbLookasideList); if (iBcb == NULL) { - CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); + CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE); return NULL; } @@ -172,7 +171,7 @@ CcpGetAppropriateBcb( if (DupBcb != NULL) { /* Delete the loser */ - CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); + CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE); ExDeleteResourceLite(&iBcb->Lock); ExFreeToNPagedLookasideList(&iBcbLookasideList, iBcb); } @@ -282,7 +281,7 @@ CcpPinData( NewBcb = CcpGetAppropriateBcb(SharedCacheMap, Vacb, FileOffset, Length, Flags, TRUE); if (NewBcb == NULL) { - CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); + CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE); return FALSE; } } @@ -383,7 +382,7 @@ CcMapData ( iBcb = CcpGetAppropriateBcb(SharedCacheMap, Vacb, FileOffset, Length, 0, FALSE); if (iBcb == NULL) { - CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); + CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE); CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> FALSE\n", SharedCacheMap->FileObject, FileOffset, Length, Flags); return FALSE; @@ -660,7 +659,6 @@ CcUnpinRepinnedBcb ( */ CcRosReleaseVacb(iBcb->Vacb->SharedCacheMap, iBcb->Vacb, - TRUE, FALSE, FALSE); diff --git a/ntoskrnl/cc/view.c b/ntoskrnl/cc/view.c index b6b7ec57560..1bd86b2cab7 100644 --- a/ntoskrnl/cc/view.c +++ b/ntoskrnl/cc/view.c @@ -342,17 +342,13 @@ NTAPI CcRosReleaseVacb ( PROS_SHARED_CACHE_MAP SharedCacheMap, PROS_VACB Vacb, - BOOLEAN Valid, BOOLEAN Dirty, BOOLEAN Mapped) { ULONG Refs; ASSERT(SharedCacheMap); - DPRINT("CcRosReleaseVacb(SharedCacheMap 0x%p, Vacb 0x%p, Valid %u)\n", - SharedCacheMap, Vacb, Valid); - - Vacb->Valid = Valid; + DPRINT("CcRosReleaseVacb(SharedCacheMap 0x%p, Vacb 0x%p)\n", SharedCacheMap, Vacb); if (Dirty && !Vacb->Dirty) { @@ -595,7 +591,6 @@ CcRosCreateVacb ( current = ExAllocateFromNPagedLookasideList(&VacbLookasideList); current->BaseAddress = NULL; - current->Valid = FALSE; current->Dirty = FALSE; current->PageOut = FALSE; current->FileOffset.QuadPart = ROUND_DOWN(FileOffset, VACB_MAPPING_GRANULARITY); @@ -947,7 +942,7 @@ CcFlushCache ( } } - CcRosReleaseVacb(SharedCacheMap, current, current->Valid, FALSE, FALSE); + CcRosReleaseVacb(SharedCacheMap, current, FALSE, FALSE); } Offset.QuadPart += VACB_MAPPING_GRANULARITY; @@ -1371,14 +1366,14 @@ ExpKdbgExtFileCache(ULONG Argc, PCHAR Argv[]) UNICODE_STRING NoName = RTL_CONSTANT_STRING(L"No name for File"); KdbpPrint(" Usage Summary (in kb)\n"); - KdbpPrint("Shared\t\tValid\tDirty\tName\n"); + KdbpPrint("Shared\t\tMapped\tDirty\tName\n"); /* No need to lock the spin lock here, we're in DBG */ for (ListEntry = CcCleanSharedCacheMapList.Flink; ListEntry != &CcCleanSharedCacheMapList; ListEntry = ListEntry->Flink) { PLIST_ENTRY Vacbs; - ULONG Valid = 0, Dirty = 0; + ULONG Mapped = 0, Dirty = 0; PROS_SHARED_CACHE_MAP SharedCacheMap; PUNICODE_STRING FileName; PWSTR Extra = L""; @@ -1393,13 +1388,7 @@ ExpKdbgExtFileCache(ULONG Argc, PCHAR Argv[]) Vacbs != &SharedCacheMap->CacheMapVacbListHead; Vacbs = Vacbs->Flink) { - PROS_VACB Vacb; - - Vacb = CONTAINING_RECORD(Vacbs, ROS_VACB, CacheMapVacbListEntry); - if (Vacb->Valid) - { - Valid += VACB_MAPPING_GRANULARITY / 1024; - } + Mapped += VACB_MAPPING_GRANULARITY / 1024; } /* Setup name */ @@ -1423,7 +1412,7 @@ ExpKdbgExtFileCache(ULONG Argc, PCHAR Argv[]) } /* And print */ - KdbpPrint("%p\t%d\t%d\t%wZ%S\n", SharedCacheMap, Valid, Dirty, FileName, Extra); + KdbpPrint("%p\t%d\t%d\t%wZ%S\n", SharedCacheMap, Mapped, Dirty, FileName, Extra); } return TRUE; diff --git a/ntoskrnl/include/internal/cc.h b/ntoskrnl/include/internal/cc.h index 32692c01db0..6416b2e7d3d 100644 --- a/ntoskrnl/include/internal/cc.h +++ b/ntoskrnl/include/internal/cc.h @@ -206,8 +206,6 @@ typedef struct _ROS_VACB { /* Base address of the region where the view's data is mapped. */ PVOID BaseAddress; - /* Are the contents of the view valid. */ - BOOLEAN Valid; /* Are the contents of the view newer than those on disk. */ BOOLEAN Dirty; /* Page out in progress */ @@ -387,7 +385,6 @@ NTAPI CcRosReleaseVacb( PROS_SHARED_CACHE_MAP SharedCacheMap, PROS_VACB Vacb, - BOOLEAN Valid, BOOLEAN Dirty, BOOLEAN Mapped ); From 20fe42c9e92093996cbfafbc046ad037b0f07c0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Tue, 5 Jan 2021 10:45:39 +0100 Subject: [PATCH 73/99] [NTOS:CC] Simplify CcFlushCache implementation --- ntoskrnl/cc/view.c | 100 +++++++++++++++++++++++++-------------------- 1 file changed, 55 insertions(+), 45 deletions(-) diff --git a/ntoskrnl/cc/view.c b/ntoskrnl/cc/view.c index 1bd86b2cab7..2aeb9df5b3b 100644 --- a/ntoskrnl/cc/view.c +++ b/ntoskrnl/cc/view.c @@ -899,62 +899,72 @@ CcFlushCache ( OUT PIO_STATUS_BLOCK IoStatus) { PROS_SHARED_CACHE_MAP SharedCacheMap; - LARGE_INTEGER Offset; - LONGLONG RemainingLength; - PROS_VACB current; + LONGLONG FlushStart, FlushEnd; NTSTATUS Status; CCTRACE(CC_API_DEBUG, "SectionObjectPointers=%p FileOffset=0x%I64X Length=%lu\n", SectionObjectPointers, FileOffset ? FileOffset->QuadPart : 0LL, Length); - if (SectionObjectPointers && SectionObjectPointers->SharedCacheMap) + if (!SectionObjectPointers || !SectionObjectPointers->SharedCacheMap) { - SharedCacheMap = SectionObjectPointers->SharedCacheMap; - ASSERT(SharedCacheMap); - if (FileOffset) - { - Offset = *FileOffset; - RemainingLength = Length; - } - else - { - Offset.QuadPart = 0; - RemainingLength = SharedCacheMap->FileSize.QuadPart; - } + Status = STATUS_INVALID_PARAMETER; + goto quit; + } - if (IoStatus) - { - IoStatus->Status = STATUS_SUCCESS; - IoStatus->Information = 0; - } - - while (RemainingLength > 0) - { - current = CcRosLookupVacb(SharedCacheMap, Offset.QuadPart); - if (current != NULL) - { - if (current->Dirty) - { - Status = CcRosFlushVacb(current); - if (!NT_SUCCESS(Status) && IoStatus != NULL) - { - IoStatus->Status = Status; - } - } - - CcRosReleaseVacb(SharedCacheMap, current, FALSE, FALSE); - } - - Offset.QuadPart += VACB_MAPPING_GRANULARITY; - RemainingLength -= min(RemainingLength, VACB_MAPPING_GRANULARITY); - } + SharedCacheMap = SectionObjectPointers->SharedCacheMap; + ASSERT(SharedCacheMap); + if (FileOffset) + { + FlushStart = FileOffset->QuadPart; + Status = RtlLongLongAdd(FlushStart, Length, &FlushEnd); + if (!NT_SUCCESS(Status)) + goto quit; } else { - if (IoStatus) + FlushStart = 0; + FlushEnd = SharedCacheMap->FileSize.QuadPart; + } + + Status = STATUS_SUCCESS; + + if (IoStatus) + { + IoStatus->Information = 0; + } + + while (FlushStart < FlushEnd) + { + PROS_VACB vacb = CcRosLookupVacb(SharedCacheMap, FlushStart); + + if (vacb != NULL) { - IoStatus->Status = STATUS_INVALID_PARAMETER; + if (vacb->Dirty) + { + Status = CcRosFlushVacb(vacb); + if (!NT_SUCCESS(Status)) + { + goto quit; + } + } + + CcRosReleaseVacb(SharedCacheMap, vacb, FALSE, FALSE); + + if (IoStatus) + IoStatus->Information += VACB_MAPPING_GRANULARITY; } + + if (!NT_SUCCESS(RtlLongLongAdd(FlushStart, VACB_MAPPING_GRANULARITY, &FlushStart))) + { + /* We're at the end of file ! */ + break; + } + } + +quit: + if (IoStatus) + { + IoStatus->Status = Status; } } @@ -1188,7 +1198,7 @@ CcRosInitializeFileCache ( FileObject->SectionObjectPointer->SharedCacheMap = SharedCacheMap; - // CcRosTraceCacheMap(SharedCacheMap, TRUE); + //CcRosTraceCacheMap(SharedCacheMap, TRUE); } else if (SharedCacheMap->Flags & SHARED_CACHE_MAP_IN_CREATION) { From 804f5a41edd0af35e881ea69a1595220587777c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Tue, 5 Jan 2021 12:32:09 +0100 Subject: [PATCH 74/99] [NTOS:CC] Improve trace messages --- ntoskrnl/cc/view.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/ntoskrnl/cc/view.c b/ntoskrnl/cc/view.c index 2aeb9df5b3b..dab292bd273 100644 --- a/ntoskrnl/cc/view.c +++ b/ntoskrnl/cc/view.c @@ -138,8 +138,8 @@ CcRosTraceCacheMap ( current = CONTAINING_RECORD(current_entry, ROS_VACB, CacheMapVacbListEntry); current_entry = current_entry->Flink; - DPRINT1(" VACB 0x%p enabled, RefCount %lu, Dirty %u, PageOut %lu\n", - current, current->ReferenceCount, current->Dirty, current->PageOut ); + DPRINT1(" VACB 0x%p enabled, RefCount %lu, Dirty %u, PageOut %lu, BaseAddress %p, FileOffset %I64d\n", + current, current->ReferenceCount, current->Dirty, current->PageOut, current->BaseAddress, current->FileOffset.QuadPart); } KeReleaseSpinLockFromDpcLevel(&SharedCacheMap->CacheMapLock); @@ -595,12 +595,6 @@ CcRosCreateVacb ( current->PageOut = FALSE; current->FileOffset.QuadPart = ROUND_DOWN(FileOffset, VACB_MAPPING_GRANULARITY); current->SharedCacheMap = SharedCacheMap; -#if DBG - if (SharedCacheMap->Trace) - { - DPRINT1("CacheMap 0x%p: new VACB: 0x%p\n", SharedCacheMap, current); - } -#endif current->MappedCount = 0; current->ReferenceCount = 0; InitializeListHead(¤t->CacheMapVacbListEntry); @@ -633,6 +627,14 @@ Retry: return Status; } +#if DBG + if (SharedCacheMap->Trace) + { + DPRINT1("CacheMap 0x%p: new VACB: 0x%p, file offset %I64d, BaseAddress %p\n", + SharedCacheMap, current, current->FileOffset.QuadPart, current->BaseAddress); + } +#endif + oldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); *Vacb = current; From bdb73edab7d444cdb5e49d1a166ccd7d0a721bf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Tue, 5 Jan 2021 12:32:26 +0100 Subject: [PATCH 75/99] [NTOS:CC] Flush the whole VACB Let Mm know what it has to do. --- ntoskrnl/cc/view.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ntoskrnl/cc/view.c b/ntoskrnl/cc/view.c index dab292bd273..21fd37ad681 100644 --- a/ntoskrnl/cc/view.c +++ b/ntoskrnl/cc/view.c @@ -169,8 +169,7 @@ CcRosFlushVacb ( PROS_VACB Vacb) { IO_STATUS_BLOCK Iosb; - SIZE_T FlushSize = min(VACB_MAPPING_GRANULARITY, - Vacb->SharedCacheMap->SectionSize.QuadPart - Vacb->FileOffset.QuadPart); + SIZE_T FlushSize = VACB_MAPPING_GRANULARITY; NTSTATUS Status; BOOLEAN HaveLock = FALSE; @@ -1024,7 +1023,7 @@ CcRosDeleteFileCache ( /* Flush to disk, if needed */ if (Vacb->Dirty) { - SIZE_T FlushSize = min(VACB_MAPPING_GRANULARITY, Vacb->SharedCacheMap->SectionSize.QuadPart - Vacb->FileOffset.QuadPart); + SIZE_T FlushSize = VACB_MAPPING_GRANULARITY; IO_STATUS_BLOCK Iosb; NTSTATUS Status; From 5949d5095d95d1ab5118f24f00193ded2670b825 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Tue, 5 Jan 2021 18:39:55 +0100 Subject: [PATCH 76/99] [NTOS:CC][NTOS:MM] Try respecting ValidDataLength --- ntoskrnl/cc/copy.c | 38 ++++++++++++++++++++++++++++++++-- ntoskrnl/cc/fs.c | 1 + ntoskrnl/cc/pin.c | 5 +++++ ntoskrnl/cc/view.c | 1 + ntoskrnl/include/internal/cc.h | 1 + ntoskrnl/mm/section.c | 15 +++++++++++--- 6 files changed, 56 insertions(+), 5 deletions(-) diff --git a/ntoskrnl/cc/copy.c b/ntoskrnl/cc/copy.c index 677ab00dd2f..8ed24f2bd92 100644 --- a/ntoskrnl/cc/copy.c +++ b/ntoskrnl/cc/copy.c @@ -504,6 +504,24 @@ CcCopyRead ( CurrentOffset = FileOffset->QuadPart; while(CurrentOffset < ReadEnd) { + if (CurrentOffset >= SharedCacheMap->ValidDataLength.QuadPart) + { + DPRINT1("Zeroing buffer because we are beyond the VDL.\n"); + /* We are beyond what is valid. Just zero this out */ + _SEH2_TRY + { + RtlZeroMemory(Buffer, Length); + } + _SEH2_EXCEPT(CcpCheckInvalidUserBuffer(_SEH2_GetExceptionInformation(), Buffer, Length)) + { + ExRaiseStatus(STATUS_INVALID_USER_BUFFER); + } + _SEH2_END; + + ReadLength += Length; + break; + } + Status = CcRosGetVacb(SharedCacheMap, CurrentOffset, &Vacb); if (!NT_SUCCESS(Status)) { @@ -598,7 +616,7 @@ CcCopyWrite ( PROS_SHARED_CACHE_MAP SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; NTSTATUS Status; LONGLONG CurrentOffset; - LONGLONG WriteEnd = FileOffset->QuadPart + Length; + LONGLONG WriteEnd; CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%I64d Length=%lu Wait=%d Buffer=%p\n", FileObject, FileOffset->QuadPart, Length, Wait, Buffer); @@ -610,7 +628,11 @@ CcCopyWrite ( if (!SharedCacheMap) return FALSE; - ASSERT((FileOffset->QuadPart + Length) <= SharedCacheMap->SectionSize.QuadPart); + Status = RtlLongLongAdd(FileOffset->QuadPart, Length, &WriteEnd); + if (!NT_SUCCESS(Status)) + ExRaiseStatus(Status); + + ASSERT(WriteEnd <= SharedCacheMap->SectionSize.QuadPart); CurrentOffset = FileOffset->QuadPart; while(CurrentOffset < WriteEnd) @@ -662,6 +684,10 @@ CcCopyWrite ( if (FileObject->Flags & FO_WRITE_THROUGH) CcFlushCache(FileObject->SectionObjectPointer, FileOffset, Length, NULL); + /* Update VDL */ + if (WriteEnd > SharedCacheMap->ValidDataLength.QuadPart) + SharedCacheMap->ValidDataLength.QuadPart = WriteEnd; + return TRUE; } @@ -871,6 +897,14 @@ CcZeroData ( return TRUE; } + /* See if we should simply truncate the valid data length */ + if ((StartOffset->QuadPart < SharedCacheMap->ValidDataLength.QuadPart) && (EndOffset->QuadPart > SharedCacheMap->ValidDataLength.QuadPart)) + { + DPRINT1("Truncating VDL.\n"); + SharedCacheMap->ValidDataLength = *StartOffset; + return TRUE; + } + ASSERT(EndOffset->QuadPart <= SharedCacheMap->SectionSize.QuadPart); while(WriteOffset.QuadPart < EndOffset->QuadPart) diff --git a/ntoskrnl/cc/fs.c b/ntoskrnl/cc/fs.c index 7a4c8cf4df2..161833d14e2 100644 --- a/ntoskrnl/cc/fs.c +++ b/ntoskrnl/cc/fs.c @@ -297,6 +297,7 @@ CcSetFileSizes ( OldSectionSize = SharedCacheMap->SectionSize; SharedCacheMap->SectionSize = FileSizes->AllocationSize; SharedCacheMap->FileSize = FileSizes->FileSize; + SharedCacheMap->ValidDataLength = FileSizes->ValidDataLength; KeReleaseSpinLock(&SharedCacheMap->CacheMapLock, OldIrql); if (FileSizes->AllocationSize.QuadPart < OldSectionSize.QuadPart) diff --git a/ntoskrnl/cc/pin.c b/ntoskrnl/cc/pin.c index ae62d2da0bc..07bfd63241c 100644 --- a/ntoskrnl/cc/pin.c +++ b/ntoskrnl/cc/pin.c @@ -543,6 +543,7 @@ CcSetDirtyPinnedData ( IN PLARGE_INTEGER Lsn) { PINTERNAL_BCB iBcb = CONTAINING_RECORD(Bcb, INTERNAL_BCB, PFCB); + PROS_SHARED_CACHE_MAP SharedCacheMap = iBcb->Vacb->SharedCacheMap; CCTRACE(CC_API_DEBUG, "Bcb=%p Lsn=%p\n", Bcb, Lsn); @@ -555,6 +556,10 @@ CcSetDirtyPinnedData ( { CcRosMarkDirtyVacb(iBcb->Vacb); } + + /* Update VDL */ + if (SharedCacheMap->ValidDataLength.QuadPart < (iBcb->PFCB.MappedFileOffset.QuadPart + iBcb->PFCB.MappedLength)) + SharedCacheMap->ValidDataLength.QuadPart = iBcb->PFCB.MappedFileOffset.QuadPart + iBcb->PFCB.MappedLength; } diff --git a/ntoskrnl/cc/view.c b/ntoskrnl/cc/view.c index 21fd37ad681..06ec86788a9 100644 --- a/ntoskrnl/cc/view.c +++ b/ntoskrnl/cc/view.c @@ -1182,6 +1182,7 @@ CcRosInitializeFileCache ( SharedCacheMap->LazyWriteContext = LazyWriterContext; SharedCacheMap->SectionSize = FileSizes->AllocationSize; SharedCacheMap->FileSize = FileSizes->FileSize; + SharedCacheMap->ValidDataLength = FileSizes->ValidDataLength; SharedCacheMap->PinAccess = PinAccess; SharedCacheMap->DirtyPageThreshold = 0; SharedCacheMap->DirtyPages = 0; diff --git a/ntoskrnl/include/internal/cc.h b/ntoskrnl/include/internal/cc.h index 6416b2e7d3d..7384d9be0d3 100644 --- a/ntoskrnl/include/internal/cc.h +++ b/ntoskrnl/include/internal/cc.h @@ -175,6 +175,7 @@ typedef struct _ROS_SHARED_CACHE_MAP LARGE_INTEGER FileSize; LIST_ENTRY BcbList; LARGE_INTEGER SectionSize; + LARGE_INTEGER ValidDataLength; PFILE_OBJECT FileObject; ULONG DirtyPages; LIST_ENTRY SharedCacheMapLinks; diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index b30462094c7..7ff5485ae81 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -1167,11 +1167,13 @@ MiCopyFromUserPage(PFN_NUMBER DestPage, const VOID *SrcAddress) } #ifndef NEWCC +static NTSTATUS NTAPI MiReadPage(PMEMORY_AREA MemoryArea, LONGLONG SegOffset, - PPFN_NUMBER Page) + PPFN_NUMBER Page, + BOOLEAN IgnoreSize) /* * FUNCTION: Read a page for a section backed memory area. * PARAMETERS: @@ -1188,6 +1190,7 @@ MiReadPage(PMEMORY_AREA MemoryArea, PFILE_OBJECT FileObject = MemoryArea->SectionData.Segment->FileObject; LARGE_INTEGER FileOffset; KIRQL OldIrql; + PFSRTL_ADVANCED_FCB_HEADER FcbHeader = FileObject->FsContext; FileOffset.QuadPart = MemoryArea->SectionData.Segment->Image.FileOffset + SegOffset; @@ -1197,6 +1200,12 @@ MiReadPage(PMEMORY_AREA MemoryArea, if (!NT_SUCCESS(Status)) return Status; + if ((FileOffset.QuadPart > FcbHeader->ValidDataLength.QuadPart) && !IgnoreSize) + { + /* Quick path : data is not valid; return a zero-page */ + return STATUS_SUCCESS; + } + RtlZeroMemory(MdlBase, sizeof(MdlBase)); MmInitializeMdl(Mdl, NULL, PAGE_SIZE); MmBuildMdlFromPages(Mdl, Page); @@ -1616,7 +1625,7 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, else { DPRINT("Getting fresh page for file %wZ at offset %I64d.\n", &Segment->FileObject->FileName, Offset.QuadPart); - Status = MiReadPage(MemoryArea, Offset.QuadPart, &Page); + Status = MiReadPage(MemoryArea, Offset.QuadPart, &Page, FALSE); if (!NT_SUCCESS(Status)) { DPRINT1("MiReadPage failed (Status %x)\n", Status); @@ -4592,7 +4601,7 @@ MmMakePagesResident( /* FIXME: Read the whole range at once instead of one page at a time */ /* Ignore file size, as Cc already checked on its side. */ - Status = MiReadPage(MemoryArea, SegmentOffset.QuadPart, &Page); + Status = MiReadPage(MemoryArea, SegmentOffset.QuadPart, &Page, TRUE); if (!NT_SUCCESS(Status)) { /* Reset the Segment entry and fail */ From 48e1c8593f97b278ce02a78cd3002368ee92d907 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Thu, 21 Jan 2021 16:39:27 +0100 Subject: [PATCH 77/99] [NTOS:PO] Flush dirty pages after flushing volumes --- ntoskrnl/po/power.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ntoskrnl/po/power.c b/ntoskrnl/po/power.c index 6dee7054c02..125edbc11df 100644 --- a/ntoskrnl/po/power.c +++ b/ntoskrnl/po/power.c @@ -1072,18 +1072,19 @@ NtSetSystemPowerState(IN POWER_ACTION SystemAction, /* Check if we're still in an invalid status */ if (!NT_SUCCESS(Status)) break; + /* Flush all volumes and the registry */ + DPRINT("Flushing volumes\n"); + PopFlushVolumes(PopAction.Shutdown); + #ifndef NEWCC /* Flush dirty cache pages */ /* XXX: Is that still mandatory? As now we'll wait on lazy writer to complete? */ - CcRosFlushDirtyPages(MAXULONG, &Dummy, TRUE, FALSE); //HACK: We really should wait here! + CcRosFlushDirtyPages(MAXULONG, &Dummy, TRUE, FALSE); + DPRINT("Cache flushed %lu pages\n", Dummy); #else Dummy = 0; #endif - /* Flush all volumes and the registry */ - DPRINT("Flushing volumes, cache flushed %lu pages\n", Dummy); - PopFlushVolumes(PopAction.Shutdown); - /* Set IRP for drivers */ PopAction.IrpMinor = IRP_MN_SET_POWER; if (PopAction.Shutdown) From 727e73f253a665cb8a74b2144d884d4c03241677 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Mon, 25 Jan 2021 09:23:46 +0100 Subject: [PATCH 78/99] |NTOS:MM] Simplify some code --- ntoskrnl/cache/section/sptab.c | 47 ++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/ntoskrnl/cache/section/sptab.c b/ntoskrnl/cache/section/sptab.c index 7a2481a4752..f0cb3d36c2a 100644 --- a/ntoskrnl/cache/section/sptab.c +++ b/ntoskrnl/cache/section/sptab.c @@ -204,27 +204,30 @@ _MmSetPageEntrySectionSegment(PMM_SECTION_SEGMENT Segment, OldEntry, Entry); - if (PFN_FROM_SSE(Entry) == PFN_FROM_SSE(OldEntry)) { - /* Nothing */ - } else if (Entry && !IS_SWAP_FROM_SSE(Entry)) { - ASSERT(!OldEntry || IS_SWAP_FROM_SSE(OldEntry)); - MmSetSectionAssociation(PFN_FROM_SSE(Entry), Segment, Offset); - } else if (OldEntry && !IS_SWAP_FROM_SSE(OldEntry)) { - ASSERT(!Entry || IS_SWAP_FROM_SSE(Entry)); - MmDeleteSectionAssociation(PFN_FROM_SSE(OldEntry)); - } else if (IS_SWAP_FROM_SSE(Entry)) { - ASSERT(!IS_SWAP_FROM_SSE(OldEntry) || - SWAPENTRY_FROM_SSE(OldEntry) == MM_WAIT_ENTRY); - if (OldEntry && SWAPENTRY_FROM_SSE(OldEntry) != MM_WAIT_ENTRY) - MmDeleteSectionAssociation(PFN_FROM_SSE(OldEntry)); - } else if (IS_SWAP_FROM_SSE(OldEntry)) { - ASSERT(!IS_SWAP_FROM_SSE(Entry)); - if (Entry) - MmSetSectionAssociation(PFN_FROM_SSE(OldEntry), Segment, Offset); - } else { - /* We should not be replacing a page like this */ - ASSERT(FALSE); + if (Entry && !IS_SWAP_FROM_SSE(Entry)) + { + /* We have a valid entry. See if we must do something */ + if (OldEntry && !IS_SWAP_FROM_SSE(OldEntry)) + { + /* The previous entry was valid. Shall we swap the Rmaps ? */ + if (PFN_FROM_SSE(Entry) != PFN_FROM_SSE(OldEntry)) + { + MmDeleteSectionAssociation(PFN_FROM_SSE(OldEntry)); + MmSetSectionAssociation(PFN_FROM_SSE(Entry), Segment, Offset); + } + } + else + { + /* We're switching to a valid entry from an invalid one. Add the Rmap */ + MmSetSectionAssociation(PFN_FROM_SSE(Entry), Segment, Offset); + } } + else if (OldEntry && !IS_SWAP_FROM_SSE(OldEntry)) + { + /* We're switching to an invalid entry from a valid one */ + MmDeleteSectionAssociation(PFN_FROM_SSE(OldEntry)); + } + PageTable->PageEntries[PageIndex] = Entry; return STATUS_SUCCESS; } @@ -332,13 +335,13 @@ MmGetSectionAssociation(PFN_NUMBER Page, PMM_SECTION_SEGMENT Segment = NULL; PCACHE_SECTION_PAGE_TABLE PageTable; - PageTable = (PCACHE_SECTION_PAGE_TABLE)MmGetSegmentRmap(Page, - &RawOffset); + PageTable = MmGetSegmentRmap(Page, &RawOffset); if (PageTable) { Segment = PageTable->Segment; Offset->QuadPart = PageTable->FileOffset.QuadPart + ((ULONG64)RawOffset << PAGE_SHIFT); + ASSERT(PFN_FROM_SSE(PageTable->PageEntries[RawOffset]) == Page); } return Segment; From f201b8af788349cc081b68d0868cfff4df3ece1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Mon, 25 Jan 2021 09:24:54 +0100 Subject: [PATCH 79/99] [NTOS:MM] Do not acquire Pfn lock in MmGetRmapListHead All callers already hold it --- ntoskrnl/mm/freelist.c | 15 ++++----------- ntoskrnl/mm/rmap.c | 4 ++++ 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/ntoskrnl/mm/freelist.c b/ntoskrnl/mm/freelist.c index ba16362f546..5cc22ce38b1 100644 --- a/ntoskrnl/mm/freelist.c +++ b/ntoskrnl/mm/freelist.c @@ -426,12 +426,10 @@ PMM_RMAP_ENTRY NTAPI MmGetRmapListHeadPage(PFN_NUMBER Pfn) { - KIRQL oldIrql; - PMM_RMAP_ENTRY ListHead; PMMPFN Pfn1; - /* Lock PFN database */ - oldIrql = MiAcquirePfnLock(); + /* PFN database must be locked */ + MI_ASSERT_PFN_LOCK_HELD(); /* Get the entry */ Pfn1 = MiGetPfnEntry(Pfn); @@ -439,19 +437,14 @@ MmGetRmapListHeadPage(PFN_NUMBER Pfn) if (!MI_IS_ROS_PFN(Pfn1)) { - MiReleasePfnLock(oldIrql); return NULL; } - /* Get the list head */ - ListHead = Pfn1->RmapListHead; - /* Should not have an RMAP for a non-active page */ ASSERT(MiIsPfnInUse(Pfn1) == TRUE); - /* Release PFN database and return rmap list head */ - MiReleasePfnLock(oldIrql); - return ListHead; + /* Get the list head */ + return Pfn1->RmapListHead; } VOID diff --git a/ntoskrnl/mm/rmap.c b/ntoskrnl/mm/rmap.c index 785539ea58d..f50e680e401 100644 --- a/ntoskrnl/mm/rmap.c +++ b/ntoskrnl/mm/rmap.c @@ -250,7 +250,11 @@ MmPageOutPhysicalAddress(PFN_NUMBER Page) /* We can finally let this page go */ MmDeleteRmap(Page, Process, Address); +#if DBG + OldIrql = MiAcquirePfnLock(); ASSERT(MmGetRmapListHeadPage(Page) == NULL); + MiReleasePfnLock(OldIrql); +#endif MmReleasePageMemoryConsumer(MC_USER, Page); if (Address < MmSystemRangeStart) From d23dbdda4cf1d30a7d518cc1c01bb4daebc33b7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Mon, 25 Jan 2021 09:43:50 +0100 Subject: [PATCH 80/99] [NTOS:MM] Remove unused field --- ntoskrnl/include/internal/mm.h | 1 - 1 file changed, 1 deletion(-) diff --git a/ntoskrnl/include/internal/mm.h b/ntoskrnl/include/internal/mm.h index 05f89c50cb9..4db21f72199 100644 --- a/ntoskrnl/include/internal/mm.h +++ b/ntoskrnl/include/internal/mm.h @@ -185,7 +185,6 @@ typedef struct _MM_SECTION_SEGMENT ULONG RefCount; ULONG SegFlags; - LIST_ENTRY ListOfSegments; RTL_GENERIC_TABLE PageTable; } MM_SECTION_SEGMENT, *PMM_SECTION_SEGMENT; From 3efc5b17347ad614cd807535625fbf12ca82a8e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Tue, 26 Jan 2021 11:21:08 +0100 Subject: [PATCH 81/99] [NTOS:MM] Fix unlocking on failure path --- ntoskrnl/mm/ARM3/section.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ntoskrnl/mm/ARM3/section.c b/ntoskrnl/mm/ARM3/section.c index fddaf752031..aeccb2adf28 100644 --- a/ntoskrnl/mm/ARM3/section.c +++ b/ntoskrnl/mm/ARM3/section.c @@ -1243,6 +1243,8 @@ MiLoadUserSymbols(IN PCONTROL_AREA ControlArea, _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER) { ExFreePoolWithTag(LdrEntry, 'bDmM'); + ExReleaseResourceLite(&PsLoadedModuleResource); + KeLeaveCriticalRegion(); _SEH2_YIELD(return); } _SEH2_END; From bde2ee571a408088995127ef654e88c9512786bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Wed, 27 Jan 2021 09:22:32 +0100 Subject: [PATCH 82/99] [NTOS:FSRTL] Properly leave FsRtl when returning STATUS_CANT_WAIT --- ntoskrnl/fsrtl/fastio.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ntoskrnl/fsrtl/fastio.c b/ntoskrnl/fsrtl/fastio.c index a056b67e109..ce629f77a02 100644 --- a/ntoskrnl/fsrtl/fastio.c +++ b/ntoskrnl/fsrtl/fastio.c @@ -1714,9 +1714,13 @@ FsRtlAcquireFileForCcFlushEx(IN PFILE_OBJECT FileObject) /* Return either success or inability to wait. In case of other failure - fall through */ - if (Status == STATUS_SUCCESS || - Status == STATUS_CANT_WAIT) + if (NT_SUCCESS(Status)) + return Status; + + if (Status == STATUS_CANT_WAIT) { + DPRINT1("STATUS_CANT_WAIT\n"); + FsRtlExitFileSystem(); return Status; } } From 8a8b4db447c36232addbe4693953182d355c0408 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Wed, 27 Jan 2021 10:29:07 +0100 Subject: [PATCH 83/99] [NTOS:MM] Make the page LRU list a real LRU list. Also, implement flushing mapped sections to disk on shutdown. --- ntoskrnl/cache/section/newmm.h | 57 ----------- ntoskrnl/include/internal/mm.h | 69 +++++++++++-- ntoskrnl/mm/balance.c | 27 ++---- ntoskrnl/mm/freelist.c | 170 +++++++++++++++++++++------------ ntoskrnl/mm/mminit.c | 3 - ntoskrnl/mm/shutdown.c | 38 ++++++++ 6 files changed, 217 insertions(+), 147 deletions(-) diff --git a/ntoskrnl/cache/section/newmm.h b/ntoskrnl/cache/section/newmm.h index edff99f3d85..3fb731c6ace 100644 --- a/ntoskrnl/cache/section/newmm.h +++ b/ntoskrnl/cache/section/newmm.h @@ -3,22 +3,6 @@ #include /* TYPES *********************************************************************/ - -#define PFN_FROM_SSE(E) ((PFN_NUMBER)((E) >> PAGE_SHIFT)) -#define IS_SWAP_FROM_SSE(E) ((E) & 0x00000001) -#define MM_IS_WAIT_PTE(E) \ - (IS_SWAP_FROM_SSE(E) && SWAPENTRY_FROM_SSE(E) == MM_WAIT_ENTRY) -#define MAKE_PFN_SSE(P) ((ULONG_PTR)((P) << PAGE_SHIFT)) -#define SWAPENTRY_FROM_SSE(E) ((E) >> 1) -#define MAKE_SWAP_SSE(S) (((ULONG_PTR)(S) << 1) | 0x1) -#define DIRTY_SSE(E) ((E) | 2) -#define CLEAN_SSE(E) ((E) & ~2) -#define IS_DIRTY_SSE(E) ((E) & 2) -#define PAGE_FROM_SSE(E) ((E) & 0xFFFFF000) -#define SHARE_COUNT_FROM_SSE(E) (((E) & 0x00000FFC) >> 2) -#define MAX_SHARE_COUNT 0x3FF -#define MAKE_SSE(P, C) ((ULONG_PTR)((P) | ((C) << 2))) - #define MM_SEGMENT_FINALIZE (0x40000000) #define RMAP_SEGMENT_MASK ~((ULONG_PTR)0xff) @@ -123,25 +107,6 @@ VOID NTAPI MiInitializeSectionPageTable(PMM_SECTION_SEGMENT Segment); -NTSTATUS -NTAPI -_MmSetPageEntrySectionSegment(PMM_SECTION_SEGMENT Segment, - PLARGE_INTEGER Offset, - ULONG_PTR Entry, - const char *file, - int line); - -ULONG_PTR -NTAPI -_MmGetPageEntrySectionSegment(PMM_SECTION_SEGMENT Segment, - PLARGE_INTEGER Offset, - const char *file, - int line); - -#define MmSetPageEntrySectionSegment(S,O,E) _MmSetPageEntrySectionSegment(S,O,E,__FILE__,__LINE__) - -#define MmGetPageEntrySectionSegment(S,O) _MmGetPageEntrySectionSegment(S,O,__FILE__,__LINE__) - typedef VOID (NTAPI *FREE_SECTION_PAGE_FUN)( PMM_SECTION_SEGMENT Segment, PLARGE_INTEGER Offset); @@ -151,12 +116,6 @@ NTAPI MmFreePageTablesSectionSegment(PMM_SECTION_SEGMENT Segment, FREE_SECTION_PAGE_FUN FreePage); -/* Yields a lock */ -PMM_SECTION_SEGMENT -NTAPI -MmGetSectionAssociation(PFN_NUMBER Page, - PLARGE_INTEGER Offset); - NTSTATUS NTAPI MmSetSectionAssociation(PFN_NUMBER Page, @@ -267,22 +226,6 @@ MmPageOutDeleteMapping(PVOID Context, PEPROCESS Process, PVOID Address); -VOID -NTAPI -_MmLockSectionSegment(PMM_SECTION_SEGMENT Segment, - const char *file, - int line); - -#define MmLockSectionSegment(x) _MmLockSectionSegment(x,__FILE__,__LINE__) - -VOID -NTAPI -_MmUnlockSectionSegment(PMM_SECTION_SEGMENT Segment, - const char *file, - int line); - -#define MmUnlockSectionSegment(x) _MmUnlockSectionSegment(x,__FILE__,__LINE__) - VOID MmFreeCacheSectionPage(PVOID Context, MEMORY_AREA* MemoryArea, diff --git a/ntoskrnl/include/internal/mm.h b/ntoskrnl/include/internal/mm.h index 4db21f72199..bd9ea25e0b7 100644 --- a/ntoskrnl/include/internal/mm.h +++ b/ntoskrnl/include/internal/mm.h @@ -357,6 +357,8 @@ typedef struct _MMPFN // HACK until WS lists are supported MMWSLE Wsle; + struct _MMPFN* NextLRU; + struct _MMPFN* PreviousLRU; } MMPFN, *PMMPFN; extern PMMPFN MmPfnDatabase; @@ -877,6 +879,11 @@ NTSTATUS NTAPI MmPageOutPhysicalAddress(PFN_NUMBER Page); +PMM_SECTION_SEGMENT +NTAPI +MmGetSectionAssociation(PFN_NUMBER Page, + PLARGE_INTEGER Offset); + /* freelist.c **********************************************************/ FORCEINLINE @@ -950,20 +957,12 @@ MiGetPfnEntryIndex(IN PMMPFN Pfn1) PFN_NUMBER NTAPI -MmGetLRUNextUserPage(PFN_NUMBER PreviousPage); +MmGetLRUNextUserPage(PFN_NUMBER PreviousPage, BOOLEAN MoveToLast); PFN_NUMBER NTAPI MmGetLRUFirstUserPage(VOID); -VOID -NTAPI -MmInsertLRULastUserPage(PFN_NUMBER Page); - -VOID -NTAPI -MmRemoveLRUUserPage(PFN_NUMBER Page); - VOID NTAPI MmDumpArmPfnDatabase( @@ -1232,6 +1231,37 @@ MmFindRegion( /* section.c *****************************************************************/ +#define PFN_FROM_SSE(E) ((PFN_NUMBER)((E) >> PAGE_SHIFT)) +#define IS_SWAP_FROM_SSE(E) ((E) & 0x00000001) +#define MM_IS_WAIT_PTE(E) \ + (IS_SWAP_FROM_SSE(E) && SWAPENTRY_FROM_SSE(E) == MM_WAIT_ENTRY) +#define MAKE_PFN_SSE(P) ((ULONG_PTR)((P) << PAGE_SHIFT)) +#define SWAPENTRY_FROM_SSE(E) ((E) >> 1) +#define MAKE_SWAP_SSE(S) (((ULONG_PTR)(S) << 1) | 0x1) +#define DIRTY_SSE(E) ((E) | 2) +#define CLEAN_SSE(E) ((E) & ~2) +#define IS_DIRTY_SSE(E) ((E) & 2) +#define PAGE_FROM_SSE(E) ((E) & 0xFFFFF000) +#define SHARE_COUNT_FROM_SSE(E) (((E) & 0x00000FFC) >> 2) +#define MAX_SHARE_COUNT 0x3FF +#define MAKE_SSE(P, C) ((ULONG_PTR)((P) | ((C) << 2))) + +VOID +NTAPI +_MmLockSectionSegment(PMM_SECTION_SEGMENT Segment, + const char *file, + int line); + +#define MmLockSectionSegment(x) _MmLockSectionSegment(x,__FILE__,__LINE__) + +VOID +NTAPI +_MmUnlockSectionSegment(PMM_SECTION_SEGMENT Segment, + const char *file, + int line); + +#define MmUnlockSectionSegment(x) _MmUnlockSectionSegment(x,__FILE__,__LINE__) + VOID NTAPI MmGetImageInformation( @@ -1372,6 +1402,27 @@ MmExtendSection( _In_ PVOID Section, _Inout_ PLARGE_INTEGER NewSize); +/* sptab.c *******************************************************************/ + +NTSTATUS +NTAPI +_MmSetPageEntrySectionSegment(PMM_SECTION_SEGMENT Segment, + PLARGE_INTEGER Offset, + ULONG_PTR Entry, + const char *file, + int line); + +ULONG_PTR +NTAPI +_MmGetPageEntrySectionSegment(PMM_SECTION_SEGMENT Segment, + PLARGE_INTEGER Offset, + const char *file, + int line); + +#define MmSetPageEntrySectionSegment(S,O,E) _MmSetPageEntrySectionSegment(S,O,E,__FILE__,__LINE__) + +#define MmGetPageEntrySectionSegment(S,O) _MmGetPageEntrySectionSegment(S,O,__FILE__,__LINE__) + /* sysldr.c ******************************************************************/ VOID diff --git a/ntoskrnl/mm/balance.c b/ntoskrnl/mm/balance.c index 34dddf5dbfa..f9067c58a13 100644 --- a/ntoskrnl/mm/balance.c +++ b/ntoskrnl/mm/balance.c @@ -80,11 +80,7 @@ MmReleasePageMemoryConsumer(ULONG Consumer, PFN_NUMBER Page) KeBugCheck(MEMORY_MANAGEMENT); } - if (MmGetReferenceCountPage(Page) == 1) - { - if(Consumer == MC_USER) MmRemoveLRUUserPage(Page); - (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed); - } + (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed); MmDereferencePage(Page); @@ -142,7 +138,6 @@ NTSTATUS MmTrimUserMemory(ULONG Target, ULONG Priority, PULONG NrFreedPages) { PFN_NUMBER CurrentPage; - PFN_NUMBER NextPage; NTSTATUS Status; (*NrFreedPages) = 0; @@ -158,13 +153,14 @@ MmTrimUserMemory(ULONG Target, ULONG Priority, PULONG NrFreedPages) (*NrFreedPages)++; } - NextPage = MmGetLRUNextUserPage(CurrentPage); - if (NextPage <= CurrentPage) - { - /* We wrapped around, so we're done */ - break; - } - CurrentPage = NextPage; + CurrentPage = MmGetLRUNextUserPage(CurrentPage, TRUE); + } + + if (CurrentPage) + { + KIRQL OldIrql = MiAcquirePfnLock(); + MmDereferencePage(CurrentPage); + MiReleasePfnLock(OldIrql); } return STATUS_SUCCESS; @@ -209,14 +205,13 @@ MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait, /* * Allocate always memory for the non paged pool and for the pager thread. */ - if ((Consumer == MC_SYSTEM) /* || MiIsBalancerThread() */) + if (Consumer == MC_SYSTEM) { Page = MmAllocPage(Consumer); if (Page == 0) { KeBugCheck(NO_PAGES_AVAILABLE); } - if (Consumer == MC_USER) MmInsertLRULastUserPage(Page); *AllocatedPage = Page; if (MmAvailablePages < MiMinimumAvailablePages) MmRebalanceMemoryConsumers(); @@ -257,7 +252,6 @@ MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait, KeBugCheck(NO_PAGES_AVAILABLE); } - if(Consumer == MC_USER) MmInsertLRULastUserPage(Page); *AllocatedPage = Page; if (MmAvailablePages < MiMinimumAvailablePages) @@ -276,7 +270,6 @@ MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait, { KeBugCheck(NO_PAGES_AVAILABLE); } - if(Consumer == MC_USER) MmInsertLRULastUserPage(Page); *AllocatedPage = Page; if (MmAvailablePages < MiMinimumAvailablePages) diff --git a/ntoskrnl/mm/freelist.c b/ntoskrnl/mm/freelist.c index 5cc22ce38b1..35c6336dcc6 100644 --- a/ntoskrnl/mm/freelist.c +++ b/ntoskrnl/mm/freelist.c @@ -35,95 +35,127 @@ SIZE_T MmPagedPoolCommit; SIZE_T MmPeakCommitment; SIZE_T MmtotalCommitLimitMaximum; -static RTL_BITMAP MiUserPfnBitMap; +PMMPFN FirstUserLRUPfn; +PMMPFN LastUserLRUPfn; /* FUNCTIONS *************************************************************/ -VOID -NTAPI -MiInitializeUserPfnBitmap(VOID) -{ - PVOID Bitmap; - - /* Allocate enough buffer for the PFN bitmap and align it on 32-bits */ - Bitmap = ExAllocatePoolWithTag(NonPagedPool, - (((MmHighestPhysicalPage + 1) + 31) / 32) * 4, - TAG_MM); - ASSERT(Bitmap); - - /* Initialize it and clear all the bits to begin with */ - RtlInitializeBitMap(&MiUserPfnBitMap, - Bitmap, - (ULONG)MmHighestPhysicalPage + 1); - RtlClearAllBits(&MiUserPfnBitMap); -} - PFN_NUMBER NTAPI MmGetLRUFirstUserPage(VOID) { - ULONG Position; + PFN_NUMBER Page; KIRQL OldIrql; /* Find the first user page */ OldIrql = MiAcquirePfnLock(); - Position = RtlFindSetBits(&MiUserPfnBitMap, 1, 0); - MiReleasePfnLock(OldIrql); - if (Position == 0xFFFFFFFF) return 0; - /* Return it */ - ASSERT(Position != 0); - ASSERT_IS_ROS_PFN(MiGetPfnEntry(Position)); - return Position; + if (FirstUserLRUPfn == NULL) + { + MiReleasePfnLock(OldIrql); + return 0; + } + + Page = MiGetPfnEntryIndex(FirstUserLRUPfn); + MmReferencePage(Page); + + MiReleasePfnLock(OldIrql); + + return Page; } +static VOID -NTAPI -MmInsertLRULastUserPage(PFN_NUMBER Pfn) +MmInsertLRULastUserPage(PFN_NUMBER Page) { - KIRQL OldIrql; + MI_ASSERT_PFN_LOCK_HELD(); - /* Set the page as a user page */ - ASSERT(Pfn != 0); - ASSERT_IS_ROS_PFN(MiGetPfnEntry(Pfn)); - ASSERT(!RtlCheckBit(&MiUserPfnBitMap, (ULONG)Pfn)); - OldIrql = MiAcquirePfnLock(); - RtlSetBit(&MiUserPfnBitMap, (ULONG)Pfn); - MiReleasePfnLock(OldIrql); + PMMPFN Pfn = MiGetPfnEntry(Page); + + if (FirstUserLRUPfn == NULL) + FirstUserLRUPfn = Pfn; + + Pfn->PreviousLRU = LastUserLRUPfn; + + if (LastUserLRUPfn != NULL) + LastUserLRUPfn->NextLRU = Pfn; + LastUserLRUPfn = Pfn; +} + +static +VOID +MmRemoveLRUUserPage(PFN_NUMBER Page) +{ + MI_ASSERT_PFN_LOCK_HELD(); + + /* Unset the page as a user page */ + ASSERT(Page != 0); + + PMMPFN Pfn = MiGetPfnEntry(Page); + + ASSERT_IS_ROS_PFN(Pfn); + + if (Pfn->PreviousLRU) + { + ASSERT(Pfn->PreviousLRU->NextLRU == Pfn); + Pfn->PreviousLRU->NextLRU = Pfn->NextLRU; + } + else + { + ASSERT(FirstUserLRUPfn == Pfn); + FirstUserLRUPfn = Pfn->NextLRU; + } + + if (Pfn->NextLRU) + { + ASSERT(Pfn->NextLRU->PreviousLRU == Pfn); + Pfn->NextLRU->PreviousLRU = Pfn->PreviousLRU; + } + else + { + ASSERT(Pfn == LastUserLRUPfn); + LastUserLRUPfn = Pfn->PreviousLRU; + } + + Pfn->PreviousLRU = Pfn->NextLRU = NULL; } PFN_NUMBER NTAPI -MmGetLRUNextUserPage(PFN_NUMBER PreviousPfn) +MmGetLRUNextUserPage(PFN_NUMBER PreviousPage, BOOLEAN MoveToLast) { - ULONG Position; + PFN_NUMBER Page = 0; KIRQL OldIrql; /* Find the next user page */ OldIrql = MiAcquirePfnLock(); - Position = RtlFindSetBits(&MiUserPfnBitMap, 1, (ULONG)PreviousPfn + 1); + + PMMPFN PreviousPfn = MiGetPfnEntry(PreviousPage); + PMMPFN NextPfn = PreviousPfn->NextLRU; + + /* + * Move this one at the end of the list. + * It may be freed by MmDereferencePage below. + * If it's not, then it means it is still hanging in some process address space. + * This avoids paging-out e.g. ntdll early just because it's mapped first time. + */ + if (MoveToLast) + { + MmRemoveLRUUserPage(PreviousPage); + MmInsertLRULastUserPage(PreviousPage); + } + + if (NextPfn) + { + Page = MiGetPfnEntryIndex(NextPfn); + MmReferencePage(Page); + } + + MmDereferencePage(PreviousPage); + MiReleasePfnLock(OldIrql); - if (Position == 0xFFFFFFFF) return 0; - /* Return it */ - ASSERT(Position != 0); - ASSERT_IS_ROS_PFN(MiGetPfnEntry(Position)); - return Position; -} - -VOID -NTAPI -MmRemoveLRUUserPage(PFN_NUMBER Page) -{ - KIRQL OldIrql; - - /* Unset the page as a user page */ - ASSERT(Page != 0); - ASSERT_IS_ROS_PFN(MiGetPfnEntry(Page)); - ASSERT(RtlCheckBit(&MiUserPfnBitMap, (ULONG)Page)); - OldIrql = MiAcquirePfnLock(); - RtlClearBit(&MiUserPfnBitMap, (ULONG)Page); - MiReleasePfnLock(OldIrql); + return Page; } BOOLEAN @@ -548,6 +580,13 @@ MmDereferencePage(PFN_NUMBER Pfn) Pfn1->u3.e2.ReferenceCount--; if (Pfn1->u3.e2.ReferenceCount == 0) { + /* Apply LRU hack */ + if (Pfn1->u4.MustBeCached) + { + MmRemoveLRUUserPage(Pfn); + Pfn1->u4.MustBeCached = 0; + } + /* Mark the page temporarily as valid, we're going to make it free soon */ Pfn1->u3.e1.PageLocation = ActiveAndValid; @@ -590,6 +629,15 @@ MmAllocPage(ULONG Type) Pfn1->u1.SwapEntry = 0; Pfn1->RmapListHead = NULL; + Pfn1->NextLRU = NULL; + Pfn1->PreviousLRU = NULL; + + if (Type == MC_USER) + { + Pfn1->u4.MustBeCached = 1; /* HACK again */ + MmInsertLRULastUserPage(PfnOffset); + } + MiReleasePfnLock(OldIrql); return PfnOffset; } diff --git a/ntoskrnl/mm/mminit.c b/ntoskrnl/mm/mminit.c index 6c1a5f274d6..3e58ad9bbc8 100644 --- a/ntoskrnl/mm/mminit.c +++ b/ntoskrnl/mm/mminit.c @@ -17,8 +17,6 @@ /* GLOBALS *******************************************************************/ -VOID NTAPI MiInitializeUserPfnBitmap(VOID); - BOOLEAN Mm64BitPhysicalAddress = FALSE; ULONG MmReadClusterSize; // @@ -235,7 +233,6 @@ MmInitSystem(IN ULONG Phase, MiDbgDumpAddressSpace(); MmInitGlobalKernelPageDirectory(); - MiInitializeUserPfnBitmap(); MmInitializeMemoryConsumer(MC_USER, MmTrimUserMemory); MmInitializeRmapList(); MmInitSectionImplementation(); diff --git a/ntoskrnl/mm/shutdown.c b/ntoskrnl/mm/shutdown.c index c80cc527a30..89beca88fac 100644 --- a/ntoskrnl/mm/shutdown.c +++ b/ntoskrnl/mm/shutdown.c @@ -21,6 +21,44 @@ VOID MiShutdownSystem(VOID) { ULONG i; + PFN_NUMBER Page; + BOOLEAN Dirty; + + /* Loop through all the pages owned by the legacy Mm and page them out, if needed. */ + /* We do it twice, since flushing can cause the FS to dirtify new pages */ + do + { + Dirty = FALSE; + + Page = MmGetLRUFirstUserPage(); + while (Page) + { + LARGE_INTEGER SegmentOffset; + PMM_SECTION_SEGMENT Segment = MmGetSectionAssociation(Page, &SegmentOffset); + + if (Segment) + { + if ((*Segment->Flags) & MM_DATAFILE_SEGMENT) + { + MmLockSectionSegment(Segment); + + ULONG_PTR Entry = MmGetPageEntrySectionSegment(Segment, &SegmentOffset); + + if (!IS_SWAP_FROM_SSE(Entry) && IS_DIRTY_SSE(Entry)) + { + Dirty = TRUE; + MmCheckDirtySegment(Segment, &SegmentOffset, FALSE, TRUE); + } + + MmUnlockSectionSegment(Segment); + } + + MmDereferenceSegment(Segment); + } + + Page = MmGetLRUNextUserPage(Page, FALSE); + } + } while (Dirty); /* Loop through all the paging files */ for (i = 0; i < MmNumberOfPagingFiles; i++) From 90c6a65efe7384269f27fe419e164868c8654fff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Wed, 27 Jan 2021 13:03:06 +0100 Subject: [PATCH 84/99] [NTOS:MM] Introduce MmPurgeSegment & MmFlushSegment Those will back CcFlushCache and CcPurgeCache. --- ntoskrnl/include/internal/mm.h | 25 +++- ntoskrnl/mm/section.c | 241 ++++++++++++++++++++++++++++++--- 2 files changed, 247 insertions(+), 19 deletions(-) diff --git a/ntoskrnl/include/internal/mm.h b/ntoskrnl/include/internal/mm.h index bd9ea25e0b7..dad1a08bf33 100644 --- a/ntoskrnl/include/internal/mm.h +++ b/ntoskrnl/include/internal/mm.h @@ -1241,10 +1241,14 @@ MmFindRegion( #define DIRTY_SSE(E) ((E) | 2) #define CLEAN_SSE(E) ((E) & ~2) #define IS_DIRTY_SSE(E) ((E) & 2) +#define WRITE_SSE(E) ((E) | 4) +#define IS_WRITE_SSE(E) ((E) & 4) #define PAGE_FROM_SSE(E) ((E) & 0xFFFFF000) -#define SHARE_COUNT_FROM_SSE(E) (((E) & 0x00000FFC) >> 2) -#define MAX_SHARE_COUNT 0x3FF -#define MAKE_SSE(P, C) ((ULONG_PTR)((P) | ((C) << 2))) +#define SHARE_COUNT_FROM_SSE(E) (((E) & 0x00000FFC) >> 3) +#define MAX_SHARE_COUNT 0x1FF +#define MAKE_SSE(P, C) ((ULONG_PTR)((P) | ((C) << 3))) +#define BUMPREF_SSE(E) (PAGE_FROM_SSE(E) | ((SHARE_COUNT_FROM_SSE(E) + 1) << 3) | ((E) & 0x7)) +#define DECREF_SSE(E) (PAGE_FROM_SSE(E) | ((SHARE_COUNT_FROM_SSE(E) - 1) << 3) | ((E) & 0x7)) VOID NTAPI @@ -1375,6 +1379,21 @@ MmRosFlushVirtualMemory( _Inout_ PSIZE_T Length, _Out_ PIO_STATUS_BLOCK Iosb); +NTSTATUS +NTAPI +MmFlushSegment( + _In_ PSECTION_OBJECT_POINTERS SectionObjectPointer, + _In_opt_ PLARGE_INTEGER Offset, + _In_ ULONG Length, + _In_opt_ PIO_STATUS_BLOCK Iosb); + +BOOLEAN +NTAPI +MmPurgeSegment( + _In_ PSECTION_OBJECT_POINTERS SectionObjectPointer, + _In_opt_ PLARGE_INTEGER Offset, + _In_ ULONG Length); + BOOLEAN NTAPI MmCheckDirtySegment( diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index 7ff5485ae81..a223ba0e302 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -60,6 +60,8 @@ extern MMSESSION MmSession; +static LARGE_INTEGER TinyTime = {{-1L, -1L}}; + #ifndef NEWCC KEVENT MmWaitPageEvent; @@ -83,6 +85,37 @@ _MmUnlockSectionSegment(PMM_SECTION_SEGMENT Segment, const char *file, int line) } #endif +static +PMM_SECTION_SEGMENT +MiGrabDataSection(PSECTION_OBJECT_POINTERS SectionObjectPointer) +{ + KIRQL OldIrql = MiAcquirePfnLock(); + PMM_SECTION_SEGMENT Segment = NULL; + + while (TRUE) + { + Segment = SectionObjectPointer->DataSectionObject; + if (!Segment) + break; + + if (Segment->SegFlags & (MM_SEGMENT_INCREATE | MM_SEGMENT_INDELETE)) + { + MiReleasePfnLock(OldIrql); + KeDelayExecutionThread(KernelMode, FALSE, &TinyTime); + OldIrql = MiAcquirePfnLock(); + continue; + } + + ASSERT(Segment->SegFlags & MM_DATAFILE_SEGMENT); + InterlockedIncrementUL(&Segment->RefCount); + break; + } + + MiReleasePfnLock(OldIrql); + + return Segment; +} + /* Somewhat grotesque, but eh... */ PMM_IMAGE_SECTION_OBJECT ImageSectionObjectFromSegment(PMM_SECTION_SEGMENT Segment) { @@ -1028,7 +1061,6 @@ MmSharePageEntrySectionSegment(PMM_SECTION_SEGMENT Segment, PLARGE_INTEGER Offset) { ULONG_PTR Entry; - BOOLEAN Dirty; Entry = MmGetPageEntrySectionSegment(Segment, Offset); if (Entry == 0) @@ -1045,11 +1077,7 @@ MmSharePageEntrySectionSegment(PMM_SECTION_SEGMENT Segment, { KeBugCheck(MEMORY_MANAGEMENT); } - Dirty = IS_DIRTY_SSE(Entry); - Entry = MAKE_SSE(PAGE_FROM_SSE(Entry), SHARE_COUNT_FROM_SSE(Entry) + 1); - if (Dirty) - Entry = DIRTY_SSE(Entry); - MmSetPageEntrySectionSegment(Segment, Offset, Entry); + MmSetPageEntrySectionSegment(Segment, Offset, BUMPREF_SSE(Entry)); } BOOLEAN @@ -1080,8 +1108,7 @@ MmUnsharePageEntrySectionSegment(PMEMORY_AREA MemoryArea, { KeBugCheck(MEMORY_MANAGEMENT); } - Dirty = Dirty || IS_DIRTY_SSE(Entry); - Entry = MAKE_SSE(PAGE_FROM_SSE(Entry), SHARE_COUNT_FROM_SSE(Entry) - 1); + Entry = DECREF_SSE(Entry); if (Dirty) Entry = DIRTY_SSE(Entry); if (SHARE_COUNT_FROM_SSE(Entry) > 0) @@ -1094,7 +1121,7 @@ MmUnsharePageEntrySectionSegment(PMEMORY_AREA MemoryArea, return FALSE; } - if (Dirty && (MemoryArea->VadNode.u.VadFlags.VadType != VadImageMap)) + if (IS_DIRTY_SSE(Entry) && (MemoryArea->VadNode.u.VadFlags.VadType != VadImageMap)) { ASSERT(!Segment->WriteCopy); ASSERT(MmGetSavedSwapEntryPage(Page) == 0); @@ -1105,10 +1132,10 @@ MmUnsharePageEntrySectionSegment(PMEMORY_AREA MemoryArea, } /* Only valid case for shared dirty pages is shared image section */ - ASSERT(!Dirty || (Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)); + ASSERT(!IS_DIRTY_SSE(Entry) || (Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)); SwapEntry = MmGetSavedSwapEntryPage(Page); - if (Dirty && !SwapEntry) + if (IS_DIRTY_SSE(Entry) && !SwapEntry) { SwapEntry = MmAllocSwapPage(); if (!SwapEntry) @@ -1119,7 +1146,7 @@ MmUnsharePageEntrySectionSegment(PMEMORY_AREA MemoryArea, } } - if (Dirty) + if (IS_DIRTY_SSE(Entry)) { NTSTATUS Status = MmWriteToSwapPage(SwapEntry, Page); if (!NT_SUCCESS(Status)) @@ -4703,6 +4730,185 @@ MmRosFlushVirtualMemory( return STATUS_SUCCESS; } +/* Like CcPurgeCache but for the in-memory segment */ +BOOLEAN +NTAPI +MmPurgeSegment( + _In_ PSECTION_OBJECT_POINTERS SectionObjectPointer, + _In_opt_ PLARGE_INTEGER Offset, + _In_ ULONG Length) +{ + LARGE_INTEGER PurgeStart, PurgeEnd; + PMM_SECTION_SEGMENT Segment; + + Segment = MiGrabDataSection(SectionObjectPointer); + if (!Segment) + { + /* Nothing to purge */ + return STATUS_SUCCESS; + } + + PurgeStart.QuadPart = Offset ? Offset->QuadPart : 0LL; + if (Length && Offset) + { + if (!NT_SUCCESS(RtlLongLongAdd(PurgeStart.QuadPart, Length, &PurgeEnd.QuadPart))) + return FALSE; + } + + MmLockSectionSegment(Segment); + + if (!Length || !Offset) + { + /* We must calculate the length for ourselves */ + /* FIXME: All of this is suboptimal */ + ULONG ElemCount = RtlNumberGenericTableElements(&Segment->PageTable); + /* No page. Nothing to purge */ + if (!ElemCount) + { + MmUnlockSectionSegment(Segment); + MmDereferenceSegment(Segment); + return TRUE; + } + + PCACHE_SECTION_PAGE_TABLE PageTable = RtlGetElementGenericTable(&Segment->PageTable, ElemCount - 1); + PurgeEnd.QuadPart = PageTable->FileOffset.QuadPart + _countof(PageTable->PageEntries) * PAGE_SIZE; + } + + while (PurgeStart.QuadPart < PurgeEnd.QuadPart) + { + ULONG_PTR Entry = MmGetPageEntrySectionSegment(Segment, &PurgeStart); + + if (Entry == 0) + { + PurgeStart.QuadPart += PAGE_SIZE; + continue; + } + + if (IS_SWAP_FROM_SSE(Entry)) + { + ASSERT(SWAPENTRY_FROM_SSE(Entry) == MM_WAIT_ENTRY); + /* The page is currently being read. Meaning someone will need it soon. Bad luck */ + MmUnlockSectionSegment(Segment); + MmDereferenceSegment(Segment); + return FALSE; + } + + if (IS_WRITE_SSE(Entry)) + { + /* We're trying to purge an entry which is being written. Restart this loop iteration */ + MmUnlockSectionSegment(Segment); + KeDelayExecutionThread(KernelMode, FALSE, &TinyTime); + MmLockSectionSegment(Segment); + continue; + } + + if (SHARE_COUNT_FROM_SSE(Entry) > 0) + { + /* This page is currently in use. Bad luck */ + MmUnlockSectionSegment(Segment); + MmDereferenceSegment(Segment); + return FALSE; + } + + /* We can let this page go */ + MmSetPageEntrySectionSegment(Segment, &PurgeStart, 0); + MmReleasePageMemoryConsumer(MC_USER, PFN_FROM_SSE(Entry)); + + PurgeStart.QuadPart += PAGE_SIZE; + } + + /* This page is currently in use. Bad luck */ + MmUnlockSectionSegment(Segment); + MmDereferenceSegment(Segment); + return TRUE; +} + +NTSTATUS +NTAPI +MmFlushSegment( + _In_ PSECTION_OBJECT_POINTERS SectionObjectPointer, + _In_opt_ PLARGE_INTEGER Offset, + _In_ ULONG Length, + _In_opt_ PIO_STATUS_BLOCK Iosb) +{ + LARGE_INTEGER FlushStart, FlushEnd; + NTSTATUS Status; + + if (Offset) + { + FlushStart = *Offset; + Status = RtlLongLongAdd(FlushStart.QuadPart, Length, &FlushEnd.QuadPart); + if (!NT_SUCCESS(Status)) + return Status; + } + + if (Iosb) + Iosb->Information = 0; + + PMM_SECTION_SEGMENT Segment = MiGrabDataSection(SectionObjectPointer); + if (!Segment) + { + /* Nothing to flush */ + if (Iosb) + Iosb->Status = STATUS_SUCCESS; + return STATUS_SUCCESS; + } + + ASSERT(*Segment->Flags & MM_DATAFILE_SEGMENT); + + MmLockSectionSegment(Segment); + + if (!Offset) + { + FlushStart.QuadPart = 0; + + /* FIXME: All of this is suboptimal */ + ULONG ElemCount = RtlNumberGenericTableElements(&Segment->PageTable); + /* No page. Nothing to flush */ + if (!ElemCount) + { + MmUnlockSectionSegment(Segment); + MmDereferenceSegment(Segment); + if (Iosb) + { + Iosb->Status = STATUS_SUCCESS; + Iosb->Information = 0; + } + return STATUS_SUCCESS; + } + + PCACHE_SECTION_PAGE_TABLE PageTable = RtlGetElementGenericTable(&Segment->PageTable, ElemCount - 1); + FlushEnd.QuadPart = PageTable->FileOffset.QuadPart + _countof(PageTable->PageEntries) * PAGE_SIZE; + } + + FlushStart.QuadPart >>= PAGE_SHIFT; + FlushStart.QuadPart <<= PAGE_SHIFT; + + while (FlushStart.QuadPart < FlushEnd.QuadPart) + { + ULONG_PTR Entry = MmGetPageEntrySectionSegment(Segment, &FlushStart); + + if (IS_DIRTY_SSE(Entry)) + { + MmCheckDirtySegment(Segment, &FlushStart, FALSE, FALSE); + + if (Iosb) + Iosb->Information += PAGE_SIZE; + } + + FlushStart.QuadPart += PAGE_SIZE; + } + + MmUnlockSectionSegment(Segment); + + MmDereferenceSegment(Segment); + + if (Iosb) + Iosb->Status = STATUS_SUCCESS; + + return STATUS_SUCCESS; +} + _Requires_exclusive_lock_held_(Segment->Lock) BOOLEAN NTAPI @@ -4718,6 +4924,8 @@ MmCheckDirtySegment( ASSERT(Segment->Locked); + ASSERT((Offset->QuadPart % PAGE_SIZE) == 0); + DPRINT("Checking segment for file %wZ at offset 0x%I64X.\n", &Segment->FileObject->FileName, Offset->QuadPart); Entry = MmGetPageEntrySectionSegment(Segment, Offset); @@ -4733,9 +4941,10 @@ MmCheckDirtySegment( ASSERT(!Segment->WriteCopy); ASSERT(Segment->SegFlags & MM_DATAFILE_SEGMENT); - /* Insert the cleaned entry back. Keep one ref to the page so nobody pages it out again behind us */ - MmSetPageEntrySectionSegment(Segment, Offset, - MAKE_SSE(Page << PAGE_SHIFT, SHARE_COUNT_FROM_SSE(Entry) + 1)); + /* Insert the cleaned entry back. Mark it as write in progress, and clear the dirty bit. */ + Entry = MAKE_SSE(PAGE_FROM_SSE(Entry), SHARE_COUNT_FROM_SSE(Entry) + 1); + Entry = WRITE_SSE(Entry); + MmSetPageEntrySectionSegment(Segment, Offset, Entry); /* Tell the other users that we are clean again */ MmSetCleanAllRmaps(Page); @@ -4772,7 +4981,7 @@ MmCheckDirtySegment( DirtyAgain = IS_DIRTY_SSE(Entry) || MmIsDirtyPageRmap(Page); } - /* Drop the reference we got */ + /* Drop the reference we got, deleting the write altogether. */ Entry = MAKE_SSE(Page << PAGE_SHIFT, SHARE_COUNT_FROM_SSE(Entry) - 1); if (DirtyAgain) { From d0bf98663b198696a936dc6f8c4921db282f4bee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Wed, 27 Jan 2021 15:29:57 +0100 Subject: [PATCH 85/99] [NTOS:CC] Be sure to flush the whole file in CcFlushCache --- ntoskrnl/cc/view.c | 44 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/ntoskrnl/cc/view.c b/ntoskrnl/cc/view.c index 06ec86788a9..591168eeba0 100644 --- a/ntoskrnl/cc/view.c +++ b/ntoskrnl/cc/view.c @@ -906,12 +906,19 @@ CcFlushCache ( CCTRACE(CC_API_DEBUG, "SectionObjectPointers=%p FileOffset=0x%I64X Length=%lu\n", SectionObjectPointers, FileOffset ? FileOffset->QuadPart : 0LL, Length); - if (!SectionObjectPointers || !SectionObjectPointers->SharedCacheMap) + if (!SectionObjectPointers) { Status = STATUS_INVALID_PARAMETER; goto quit; } + if (!SectionObjectPointers->SharedCacheMap) + { + /* Forward this to Mm */ + MmFlushSegment(SectionObjectPointers, FileOffset, Length, IoStatus); + return; + } + SharedCacheMap = SectionObjectPointers->SharedCacheMap; ASSERT(SharedCacheMap); if (FileOffset) @@ -934,8 +941,14 @@ CcFlushCache ( IoStatus->Information = 0; } + /* + * We flush the VACBs that we find here. + * If there is no (dirty) VACB, it doesn't mean that there is no data to flush, so we call Mm to be sure. + * This is suboptimal, but this is due to the lack of granularity of how we track dirty cache data + */ while (FlushStart < FlushEnd) { + BOOLEAN DirtyVacb = FALSE; PROS_VACB vacb = CcRosLookupVacb(SharedCacheMap, FlushStart); if (vacb != NULL) @@ -947,6 +960,7 @@ CcFlushCache ( { goto quit; } + DirtyVacb = TRUE; } CcRosReleaseVacb(SharedCacheMap, vacb, FALSE, FALSE); @@ -955,11 +969,39 @@ CcFlushCache ( IoStatus->Information += VACB_MAPPING_GRANULARITY; } + if (!DirtyVacb) + { + IO_STATUS_BLOCK MmIosb; + LARGE_INTEGER MmOffset; + + MmOffset.QuadPart = FlushStart; + + if (FlushEnd - (FlushEnd % VACB_MAPPING_GRANULARITY) <= FlushStart) + { + /* The whole range fits within a VACB chunk. */ + Status = MmFlushSegment(SectionObjectPointers, &MmOffset, FlushEnd - FlushStart, &MmIosb); + } + else + { + ULONG MmLength = VACB_MAPPING_GRANULARITY - (FlushStart % VACB_MAPPING_GRANULARITY); + Status = MmFlushSegment(SectionObjectPointers, &MmOffset, MmLength, &MmIosb); + } + + if (!NT_SUCCESS(Status)) + goto quit; + + if (IoStatus) + IoStatus->Information += MmIosb.Information; + } + if (!NT_SUCCESS(RtlLongLongAdd(FlushStart, VACB_MAPPING_GRANULARITY, &FlushStart))) { /* We're at the end of file ! */ break; } + + /* Round down to next VACB start now */ + FlushStart -= FlushStart % VACB_MAPPING_GRANULARITY; } quit: From cc6bc02b916cd2084c9aa8ef5efccfdc7f14d7a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Wed, 27 Jan 2021 16:19:07 +0100 Subject: [PATCH 86/99] [NTOS:CC] Make sure to purge the Mm side in CcPurgeCacheSection --- ntoskrnl/cc/fs.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/ntoskrnl/cc/fs.c b/ntoskrnl/cc/fs.c index 161833d14e2..bef6502f73f 100644 --- a/ntoskrnl/cc/fs.c +++ b/ntoskrnl/cc/fs.c @@ -180,7 +180,10 @@ CcPurgeCacheSection ( SharedCacheMap = SectionObjectPointer->SharedCacheMap; if (!SharedCacheMap) - return FALSE; + { + Success = TRUE; + goto purgeMm; + } StartOffset = FileOffset != NULL ? FileOffset->QuadPart : 0; if (Length == 0 || FileOffset == NULL) @@ -257,6 +260,11 @@ CcPurgeCacheSection ( ASSERT(Refs == 0); } + /* Now make sure that Mm doesn't hold some pages here. */ +purgeMm: + if (Success) + Success = MmPurgeSegment(SectionObjectPointer, FileOffset, Length); + return Success; } From 3e43aa1305fa2865c3e61647e5c51857c9a1c9fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Wed, 27 Jan 2021 16:20:08 +0100 Subject: [PATCH 87/99] [NTOS:MM] Use 64-bit integer to count references on segments --- ntoskrnl/include/internal/mm.h | 8 +++++--- ntoskrnl/mm/rmap.c | 2 +- ntoskrnl/mm/section.c | 25 +++---------------------- 3 files changed, 9 insertions(+), 26 deletions(-) diff --git a/ntoskrnl/include/internal/mm.h b/ntoskrnl/include/internal/mm.h index dad1a08bf33..86ed4dfb23a 100644 --- a/ntoskrnl/include/internal/mm.h +++ b/ntoskrnl/include/internal/mm.h @@ -168,7 +168,7 @@ typedef struct _MM_SECTION_SEGMENT FAST_MUTEX Lock; /* lock which protects the page directory */ LARGE_INTEGER RawLength; /* length of the segment which is part of the mapped file */ LARGE_INTEGER Length; /* absolute length of the segment */ - PULONG ReferenceCount; + PLONG64 ReferenceCount; ULONG SectionCount; ULONG Protection; PULONG Flags; @@ -182,9 +182,11 @@ typedef struct _MM_SECTION_SEGMENT ULONG Characteristics; } Image; - ULONG RefCount; + LONG64 RefCount; ULONG SegFlags; + ULONGLONG LastPage; + RTL_GENERIC_TABLE PageTable; } MM_SECTION_SEGMENT, *PMM_SECTION_SEGMENT; @@ -192,7 +194,7 @@ typedef struct _MM_IMAGE_SECTION_OBJECT { PFILE_OBJECT FileObject; - ULONG RefCount; + LONG64 RefCount; ULONG SegFlags; SECTION_IMAGE_INFORMATION ImageInformation; diff --git a/ntoskrnl/mm/rmap.c b/ntoskrnl/mm/rmap.c index f50e680e401..5b7ffe3a3da 100644 --- a/ntoskrnl/mm/rmap.c +++ b/ntoskrnl/mm/rmap.c @@ -525,7 +525,7 @@ MmGetSegmentRmap(PFN_NUMBER Page, PULONG RawOffset) return NULL; } - InterlockedIncrementUL(Result->Segment->ReferenceCount); + InterlockedIncrement64(Result->Segment->ReferenceCount); MiReleasePfnLock(OldIrql); return Result; } diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index a223ba0e302..524b282f606 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -107,7 +107,7 @@ MiGrabDataSection(PSECTION_OBJECT_POINTERS SectionObjectPointer) } ASSERT(Segment->SegFlags & MM_DATAFILE_SEGMENT); - InterlockedIncrementUL(&Segment->RefCount); + InterlockedIncrement64(&Segment->RefCount); break; } @@ -996,7 +996,7 @@ MmDereferenceSegment(PMM_SECTION_SEGMENT Segment) /* Lock the PFN lock because we mess around with SectionObjectPointers */ OldIrql = MiAcquirePfnLock(); - if (InterlockedDecrementUL(Segment->ReferenceCount) > 0) + if (InterlockedDecrement64(Segment->ReferenceCount) > 0) { /* Nothing to do yet */ MiReleasePfnLock(OldIrql); @@ -2049,25 +2049,6 @@ MmpDeleteSection(PVOID ObjectBody) ASSERT(ImageSectionObject->RefCount > 0); MmDereferenceSegment(ImageSectionObject->Segments); } -#ifdef NEWCC - else if (Section->Segment && Section->Segment->Flags & MM_DATAFILE_SEGMENT) - { - ULONG RefCount = 0; - PMM_SECTION_SEGMENT Segment = Section->Segment; - - if (Segment && - (RefCount = InterlockedDecrementUL(&Segment->ReferenceCount)) == 0) - { - DPRINT("Freeing section segment\n"); - Section->Segment = NULL; - MmFinalizeSegment(Segment); - } - else - { - DPRINT("RefCount %d\n", RefCount); - } - } -#endif else { PMM_SECTION_SEGMENT Segment = (PMM_SECTION_SEGMENT)Section->Segment; @@ -3339,7 +3320,7 @@ MmMapViewOfSegment( return(Status); } - InterlockedIncrementUL(Segment->ReferenceCount); + InterlockedIncrement64(Segment->ReferenceCount); MArea->SectionData.Segment = Segment; MArea->SectionData.ViewOffset.QuadPart = ViewOffset; From 91a4e62376d1dbd5d7e155f2a86bb4b468649a24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Wed, 27 Jan 2021 16:28:45 +0100 Subject: [PATCH 88/99] [NTOS:MM] Improve MmCanFileBeTruncated --- ntoskrnl/mm/section.c | 42 ++++++++++++------------------------------ 1 file changed, 12 insertions(+), 30 deletions(-) diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index 524b282f606..16767458bce 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -4077,54 +4077,36 @@ BOOLEAN NTAPI MmCanFileBeTruncated (IN PSECTION_OBJECT_POINTERS SectionObjectPointer, IN PLARGE_INTEGER NewFileSize) { - KIRQL OldIrql = MiAcquirePfnLock(); BOOLEAN Ret; PMM_SECTION_SEGMENT Segment; -CheckSectionPointer: /* Check whether an ImageSectionObject exists */ if (SectionObjectPointer->ImageSectionObject != NULL) { DPRINT1("ERROR: File can't be truncated because it has an image section\n"); - MiReleasePfnLock(OldIrql); - return FALSE; } - Segment = (PMM_SECTION_SEGMENT)SectionObjectPointer->DataSectionObject; - /* Wait for it to be created/deleted properly */ - while (Segment && (Segment->SegFlags & (MM_SEGMENT_INCREATE | MM_SEGMENT_INDELETE))) + Segment = MiGrabDataSection(SectionObjectPointer); + if (!Segment) { - LARGE_INTEGER ShortTime; - - ShortTime.QuadPart = -10 * 100 * 1000; - - /* Bad luck. Wait a bit for the operation to finish */ - MiReleasePfnLock(OldIrql); - KeDelayExecutionThread(KernelMode, FALSE, &ShortTime); - OldIrql = MiAcquirePfnLock(); - goto CheckSectionPointer; + /* There is no data section. It's fine to do anything. */ + return TRUE; } - if (Segment) + MmLockSectionSegment(Segment); + if ((Segment->SectionCount == 1) && (SectionObjectPointer->SharedCacheMap != NULL)) { - if ((Segment->SectionCount == 1) && (SectionObjectPointer->SharedCacheMap != NULL)) - { - /* If the cache is the only one holding a reference to the segment, then it's fine to resize */ - Ret = TRUE; - } - else - { - /* We can't shrink, but we can extend */ - Ret = NewFileSize->QuadPart >= Segment->RawLength.QuadPart; - } + /* If the cache is the only one holding a reference to the segment, then it's fine to resize */ + Ret = TRUE; } else { - Ret = TRUE; + /* We can't shrink, but we can extend */ + Ret = NewFileSize->QuadPart >= Segment->RawLength.QuadPart; } - - MiReleasePfnLock(OldIrql); + MmUnlockSectionSegment(Segment); + MmDereferenceSegment(Segment); DPRINT("FIXME: didn't check for outstanding write probes\n"); From c1c51e31e439240fec4542b002dae925ccfb9e3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Wed, 27 Jan 2021 16:59:37 +0100 Subject: [PATCH 89/99] [NTOS:MM] Keep a ref on the segment when mapping a page & keep trace of what is the highest offset mapped. --- ntoskrnl/cache/section/sptab.c | 23 ++++++++++++++++++++++- ntoskrnl/mm/section.c | 3 --- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/ntoskrnl/cache/section/sptab.c b/ntoskrnl/cache/section/sptab.c index f0cb3d36c2a..7d960a1ff08 100644 --- a/ntoskrnl/cache/section/sptab.c +++ b/ntoskrnl/cache/section/sptab.c @@ -218,14 +218,35 @@ _MmSetPageEntrySectionSegment(PMM_SECTION_SEGMENT Segment, } else { - /* We're switching to a valid entry from an invalid one. Add the Rmap */ + /* + * We're switching to a valid entry from an invalid one. + * Add the Rmap and take a ref on the segment. + */ MmSetSectionAssociation(PFN_FROM_SSE(Entry), Segment, Offset); + InterlockedIncrement64(Segment->ReferenceCount); + + if ((Offset->QuadPart > (Segment->LastPage << PAGE_SHIFT)) || !Segment->LastPage) + Segment->LastPage = (Offset->QuadPart >> PAGE_SHIFT) + 1; } } else if (OldEntry && !IS_SWAP_FROM_SSE(OldEntry)) { /* We're switching to an invalid entry from a valid one */ MmDeleteSectionAssociation(PFN_FROM_SSE(OldEntry)); + MmDereferenceSegment(Segment); + + if (Offset->QuadPart == ((Segment->LastPage - 1ULL) << PAGE_SHIFT)) + { + /* We are unsetting the last page */ + while (--Segment->LastPage) + { + LARGE_INTEGER CheckOffset; + CheckOffset.QuadPart = (Segment->LastPage - 1) << PAGE_SHIFT; + ULONG_PTR Entry = MmGetPageEntrySectionSegment(Segment, &CheckOffset); + if ((Entry != 0) && !IS_SWAP_FROM_SSE(Entry)) + break; + } + } } PageTable->PageEntries[PageIndex] = Entry; diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index 16767458bce..8515820e116 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -4113,9 +4113,6 @@ MmCanFileBeTruncated (IN PSECTION_OBJECT_POINTERS SectionObjectPointer, return Ret; } - - - /* * @implemented */ From ea354c4bc70cb7ee9302cfdd6a2c780b29639af8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Wed, 27 Jan 2021 17:02:43 +0100 Subject: [PATCH 90/99] [NTOS:MM] Unconditionally purge cache on image section creation. --- ntoskrnl/mm/section.c | 64 ++++++------------------------------------- 1 file changed, 8 insertions(+), 56 deletions(-) diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index 8515820e116..ff7d9438baa 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -2470,20 +2470,6 @@ ExeFmtpAllocateSegments(IN ULONG NrSegments) return Segments; } - -static NTSTATUS -MmMapViewOfSegment(PMMSUPPORT AddressSpace, - BOOLEAN AsImage, - PMM_SECTION_SEGMENT Segment, - PVOID* BaseAddress, - SIZE_T ViewSize, - ULONG Protect, - LONGLONG ViewOffset, - ULONG AllocationType); -static NTSTATUS -MmUnmapViewOfSegment(PMMSUPPORT AddressSpace, - PVOID BaseAddress); - static NTSTATUS NTAPI @@ -2538,45 +2524,14 @@ ExeFmtpReadFile(IN PVOID File, return STATUS_INSUFFICIENT_RESOURCES; } - if (FileObject->SectionObjectPointer->DataSectionObject) + Status = MiSimpleRead(FileObject, &FileOffset, Buffer, BufferSize, TRUE, &Iosb); + + UsedSize = (ULONG)Iosb.Information; + + if(NT_SUCCESS(Status) && UsedSize < OffsetAdjustment) { - PVOID SegmentMap = NULL; - - /* Get the data from the file mapping instead */ - MmLockAddressSpace(MmGetKernelAddressSpace()); - Status = MmMapViewOfSegment(MmGetKernelAddressSpace(), - FALSE, - FileObject->SectionObjectPointer->DataSectionObject, - &SegmentMap, - BufferSize, - PAGE_READONLY, - FileOffset.QuadPart, - 0); - MmUnlockAddressSpace(MmGetKernelAddressSpace()); - - if (!NT_SUCCESS(Status)) - return Status; - - RtlCopyMemory(Buffer, SegmentMap, BufferSize); - UsedSize = BufferSize; - - MmLockAddressSpace(MmGetKernelAddressSpace()); - - MmUnmapViewOfSegment(MmGetKernelAddressSpace(), SegmentMap); - - MmUnlockAddressSpace(MmGetKernelAddressSpace()); - } - else - { - Status = MiSimpleRead(FileObject, &FileOffset, Buffer, BufferSize, TRUE, &Iosb); - - UsedSize = (ULONG)Iosb.Information; - - if(NT_SUCCESS(Status) && UsedSize < OffsetAdjustment) - { - Status = STATUS_IN_PAGE_ERROR; - ASSERT(!NT_SUCCESS(Status)); - } + Status = STATUS_IN_PAGE_ERROR; + ASSERT(!NT_SUCCESS(Status)); } if(NT_SUCCESS(Status)) @@ -3175,10 +3130,7 @@ MmCreateImageSection(PSECTION *SectionObject, MiReleasePfnLock(OldIrql); /* Purge the cache */ - if (CcIsFileCached(FileObject)) - { - CcFlushCache(FileObject->SectionObjectPointer, NULL, 0, NULL); - } + CcFlushCache(FileObject->SectionObjectPointer, NULL, 0, NULL); StatusExeFmt = ExeFmtpCreateImageSection(FileObject, ImageSectionObject); From 7fbf90d6fdc6c297bb6b8497330a4d692e2ccf1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Thu, 28 Jan 2021 15:41:28 +0100 Subject: [PATCH 91/99] [NTOS:CC] Fix a bug --- ntoskrnl/cc/pin.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ntoskrnl/cc/pin.c b/ntoskrnl/cc/pin.c index 07bfd63241c..af7b3aa71a8 100644 --- a/ntoskrnl/cc/pin.c +++ b/ntoskrnl/cc/pin.c @@ -301,7 +301,7 @@ CcpPinData( { CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> FALSE\n", SharedCacheMap->FileObject, FileOffset, Length, Flags); - CcUnpinData(NewBcb); + CcUnpinData(&NewBcb->PFCB); return FALSE; } } From 41475dfcd71275f2c704cc3e8cfbcb8bf35aa242 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Thu, 28 Jan 2021 15:42:12 +0100 Subject: [PATCH 92/99] [NTOS:CC] Performance improvements Do not ditch the pages as soon as the section are unmapped Improve MmBalancer "algorithm" (or whatever you call that) Various needed fixes to get this going. --- ntoskrnl/cache/section/newmm.h | 2 - ntoskrnl/include/internal/mm.h | 10 ++ ntoskrnl/mm/balance.c | 197 +++++++++++++++++---------------- ntoskrnl/mm/freelist.c | 9 +- ntoskrnl/mm/i386/page.c | 40 +++++++ ntoskrnl/mm/rmap.c | 115 +++++++++++-------- ntoskrnl/mm/section.c | 143 +++++++++++++++--------- 7 files changed, 314 insertions(+), 202 deletions(-) diff --git a/ntoskrnl/cache/section/newmm.h b/ntoskrnl/cache/section/newmm.h index 3fb731c6ace..2efcd9cf7d5 100644 --- a/ntoskrnl/cache/section/newmm.h +++ b/ntoskrnl/cache/section/newmm.h @@ -5,8 +5,6 @@ /* TYPES *********************************************************************/ #define MM_SEGMENT_FINALIZE (0x40000000) -#define RMAP_SEGMENT_MASK ~((ULONG_PTR)0xff) -#define RMAP_IS_SEGMENT(x) (((ULONG_PTR)(x) & RMAP_SEGMENT_MASK) == RMAP_SEGMENT_MASK) #define MIN(x,y) (((x)<(y))?(x):(y)) #define MAX(x,y) (((x)>(y))?(x):(y)) diff --git a/ntoskrnl/include/internal/mm.h b/ntoskrnl/include/internal/mm.h index 86ed4dfb23a..0243ac55177 100644 --- a/ntoskrnl/include/internal/mm.h +++ b/ntoskrnl/include/internal/mm.h @@ -830,6 +830,8 @@ NTAPI MmRebalanceMemoryConsumers(VOID); /* rmap.c **************************************************************/ +#define RMAP_SEGMENT_MASK ~((ULONG_PTR)0xff) +#define RMAP_IS_SEGMENT(x) (((ULONG_PTR)(x) & RMAP_SEGMENT_MASK) == RMAP_SEGMENT_MASK) VOID NTAPI @@ -1189,6 +1191,14 @@ MmIsDirtyPage( PVOID Address ); +VOID +NTAPI +MmClearPageAccessedBit(PEPROCESS Process, PVOID Address); + +BOOLEAN +NTAPI +MmIsPageAccessed(PEPROCESS Process, PVOID Address); + /* wset.c ********************************************************************/ NTSTATUS diff --git a/ntoskrnl/mm/balance.c b/ntoskrnl/mm/balance.c index f9067c58a13..04231e86951 100644 --- a/ntoskrnl/mm/balance.c +++ b/ntoskrnl/mm/balance.c @@ -113,14 +113,13 @@ MiTrimMemoryConsumer(ULONG Consumer, ULONG InitialTarget) Target = (ULONG)max(Target, MiMinimumAvailablePages - MmAvailablePages); } - /* Don't be too greedy if we're not in a hurry */ - if (MmAvailablePages > MiMinimumAvailablePages) - Target = min(Target, 256); + /* Don't be too greedy in one run */ + Target = min(Target, 256); if (Target) { /* Now swap the pages out */ - Status = MiMemoryConsumers[Consumer].Trim(Target, 0, &NrFreedPages); + Status = MiMemoryConsumers[Consumer].Trim(Target, MmAvailablePages < MiMinimumAvailablePages, &NrFreedPages); DPRINT("Trimming consumer %lu: Freed %lu pages with a target of %lu pages\n", Consumer, NrFreedPages, Target); @@ -142,15 +141,104 @@ MmTrimUserMemory(ULONG Target, ULONG Priority, PULONG NrFreedPages) (*NrFreedPages) = 0; + DPRINT1("MM BALANCER: %s\n", Priority ? "Paging out!" : "Removing access bit!"); + CurrentPage = MmGetLRUFirstUserPage(); while (CurrentPage != 0 && Target > 0) { - Status = MmPageOutPhysicalAddress(CurrentPage); - if (NT_SUCCESS(Status)) + if (Priority) { - DPRINT("Succeeded\n"); + Status = MmPageOutPhysicalAddress(CurrentPage); + if (NT_SUCCESS(Status)) + { + DPRINT("Succeeded\n"); + Target--; + (*NrFreedPages)++; + } + } + else + { + /* When not paging-out agressively, just reset the accessed bit */ + PEPROCESS Process = NULL; + PVOID Address = NULL; + BOOLEAN Accessed = FALSE; + + /* + * We have a lock-ordering problem here. We cant lock the PFN DB before the Process address space. + * So we must use circonvoluted loops. + * Well... + */ + while (TRUE) + { + KAPC_STATE ApcState; + KIRQL OldIrql = MiAcquirePfnLock(); + PMM_RMAP_ENTRY Entry = MmGetRmapListHeadPage(CurrentPage); + while (Entry) + { + if (RMAP_IS_SEGMENT(Entry->Address)) + { + Entry = Entry->Next; + continue; + } + + /* Check that we didn't treat this entry before */ + if (Entry->Address < Address) + { + Entry = Entry->Next; + continue; + } + + if ((Entry->Address == Address) && (Entry->Process <= Process)) + { + Entry = Entry->Next; + continue; + } + + break; + } + + if (!Entry) + { + MiReleasePfnLock(OldIrql); + break; + } + + Process = Entry->Process; + Address = Entry->Address; + + MiReleasePfnLock(OldIrql); + + KeStackAttachProcess(&Process->Pcb, &ApcState); + + MmLockAddressSpace(&Process->Vm); + + /* Be sure this is still valid. */ + PMMPTE Pte = MiAddressToPte(Address); + if (Pte->u.Hard.Valid) + { + Accessed = Accessed || Pte->u.Hard.Accessed; + Pte->u.Hard.Accessed = 0; + + /* There is no need to invalidate, the balancer thread is never on a user process */ + //KeInvalidateTlbEntry(Address); + } + + MmUnlockAddressSpace(&Process->Vm); + + KeUnstackDetachProcess(&ApcState); + } + + if (!Accessed) + { + /* Nobody accessed this page since the last time we check. Time to clean up */ + + Status = MmPageOutPhysicalAddress(CurrentPage); + // DPRINT1("Paged-out one page: %s\n", NT_SUCCESS(Status) ? "Yes" : "No"); + (void)Status; + } + + /* Done for this page. */ Target--; - (*NrFreedPages)++; } CurrentPage = MmGetLRUNextUserPage(CurrentPage, TRUE); @@ -189,78 +277,10 @@ NTAPI MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait, PPFN_NUMBER AllocatedPage) { - ULONG PagesUsed; PFN_NUMBER Page; - /* - * Make sure we don't exceed our individual target. - */ - PagesUsed = InterlockedIncrementUL(&MiMemoryConsumers[Consumer].PagesUsed); - if (PagesUsed > MiMemoryConsumers[Consumer].PagesTarget && - !MiIsBalancerThread()) - { - MmRebalanceMemoryConsumers(); - } - - /* - * Allocate always memory for the non paged pool and for the pager thread. - */ - if (Consumer == MC_SYSTEM) - { - Page = MmAllocPage(Consumer); - if (Page == 0) - { - KeBugCheck(NO_PAGES_AVAILABLE); - } - *AllocatedPage = Page; - if (MmAvailablePages < MiMinimumAvailablePages) - MmRebalanceMemoryConsumers(); - return(STATUS_SUCCESS); - } - - /* - * Make sure we don't exceed global targets. - */ - if (((MmAvailablePages < MiMinimumAvailablePages) && !MiIsBalancerThread()) - || (MmAvailablePages < (MiMinimumAvailablePages / 2))) - { - MM_ALLOCATION_REQUEST Request; - - if (!CanWait) - { - (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed); - MmRebalanceMemoryConsumers(); - return(STATUS_NO_MEMORY); - } - - /* Insert an allocation request. */ - Request.Page = 0; - KeInitializeEvent(&Request.Event, NotificationEvent, FALSE); - - ExInterlockedInsertTailList(&AllocationListHead, &Request.ListEntry, &AllocationListLock); - MmRebalanceMemoryConsumers(); - - KeWaitForSingleObject(&Request.Event, - 0, - KernelMode, - FALSE, - NULL); - - Page = Request.Page; - if (Page == 0) - { - KeBugCheck(NO_PAGES_AVAILABLE); - } - - *AllocatedPage = Page; - - if (MmAvailablePages < MiMinimumAvailablePages) - { - MmRebalanceMemoryConsumers(); - } - - return(STATUS_SUCCESS); - } + /* Update the target */ + InterlockedIncrementUL(&MiMemoryConsumers[Consumer].PagesUsed); /* * Actually allocate the page. @@ -272,11 +292,6 @@ MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait, } *AllocatedPage = Page; - if (MmAvailablePages < MiMinimumAvailablePages) - { - MmRebalanceMemoryConsumers(); - } - return(STATUS_SUCCESS); } @@ -407,22 +422,14 @@ MiInitBalancerThread(VOID) { KPRIORITY Priority; NTSTATUS Status; -#if !defined(__GNUC__) - - LARGE_INTEGER dummyJunkNeeded; - dummyJunkNeeded.QuadPart = -20000000; /* 2 sec */ - ; -#endif - + LARGE_INTEGER Timeout; KeInitializeEvent(&MiBalancerEvent, SynchronizationEvent, FALSE); KeInitializeTimerEx(&MiBalancerTimer, SynchronizationTimer); + + Timeout.QuadPart = -20000000; /* 2 sec */ KeSetTimerEx(&MiBalancerTimer, -#if defined(__GNUC__) - (LARGE_INTEGER)(LONGLONG)-20000000LL, /* 2 sec */ -#else - dummyJunkNeeded, -#endif + Timeout, 2000, /* 2 sec */ NULL); diff --git a/ntoskrnl/mm/freelist.c b/ntoskrnl/mm/freelist.c index 35c6336dcc6..4aa15a12582 100644 --- a/ntoskrnl/mm/freelist.c +++ b/ntoskrnl/mm/freelist.c @@ -139,7 +139,7 @@ MmGetLRUNextUserPage(PFN_NUMBER PreviousPage, BOOLEAN MoveToLast) * If it's not, then it means it is still hanging in some process address space. * This avoids paging-out e.g. ntdll early just because it's mapped first time. */ - if (MoveToLast) + if ((MoveToLast) && (MmGetReferenceCountPage(PreviousPage) > 1)) { MmRemoveLRUUserPage(PreviousPage); MmInsertLRULastUserPage(PreviousPage); @@ -424,10 +424,11 @@ VOID NTAPI MmSetRmapListHeadPage(PFN_NUMBER Pfn, PMM_RMAP_ENTRY ListHead) { - KIRQL oldIrql; PMMPFN Pfn1; - oldIrql = MiAcquirePfnLock(); + /* PFN database must be locked */ + MI_ASSERT_PFN_LOCK_HELD(); + Pfn1 = MiGetPfnEntry(Pfn); ASSERT(Pfn1); ASSERT_IS_ROS_PFN(Pfn1); @@ -450,8 +451,6 @@ MmSetRmapListHeadPage(PFN_NUMBER Pfn, PMM_RMAP_ENTRY ListHead) /* ReactOS semantics will now release the page, which will make it free and enter a colored list */ } - - MiReleasePfnLock(oldIrql); } PMM_RMAP_ENTRY diff --git a/ntoskrnl/mm/i386/page.c b/ntoskrnl/mm/i386/page.c index bcc2be911b1..b541b5d4752 100644 --- a/ntoskrnl/mm/i386/page.c +++ b/ntoskrnl/mm/i386/page.c @@ -585,6 +585,46 @@ MmSetDirtyPage(PEPROCESS Process, PVOID Address) } } +VOID +NTAPI +MmClearPageAccessedBit(PEPROCESS Process, PVOID Address) +{ + PULONG Pt; + LONG Pte; + KIRQL OldIrql; + + if (Address < MmSystemRangeStart && Process == NULL) + { + DPRINT1("MmClearPageAccessedBit is called for user space without a process.\n"); + KeBugCheck(MEMORY_MANAGEMENT); + } + + Pt = MmGetPageTableForProcess(Process, Address, FALSE, &OldIrql); + if (Pt == NULL) + { + KeBugCheck(MEMORY_MANAGEMENT); + } + + do + { + Pte = *Pt; + } while (Pte != InterlockedCompareExchangePte(Pt, Pte & ~PA_ACCESSED, Pte)); + + if (!(Pte & PA_PRESENT)) + { + KeBugCheck(MEMORY_MANAGEMENT); + } + + MiFlushTlb(Pt, Address, OldIrql); +} + +BOOLEAN +NTAPI +MmIsPageAccessed(PEPROCESS Process, PVOID Address) +{ + return BooleanFlagOn(MmGetPageEntryForProcess(Process, Address), PA_ACCESSED); +} + BOOLEAN NTAPI MmIsPagePresent(PEPROCESS Process, PVOID Address) diff --git a/ntoskrnl/mm/rmap.c b/ntoskrnl/mm/rmap.c index 5b7ffe3a3da..f926afb8471 100644 --- a/ntoskrnl/mm/rmap.c +++ b/ntoskrnl/mm/rmap.c @@ -53,13 +53,14 @@ MmPageOutPhysicalAddress(PFN_NUMBER Page) PMM_RMAP_ENTRY entry; PMEMORY_AREA MemoryArea; PMMSUPPORT AddressSpace; - PVOID Address; - PEPROCESS Process; + PVOID Address = NULL; + PEPROCESS Process = NULL; NTSTATUS Status = STATUS_SUCCESS; PMM_SECTION_SEGMENT Segment; LARGE_INTEGER SegmentOffset; KIRQL OldIrql; +GetEntry: OldIrql = MiAcquirePfnLock(); entry = MmGetRmapListHeadPage(Page); @@ -67,6 +68,16 @@ MmPageOutPhysicalAddress(PFN_NUMBER Page) while (entry && RMAP_IS_SEGMENT(entry->Address)) entry = entry->Next; + /* See if we are retrying because the page is actively used */ + while (entry && ((entry->Address < Address) || RMAP_IS_SEGMENT(entry->Address))) + entry = entry->Next; + + if (entry && (entry->Address == Address)) + { + while (entry && ((entry->Process <= Process) || RMAP_IS_SEGMENT(entry->Address))) + entry = entry->Next; + } + if (entry == NULL) { MiReleasePfnLock(OldIrql); @@ -81,41 +92,42 @@ MmPageOutPhysicalAddress(PFN_NUMBER Page) KeBugCheck(MEMORY_MANAGEMENT); } - if (Address < MmSystemRangeStart) - { - if (!ExAcquireRundownProtection(&Process->RundownProtect)) - { - MiReleasePfnLock(OldIrql); - return STATUS_PROCESS_IS_TERMINATING; - } + /* This is for user-mode address only */ + ASSERT(Address < MmSystemRangeStart); - Status = ObReferenceObjectByPointer(Process, PROCESS_ALL_ACCESS, NULL, KernelMode); - MiReleasePfnLock(OldIrql); - if (!NT_SUCCESS(Status)) - { - ExReleaseRundownProtection(&Process->RundownProtect); - return Status; - } - AddressSpace = &Process->Vm; - } - else + if (!ExAcquireRundownProtection(&Process->RundownProtect)) { MiReleasePfnLock(OldIrql); - AddressSpace = MmGetKernelAddressSpace(); + return STATUS_PROCESS_IS_TERMINATING; } + Status = ObReferenceObjectByPointer(Process, PROCESS_ALL_ACCESS, NULL, KernelMode); + MiReleasePfnLock(OldIrql); + if (!NT_SUCCESS(Status)) + { + ExReleaseRundownProtection(&Process->RundownProtect); + return Status; + } + AddressSpace = &Process->Vm; + MmLockAddressSpace(AddressSpace); + if ((MmGetPfnForProcess(Process, Address) != Page) || MmIsPageAccessed(Process, Address)) + { + /* This changed in the short window where we didn't have any locks */ + MmUnlockAddressSpace(AddressSpace); + ExReleaseRundownProtection(&Process->RundownProtect); + ObDereferenceObject(Process); + goto GetEntry; + } + MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace, Address); if (MemoryArea == NULL || MemoryArea->DeleteInProgress) { MmUnlockAddressSpace(AddressSpace); - if (Address < MmSystemRangeStart) - { - ExReleaseRundownProtection(&Process->RundownProtect); - ObDereferenceObject(Process); - } - return(STATUS_UNSUCCESSFUL); + ExReleaseRundownProtection(&Process->RundownProtect); + ObDereferenceObject(Process); + goto GetEntry; } if (MemoryArea->Type == MEMORY_AREA_SECTION_VIEW) @@ -269,7 +281,7 @@ MmPageOutPhysicalAddress(PFN_NUMBER Page) MmDeleteRmap(Page, Process, Address); /* One less mapping referencing this segment */ - Released = MmUnsharePageEntrySectionSegment(MemoryArea, Segment, &Offset, Dirty, FALSE, NULL); + Released = MmUnsharePageEntrySectionSegment(MemoryArea, Segment, &Offset, Dirty, TRUE, NULL); MmUnlockSectionSegment(Segment); MmUnlockAddressSpace(AddressSpace); @@ -396,11 +408,7 @@ MmInsertRmap(PFN_NUMBER Page, PEPROCESS Process, new_entry->Address = Address; new_entry->Process = (PEPROCESS)Process; #if DBG -#ifdef __GNUC__ - new_entry->Caller = __builtin_return_address(0); -#else new_entry->Caller = _ReturnAddress(); -#endif #endif if ( @@ -417,24 +425,39 @@ MmInsertRmap(PFN_NUMBER Page, PEPROCESS Process, OldIrql = MiAcquirePfnLock(); current_entry = MmGetRmapListHeadPage(Page); - new_entry->Next = current_entry; -#if DBG - while (current_entry) + + PMM_RMAP_ENTRY previous_entry = NULL; + /* Keep the list sorted */ + while (current_entry && (current_entry->Address < Address)) { - if (current_entry->Address == new_entry->Address && current_entry->Process == new_entry->Process) - { - DbgPrint("MmInsertRmap tries to add a second rmap entry for address %p\n current caller ", - current_entry->Address); - DbgPrint("%p", new_entry->Caller); - DbgPrint("\n previous caller "); - DbgPrint("%p", current_entry->Caller); - DbgPrint("\n"); - KeBugCheck(MEMORY_MANAGEMENT); - } + previous_entry = current_entry; current_entry = current_entry->Next; } -#endif - MmSetRmapListHeadPage(Page, new_entry); + + /* In case of clash in the address, sort by process */ + if (current_entry && (current_entry->Address == Address)) + { + while (current_entry && (current_entry->Process < Process)) + { + previous_entry = current_entry; + current_entry = current_entry->Next; + } + } + + if (current_entry && (current_entry->Address == Address) && (current_entry->Process == Process)) + { + DbgPrint("MmInsertRmap tries to add a second rmap entry for address %p\n", current_entry->Address); + DbgPrint(" current caller %p\n", new_entry->Caller); + DbgPrint(" previous caller %p\n", current_entry->Caller); + KeBugCheck(MEMORY_MANAGEMENT); + } + + new_entry->Next = current_entry; + if (previous_entry) + previous_entry->Next = new_entry; + else + MmSetRmapListHeadPage(Page, new_entry); + MiReleasePfnLock(OldIrql); if (!RMAP_IS_SEGMENT(Address)) diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index ff7d9438baa..94ea1844d2d 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -1091,8 +1091,7 @@ MmUnsharePageEntrySectionSegment(PMEMORY_AREA MemoryArea, { ULONG_PTR Entry = InEntry ? *InEntry : MmGetPageEntrySectionSegment(Segment, Offset); PFN_NUMBER Page = PFN_FROM_SSE(Entry); - ULONG_PTR NewEntry = 0; - SWAPENTRY SwapEntry; + BOOLEAN IsDataMap = BooleanFlagOn(*Segment->Flags, MM_DATAFILE_SEGMENT); if (Entry == 0) { @@ -1111,64 +1110,53 @@ MmUnsharePageEntrySectionSegment(PMEMORY_AREA MemoryArea, Entry = DECREF_SSE(Entry); if (Dirty) Entry = DIRTY_SSE(Entry); - if (SHARE_COUNT_FROM_SSE(Entry) > 0) + /* If we are paging-out, pruning the page for real will be taken care of in MmCheckDirtySegment */ + if ((SHARE_COUNT_FROM_SSE(Entry) > 0) || PageOut) { /* Update the page mapping in the segment and we're done */ - if (InEntry) - *InEntry = Entry; - else - MmSetPageEntrySectionSegment(Segment, Offset, Entry); - return FALSE; - } - - if (IS_DIRTY_SSE(Entry) && (MemoryArea->VadNode.u.VadFlags.VadType != VadImageMap)) - { - ASSERT(!Segment->WriteCopy); - ASSERT(MmGetSavedSwapEntryPage(Page) == 0); - - /* The entry must be written back to the disk, so let this in the segment, the page-out thread will take care of this */ MmSetPageEntrySectionSegment(Segment, Offset, Entry); return FALSE; } - /* Only valid case for shared dirty pages is shared image section */ - ASSERT(!IS_DIRTY_SSE(Entry) || (Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)); + /* We are pruning the last mapping on this page. See if we can keep it a bit more. */ + ASSERT(!PageOut); - SwapEntry = MmGetSavedSwapEntryPage(Page); - if (IS_DIRTY_SSE(Entry) && !SwapEntry) + if (IsDataMap) { - SwapEntry = MmAllocSwapPage(); - if (!SwapEntry) - { - /* We can't have a swap entry for this page. Let the segment keep it */ - MmSetPageEntrySectionSegment(Segment, Offset, Entry); - return FALSE; - } + /* We can always keep memory in for data maps */ + MmSetPageEntrySectionSegment(Segment, Offset, Entry); + return FALSE; } - if (IS_DIRTY_SSE(Entry)) + if (!BooleanFlagOn(Segment->Image.Characteristics, IMAGE_SCN_MEM_SHARED)) { - NTSTATUS Status = MmWriteToSwapPage(SwapEntry, Page); - if (!NT_SUCCESS(Status)) - { - /* We failed. Clean up */ - MmSetSavedSwapEntryPage(Page, 0); - MmFreeSwapPage(SwapEntry); - MmSetPageEntrySectionSegment(Segment, Offset, Entry); - return FALSE; - } + /* So this must have been a read-only page. Keep it ! */ + ASSERT(Segment->WriteCopy); + ASSERT(!IS_DIRTY_SSE(Entry)); + ASSERT(MmGetSavedSwapEntryPage(Page) == 0); + MmSetPageEntrySectionSegment(Segment, Offset, Entry); + return FALSE; } + /* + * So this is a page for a shared section of a DLL. + * We can keep it if it is not dirty. + */ + SWAPENTRY SwapEntry = MmGetSavedSwapEntryPage(Page); + if ((SwapEntry == 0) && !IS_DIRTY_SSE(Entry)) + { + MmSetPageEntrySectionSegment(Segment, Offset, Entry); + return FALSE; + } + + /* No more processes are referencing this shared dirty page. Ditch it. */ if (SwapEntry) { - NewEntry = MAKE_SWAP_SSE(SwapEntry); MmSetSavedSwapEntryPage(Page, 0); + MmFreeSwapPage(SwapEntry); } - - /* We can let this go */ - MmSetPageEntrySectionSegment(Segment, Offset, NewEntry); + MmSetPageEntrySectionSegment(Segment, Offset, 0); MmReleasePageMemoryConsumer(MC_USER, Page); - MiSetPageEvent(NULL, NULL); return TRUE; } @@ -4849,9 +4837,12 @@ MmCheckDirtySegment( { BOOLEAN DirtyAgain; - /* We got a dirty entry. Is this segment copy on write */ + /* + * We got a dirty entry. This path is for the shared data, + * be-it regular file maps or shared sections of DLLs + */ ASSERT(!Segment->WriteCopy); - ASSERT(Segment->SegFlags & MM_DATAFILE_SEGMENT); + ASSERT(FlagOn(*Segment->Flags, MM_DATAFILE_SEGMENT) || FlagOn(Segment->Image.Characteristics, IMAGE_SCN_MEM_SHARED)); /* Insert the cleaned entry back. Mark it as write in progress, and clear the dirty bit. */ Entry = MAKE_SSE(PAGE_FROM_SSE(Entry), SHARE_COUNT_FROM_SSE(Entry) + 1); @@ -4863,17 +4854,52 @@ MmCheckDirtySegment( MmUnlockSectionSegment(Segment); - /* Tell the FS driver who we are */ - if (PageOut) - IoSetTopLevelIrp((PIRP)FSRTL_MOD_WRITE_TOP_LEVEL_IRP); + if (FlagOn(*Segment->Flags, MM_DATAFILE_SEGMENT)) + { + /* We have to write it back to the file. Tell the FS driver who we are */ + if (PageOut) + IoSetTopLevelIrp((PIRP)FSRTL_MOD_WRITE_TOP_LEVEL_IRP); - /* Go ahead and write the page */ - DPRINT("Writing page at offset %I64d for file %wZ, Pageout: %s\n", - Offset->QuadPart, &Segment->FileObject->FileName, PageOut ? "TRUE" : "FALSE"); - Status = MiWritePage(Segment, Offset->QuadPart, Page); + /* Go ahead and write the page */ + DPRINT("Writing page at offset %I64d for file %wZ, Pageout: %s\n", + Offset->QuadPart, &Segment->FileObject->FileName, PageOut ? "TRUE" : "FALSE"); + Status = MiWritePage(Segment, Offset->QuadPart, Page); - if (PageOut) - IoSetTopLevelIrp(NULL); + if (PageOut) + IoSetTopLevelIrp(NULL); + } + else + { + /* This must only be called by the page-out path */ + ASSERT(PageOut); + + /* And this must be for a shared section in a DLL */ + ASSERT(Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED); + + SWAPENTRY SwapEntry = MmGetSavedSwapEntryPage(Page); + if (!SwapEntry) + { + SwapEntry = MmAllocSwapPage(); + } + + if (SwapEntry) + { + Status = MmWriteToSwapPage(SwapEntry, Page); + if (NT_SUCCESS(Status)) + { + MmSetSavedSwapEntryPage(Page, SwapEntry); + } + else + { + MmFreeSwapPage(SwapEntry); + } + } + else + { + DPRINT1("Failed to allocate a swap page!\n"); + Status = STATUS_INSUFFICIENT_RESOURCES; + } + } MmLockSectionSegment(Segment); @@ -4905,8 +4931,17 @@ MmCheckDirtySegment( /* Were this page hanging there just for the sake of being present ? */ if (!IS_DIRTY_SSE(Entry) && (SHARE_COUNT_FROM_SSE(Entry) == 0) && PageOut) { + ULONG_PTR NewEntry = 0; + /* Restore the swap entry here */ + if (!FlagOn(*Segment->Flags, MM_DATAFILE_SEGMENT)) + { + SWAPENTRY SwapEntry = MmGetSavedSwapEntryPage(Page); + if (SwapEntry) + NewEntry = MAKE_SWAP_SSE(SwapEntry); + } + /* Yes. Release it */ - MmSetPageEntrySectionSegment(Segment, Offset, 0); + MmSetPageEntrySectionSegment(Segment, Offset, NewEntry); MmReleasePageMemoryConsumer(MC_USER, Page); /* Tell the caller we released the page */ return TRUE; From 625f27336176c7f1db4ad5ade75597a3820853ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Fri, 29 Jan 2021 18:47:26 +0100 Subject: [PATCH 93/99] [NTOS:MM] More fine-tuning of the memory balancer --- ntoskrnl/mm/balance.c | 21 +++++++++++---------- ntoskrnl/mm/rmap.c | 10 ---------- 2 files changed, 11 insertions(+), 20 deletions(-) diff --git a/ntoskrnl/mm/balance.c b/ntoskrnl/mm/balance.c index 04231e86951..1cab99be0ec 100644 --- a/ntoskrnl/mm/balance.c +++ b/ntoskrnl/mm/balance.c @@ -37,6 +37,8 @@ static HANDLE MiBalancerThreadHandle = NULL; static KEVENT MiBalancerEvent; static KTIMER MiBalancerTimer; +static LONG PageOutThreadActive; + /* FUNCTIONS ****************************************************************/ CODE_SEG("INIT") @@ -102,19 +104,16 @@ MiTrimMemoryConsumer(ULONG Consumer, ULONG InitialTarget) return InitialTarget; } - if (MiMemoryConsumers[Consumer].PagesUsed > MiMemoryConsumers[Consumer].PagesTarget) - { - /* Consumer page limit exceeded */ - Target = max(Target, MiMemoryConsumers[Consumer].PagesUsed - MiMemoryConsumers[Consumer].PagesTarget); - } if (MmAvailablePages < MiMinimumAvailablePages) { /* Global page limit exceeded */ Target = (ULONG)max(Target, MiMinimumAvailablePages - MmAvailablePages); } - - /* Don't be too greedy in one run */ - Target = min(Target, 256); + else if (MiMemoryConsumers[Consumer].PagesUsed > MiMemoryConsumers[Consumer].PagesTarget) + { + /* Consumer page limit exceeded */ + Target = max(Target, MiMemoryConsumers[Consumer].PagesUsed - MiMemoryConsumers[Consumer].PagesTarget); + } if (Target) { @@ -265,8 +264,7 @@ VOID NTAPI MmRebalanceMemoryConsumers(VOID) { - if (MiBalancerThreadHandle != NULL && - !MiIsBalancerThread()) + if (InterlockedCompareExchange(&PageOutThreadActive, 0, 1) == 0) { KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE); } @@ -366,6 +364,9 @@ MiBalancerThread(PVOID Unused) } } while (InitialTarget != 0); + + if (Status == STATUS_WAIT_0) + InterlockedDecrement(&PageOutThreadActive); } else { diff --git a/ntoskrnl/mm/rmap.c b/ntoskrnl/mm/rmap.c index f926afb8471..e40d9554038 100644 --- a/ntoskrnl/mm/rmap.c +++ b/ntoskrnl/mm/rmap.c @@ -68,16 +68,6 @@ GetEntry: while (entry && RMAP_IS_SEGMENT(entry->Address)) entry = entry->Next; - /* See if we are retrying because the page is actively used */ - while (entry && ((entry->Address < Address) || RMAP_IS_SEGMENT(entry->Address))) - entry = entry->Next; - - if (entry && (entry->Address == Address)) - { - while (entry && ((entry->Process <= Process) || RMAP_IS_SEGMENT(entry->Address))) - entry = entry->Next; - } - if (entry == NULL) { MiReleasePfnLock(OldIrql); From 2ba1926037223899fa2052d9a72663f095d37f54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Fri, 29 Jan 2021 18:48:32 +0100 Subject: [PATCH 94/99] [NTOS:MM][NTOS:CC] Performance improvement again Read files by 64kb chunks instead of page-sized chunks. --- ntoskrnl/cc/copy.c | 48 +-- ntoskrnl/cc/pin.c | 5 - ntoskrnl/cc/view.c | 40 ++- ntoskrnl/include/internal/cc.h | 2 +- ntoskrnl/include/internal/mm.h | 15 +- ntoskrnl/mm/section.c | 523 +++++++++++++++++---------------- 6 files changed, 310 insertions(+), 323 deletions(-) diff --git a/ntoskrnl/cc/copy.c b/ntoskrnl/cc/copy.c index 8ed24f2bd92..cdab94c8313 100644 --- a/ntoskrnl/cc/copy.c +++ b/ntoskrnl/cc/copy.c @@ -504,24 +504,6 @@ CcCopyRead ( CurrentOffset = FileOffset->QuadPart; while(CurrentOffset < ReadEnd) { - if (CurrentOffset >= SharedCacheMap->ValidDataLength.QuadPart) - { - DPRINT1("Zeroing buffer because we are beyond the VDL.\n"); - /* We are beyond what is valid. Just zero this out */ - _SEH2_TRY - { - RtlZeroMemory(Buffer, Length); - } - _SEH2_EXCEPT(CcpCheckInvalidUserBuffer(_SEH2_GetExceptionInformation(), Buffer, Length)) - { - ExRaiseStatus(STATUS_INVALID_USER_BUFFER); - } - _SEH2_END; - - ReadLength += Length; - break; - } - Status = CcRosGetVacb(SharedCacheMap, CurrentOffset, &Vacb); if (!NT_SUCCESS(Status)) { @@ -538,25 +520,15 @@ CcCopyRead ( if (!CcRosEnsureVacbResident(Vacb, Wait, FALSE, VacbOffset, VacbLength)) return FALSE; - /* Do not copy past the section */ - if (CurrentOffset + VacbLength > SharedCacheMap->SectionSize.QuadPart) - CopyLength = SharedCacheMap->SectionSize.QuadPart - CurrentOffset; - if (CopyLength != 0) + _SEH2_TRY { - _SEH2_TRY - { - RtlCopyMemory(Buffer, (PUCHAR)Vacb->BaseAddress + VacbOffset, CopyLength); - } - _SEH2_EXCEPT(CcpCheckInvalidUserBuffer(_SEH2_GetExceptionInformation(), Buffer, VacbLength)) - { - ExRaiseStatus(STATUS_INVALID_USER_BUFFER); - } - _SEH2_END; + RtlCopyMemory(Buffer, (PUCHAR)Vacb->BaseAddress + VacbOffset, CopyLength); } - - /* Zero-out the buffer tail if needed */ - if (CopyLength < VacbLength) - RtlZeroMemory((PUCHAR)Buffer + CopyLength, VacbLength - CopyLength); + _SEH2_EXCEPT(CcpCheckInvalidUserBuffer(_SEH2_GetExceptionInformation(), Buffer, VacbLength)) + { + ExRaiseStatus(STATUS_INVALID_USER_BUFFER); + } + _SEH2_END; ReadLength += VacbLength; @@ -684,10 +656,6 @@ CcCopyWrite ( if (FileObject->Flags & FO_WRITE_THROUGH) CcFlushCache(FileObject->SectionObjectPointer, FileOffset, Length, NULL); - /* Update VDL */ - if (WriteEnd > SharedCacheMap->ValidDataLength.QuadPart) - SharedCacheMap->ValidDataLength.QuadPart = WriteEnd; - return TRUE; } @@ -898,7 +866,7 @@ CcZeroData ( } /* See if we should simply truncate the valid data length */ - if ((StartOffset->QuadPart < SharedCacheMap->ValidDataLength.QuadPart) && (EndOffset->QuadPart > SharedCacheMap->ValidDataLength.QuadPart)) + if ((StartOffset->QuadPart < SharedCacheMap->ValidDataLength.QuadPart) && (EndOffset->QuadPart >= SharedCacheMap->ValidDataLength.QuadPart)) { DPRINT1("Truncating VDL.\n"); SharedCacheMap->ValidDataLength = *StartOffset; diff --git a/ntoskrnl/cc/pin.c b/ntoskrnl/cc/pin.c index af7b3aa71a8..c48f1068042 100644 --- a/ntoskrnl/cc/pin.c +++ b/ntoskrnl/cc/pin.c @@ -543,7 +543,6 @@ CcSetDirtyPinnedData ( IN PLARGE_INTEGER Lsn) { PINTERNAL_BCB iBcb = CONTAINING_RECORD(Bcb, INTERNAL_BCB, PFCB); - PROS_SHARED_CACHE_MAP SharedCacheMap = iBcb->Vacb->SharedCacheMap; CCTRACE(CC_API_DEBUG, "Bcb=%p Lsn=%p\n", Bcb, Lsn); @@ -556,10 +555,6 @@ CcSetDirtyPinnedData ( { CcRosMarkDirtyVacb(iBcb->Vacb); } - - /* Update VDL */ - if (SharedCacheMap->ValidDataLength.QuadPart < (iBcb->PFCB.MappedFileOffset.QuadPart + iBcb->PFCB.MappedLength)) - SharedCacheMap->ValidDataLength.QuadPart = iBcb->PFCB.MappedFileOffset.QuadPart + iBcb->PFCB.MappedLength; } diff --git a/ntoskrnl/cc/view.c b/ntoskrnl/cc/view.c index 591168eeba0..c0f1458287d 100644 --- a/ntoskrnl/cc/view.c +++ b/ntoskrnl/cc/view.c @@ -166,12 +166,12 @@ MmFlushVirtualMemory(IN PEPROCESS Process, NTSTATUS NTAPI CcRosFlushVacb ( - PROS_VACB Vacb) + _In_ PROS_VACB Vacb, + _In_ PIO_STATUS_BLOCK Iosb) { - IO_STATUS_BLOCK Iosb; - SIZE_T FlushSize = VACB_MAPPING_GRANULARITY; NTSTATUS Status; BOOLEAN HaveLock = FALSE; + PROS_SHARED_CACHE_MAP SharedCacheMap = Vacb->SharedCacheMap; CcRosUnmarkDirtyVacb(Vacb, TRUE); @@ -184,7 +184,10 @@ CcRosFlushVacb ( HaveLock = TRUE; } - Status = MmFlushVirtualMemory(NULL, &Vacb->BaseAddress, &FlushSize, &Iosb); + Status = MmFlushSegment(SharedCacheMap->FileObject->SectionObjectPointer, + &Vacb->FileOffset, + VACB_MAPPING_GRANULARITY, + Iosb); if (HaveLock) { @@ -194,6 +197,14 @@ CcRosFlushVacb ( quit: if (!NT_SUCCESS(Status)) CcRosMarkDirtyVacb(Vacb); + else + { + /* Update VDL */ + if (SharedCacheMap->ValidDataLength.QuadPart < (Vacb->FileOffset.QuadPart + VACB_MAPPING_GRANULARITY)) + { + SharedCacheMap->ValidDataLength.QuadPart = Vacb->FileOffset.QuadPart + VACB_MAPPING_GRANULARITY; + } + } return Status; } @@ -285,7 +296,8 @@ CcRosFlushDirtyPages ( continue; } - Status = CcRosFlushVacb(current); + IO_STATUS_BLOCK Iosb; + Status = CcRosFlushVacb(current, &Iosb); SharedCacheMap->Callbacks->ReleaseFromLazyWrite(SharedCacheMap->LazyWriteContext); @@ -308,7 +320,7 @@ CcRosFlushDirtyPages ( ULONG PagesFreed; /* How many pages did we free? */ - PagesFreed = VACB_MAPPING_GRANULARITY / PAGE_SIZE; + PagesFreed = Iosb.Information / PAGE_SIZE; (*Count) += PagesFreed; if (!Wait) @@ -756,7 +768,11 @@ CcRosEnsureVacbResident( if (!NoRead) { - NTSTATUS Status = MmMakePagesResident(NULL, BaseAddress, Length); + PROS_SHARED_CACHE_MAP SharedCacheMap = Vacb->SharedCacheMap; + NTSTATUS Status = MmMakeDataSectionResident(SharedCacheMap->FileObject->SectionObjectPointer, + Vacb->FileOffset.QuadPart + Offset, + Length, + &SharedCacheMap->ValidDataLength); if (!NT_SUCCESS(Status)) ExRaiseStatus(Status); } @@ -935,7 +951,6 @@ CcFlushCache ( } Status = STATUS_SUCCESS; - if (IoStatus) { IoStatus->Information = 0; @@ -953,9 +968,10 @@ CcFlushCache ( if (vacb != NULL) { + IO_STATUS_BLOCK VacbIosb; if (vacb->Dirty) { - Status = CcRosFlushVacb(vacb); + Status = CcRosFlushVacb(vacb, &VacbIosb); if (!NT_SUCCESS(Status)) { goto quit; @@ -966,7 +982,7 @@ CcFlushCache ( CcRosReleaseVacb(SharedCacheMap, vacb, FALSE, FALSE); if (IoStatus) - IoStatus->Information += VACB_MAPPING_GRANULARITY; + IoStatus->Information += VacbIosb.Information; } if (!DirtyVacb) @@ -992,6 +1008,10 @@ CcFlushCache ( if (IoStatus) IoStatus->Information += MmIosb.Information; + + /* Update VDL */ + if (SharedCacheMap->ValidDataLength.QuadPart < FlushEnd) + SharedCacheMap->ValidDataLength.QuadPart = FlushEnd; } if (!NT_SUCCESS(RtlLongLongAdd(FlushStart, VACB_MAPPING_GRANULARITY, &FlushStart))) diff --git a/ntoskrnl/include/internal/cc.h b/ntoskrnl/include/internal/cc.h index 7384d9be0d3..cf57319449b 100644 --- a/ntoskrnl/include/internal/cc.h +++ b/ntoskrnl/include/internal/cc.h @@ -310,7 +310,7 @@ CcMdlWriteComplete2( NTSTATUS NTAPI -CcRosFlushVacb(PROS_VACB Vacb); +CcRosFlushVacb(PROS_VACB Vacb, PIO_STATUS_BLOCK Iosb); NTSTATUS NTAPI diff --git a/ntoskrnl/include/internal/mm.h b/ntoskrnl/include/internal/mm.h index 0243ac55177..cec0d6e13d1 100644 --- a/ntoskrnl/include/internal/mm.h +++ b/ntoskrnl/include/internal/mm.h @@ -1369,13 +1369,6 @@ MmArePagesResident( _In_ PVOID BaseAddress, _In_ ULONG Length); -NTSTATUS -NTAPI -MmMakePagesResident( - _In_ PEPROCESS Process, - _In_ PVOID Address, - _In_ ULONG Length); - NTSTATUS NTAPI MmMakePagesDirty( @@ -1399,6 +1392,14 @@ MmFlushSegment( _In_ ULONG Length, _In_opt_ PIO_STATUS_BLOCK Iosb); +NTSTATUS +NTAPI +MmMakeDataSectionResident( + _In_ PSECTION_OBJECT_POINTERS SectionObjectPointer, + _In_ LONGLONG Offset, + _In_ ULONG Length, + _In_ PLARGE_INTEGER ValidDataLength); + BOOLEAN NTAPI MmPurgeSegment( diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index 94ea1844d2d..d848e337038 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -1181,127 +1181,235 @@ MiCopyFromUserPage(PFN_NUMBER DestPage, const VOID *SrcAddress) return(STATUS_SUCCESS); } -#ifndef NEWCC static NTSTATUS NTAPI -MiReadPage(PMEMORY_AREA MemoryArea, - LONGLONG SegOffset, - PPFN_NUMBER Page, - BOOLEAN IgnoreSize) -/* - * FUNCTION: Read a page for a section backed memory area. - * PARAMETERS: - * MemoryArea - Memory area to read the page for. - * Offset - Offset of the page to read. - * Page - Variable that receives a page contains the read data. - */ +MmMakeSegmentResident( + _In_ PMM_SECTION_SEGMENT Segment, + _In_ LONGLONG Offset, + _In_ ULONG Length, + _In_opt_ PLARGE_INTEGER ValidDataLength) { + /* Let's use a 64K granularity. */ + LONGLONG RangeStart, RangeEnd; NTSTATUS Status; - IO_STATUS_BLOCK IoStatus; - KEVENT Event; - UCHAR MdlBase[sizeof(MDL) + sizeof(PFN_NUMBER)]; - PMDL Mdl = (PMDL)MdlBase; - PFILE_OBJECT FileObject = MemoryArea->SectionData.Segment->FileObject; - LARGE_INTEGER FileOffset; - KIRQL OldIrql; - PFSRTL_ADVANCED_FCB_HEADER FcbHeader = FileObject->FsContext; + PFILE_OBJECT FileObject = Segment->FileObject; - FileOffset.QuadPart = MemoryArea->SectionData.Segment->Image.FileOffset + SegOffset; - - DPRINT("Reading file at offset %08x:%08x\n", FileOffset.HighPart, FileOffset.LowPart); - - Status = MmRequestPageMemoryConsumer(MC_USER, FALSE, Page); + /* Calculate our range, aligned on 64K if possible. */ + Status = RtlLongLongAdd(Offset, Length, &RangeEnd); + ASSERT(NT_SUCCESS(Status)); if (!NT_SUCCESS(Status)) return Status; - if ((FileOffset.QuadPart > FcbHeader->ValidDataLength.QuadPart) && !IgnoreSize) + RangeStart = Offset - (Offset % _64K); + if (RangeEnd % _64K) + RangeEnd += _64K - (RangeEnd % _64K); + + /* Clamp if needed */ + if (!FlagOn(*Segment->Flags, MM_DATAFILE_SEGMENT)) { - /* Quick path : data is not valid; return a zero-page */ - return STATUS_SUCCESS; + if (RangeEnd > Segment->RawLength.QuadPart) + RangeEnd = Segment->RawLength.QuadPart; } - RtlZeroMemory(MdlBase, sizeof(MdlBase)); - MmInitializeMdl(Mdl, NULL, PAGE_SIZE); - MmBuildMdlFromPages(Mdl, Page); - Mdl->MdlFlags |= MDL_PAGES_LOCKED | MDL_IO_PAGE_READ; - - KeInitializeEvent(&Event, NotificationEvent, FALSE); - - /* Disable APCs */ - KeRaiseIrql(APC_LEVEL, &OldIrql); - - Status = IoPageRead(FileObject, Mdl, &FileOffset, &Event, &IoStatus); - if (Status == STATUS_PENDING) + /* Let's gooooooooo */ + for ( ; RangeStart < RangeEnd; RangeStart += _64K) { - KeWaitForSingleObject(&Event, WrPageIn, KernelMode, FALSE, NULL); - Status = IoStatus.Status; + /* First take a look at where we miss pages */ + ULONG ToReadPageBits = 0; + LONGLONG ChunkEnd = RangeStart + _64K; + + if (ChunkEnd > RangeEnd) + ChunkEnd = RangeEnd; + + MmLockSectionSegment(Segment); + for (LONGLONG ChunkOffset = RangeStart; ChunkOffset < ChunkEnd; ChunkOffset += PAGE_SIZE) + { + LARGE_INTEGER CurrentOffset; + + CurrentOffset.QuadPart = ChunkOffset; + ULONG_PTR Entry = MmGetPageEntrySectionSegment(Segment, &CurrentOffset); + + /* Let any pending read proceed */ + while (MM_IS_WAIT_PTE(Entry)) + { + MmUnlockSectionSegment(Segment); + + KeDelayExecutionThread(KernelMode, FALSE, &TinyTime); + + MmLockSectionSegment(Segment); + Entry = MmGetPageEntrySectionSegment(Segment, &CurrentOffset); + } + + if (Entry != 0) + { + /* There is a page here. Or a swap entry. Or whatever... */ + continue; + } + + ToReadPageBits |= 1UL << ((ChunkOffset - RangeStart) >> PAGE_SHIFT); + + /* Put a wait entry here */ + MmSetPageEntrySectionSegment(Segment, &CurrentOffset, MAKE_SWAP_SSE(MM_WAIT_ENTRY)); + } + MmUnlockSectionSegment(Segment); + + if (ToReadPageBits == 0) + { + /* Nothing to do for this chunk */ + continue; + } + + /* Now perform the actual read */ + LONGLONG ChunkOffset = RangeStart; + while (ChunkOffset < ChunkEnd) + { + /* Move forward if there is a hole */ + ULONG BitSet; + if (!_BitScanForward(&BitSet, ToReadPageBits)) + { + /* Nothing more to read */ + break; + } + ToReadPageBits >>= BitSet; + ChunkOffset += BitSet * PAGE_SIZE; + ASSERT(ChunkOffset < ChunkEnd); + + /* Get the range we have to read */ + _BitScanForward(&BitSet, ~ToReadPageBits); + ULONG ReadLength = BitSet * PAGE_SIZE; + + ASSERT(ReadLength <= _64K); + + /* Clamp (This is for image mappings */ + if ((ChunkOffset + ReadLength) > ChunkEnd) + ReadLength = ChunkEnd - ChunkOffset; + + ASSERT(ReadLength != 0); + + /* Allocate a MDL */ + PMDL Mdl = IoAllocateMdl(NULL, ReadLength, FALSE, FALSE, NULL); + if (!Mdl) + { + /* Damn. Roll-back. */ + MmLockSectionSegment(Segment); + while (ChunkOffset < ChunkEnd) + { + if (ToReadPageBits & 1) + { + LARGE_INTEGER CurrentOffset; + CurrentOffset.QuadPart = ChunkOffset; + ASSERT(MM_IS_WAIT_PTE(MmGetPageEntrySectionSegment(Segment, &CurrentOffset))); + MmSetPageEntrySectionSegment(Segment, &CurrentOffset, 0); + } + ToReadPageBits >>= 1; + ChunkOffset += PAGE_SIZE; + } + MmUnlockSectionSegment(Segment); + return STATUS_INSUFFICIENT_RESOURCES; + } + + /* Get our pages */ + PPFN_NUMBER Pages = MmGetMdlPfnArray(Mdl); + RtlZeroMemory(Pages, BYTES_TO_PAGES(ReadLength) * sizeof(PFN_NUMBER)); + for (UINT i = 0; i < BYTES_TO_PAGES(ReadLength); i++) + { + /* MmRequestPageMemoryConsumer succeeds or bugchecks */ + (void)MmRequestPageMemoryConsumer(MC_USER, FALSE, &Pages[i]); + } + Mdl->MdlFlags |= MDL_PAGES_LOCKED | MDL_IO_PAGE_READ; + + LARGE_INTEGER FileOffset; + FileOffset.QuadPart = Segment->Image.FileOffset + ChunkOffset; + + /* Clamp to VDL */ + if (ValidDataLength && ((FileOffset.QuadPart + ReadLength) > ValidDataLength->QuadPart)) + { + if (FileOffset.QuadPart > ValidDataLength->QuadPart) + { + /* Great, nothing to read. */ + goto AssignPagesToSegment; + } + + Mdl->Size = (FileOffset.QuadPart + ReadLength) - ValidDataLength->QuadPart; + } + + KEVENT Event; + KeInitializeEvent(&Event, NotificationEvent, FALSE); + + /* Disable APCs */ + KIRQL OldIrql; + KeRaiseIrql(APC_LEVEL, &OldIrql); + + IO_STATUS_BLOCK Iosb; + Status = IoPageRead(FileObject, Mdl, &FileOffset, &Event, &Iosb); + if (Status == STATUS_PENDING) + { + KeWaitForSingleObject(&Event, WrPageIn, KernelMode, FALSE, NULL); + Status = Iosb.Status; + } + + if (Mdl->MdlFlags & MDL_MAPPED_TO_SYSTEM_VA) + { + MmUnmapLockedPages(Mdl->MappedSystemVa, Mdl); + } + + KeLowerIrql(OldIrql); + + if (Status == STATUS_END_OF_FILE) + { + DPRINT1("Got STATUS_END_OF_FILE at offset %I64d for file %wZ.\n", FileOffset.QuadPart, &FileObject->FileName); + Status = STATUS_SUCCESS; + } + + if (!NT_SUCCESS(Status)) + { + /* Damn. Roll back. */ + for (UINT i = 0; i < BYTES_TO_PAGES(ReadLength); i++) + MmReleasePageMemoryConsumer(MC_USER, Pages[i]); + + MmLockSectionSegment(Segment); + while (ChunkOffset < ChunkEnd) + { + if (ToReadPageBits & 1) + { + LARGE_INTEGER CurrentOffset; + CurrentOffset.QuadPart = ChunkOffset; + ASSERT(MM_IS_WAIT_PTE(MmGetPageEntrySectionSegment(Segment, &CurrentOffset))); + MmSetPageEntrySectionSegment(Segment, &CurrentOffset, 0); + } + ToReadPageBits >>= 1; + ChunkOffset += PAGE_SIZE; + } + MmUnlockSectionSegment(Segment); + IoFreeMdl(Mdl);; + return Status; + } + +AssignPagesToSegment: + MmLockSectionSegment(Segment); + + for (UINT i = 0; i < BYTES_TO_PAGES(ReadLength); i++) + { + LARGE_INTEGER CurrentOffset; + CurrentOffset.QuadPart = ChunkOffset + (i * PAGE_SIZE); + + ASSERT(MM_IS_WAIT_PTE(MmGetPageEntrySectionSegment(Segment, &CurrentOffset))); + + MmSetPageEntrySectionSegment(Segment, &CurrentOffset, MAKE_SSE(Pages[i] << PAGE_SHIFT, 0)); + } + + MmUnlockSectionSegment(Segment); + + IoFreeMdl(Mdl); + ToReadPageBits >>= BitSet; + ChunkOffset += BitSet * PAGE_SIZE; + } } - if (Mdl->MdlFlags & MDL_MAPPED_TO_SYSTEM_VA) - { - MmUnmapLockedPages (Mdl->MappedSystemVa, Mdl); - } - - KeLowerIrql(OldIrql); - - if (Status == STATUS_END_OF_FILE) - { - DPRINT1("Got STATUS_END_OF_FILE at offset %I64d for file %wZ.\n", SegOffset, &FileObject->FileName); - Status = STATUS_SUCCESS; - } - - if ((MemoryArea->VadNode.u.VadFlags.VadType == VadImageMap) - && ((SegOffset + PAGE_SIZE) > MemoryArea->SectionData.Segment->RawLength.QuadPart)) - { - KIRQL OldIrql; - PUCHAR PageMap; - - DPRINT("Zeroing at offset %I64d for file %wZ.\n", SegOffset, &FileObject->FileName); - - /* Zero out the end of it */ - PageMap = MiMapPageInHyperSpace(PsGetCurrentProcess(), *Page, &OldIrql); - RtlZeroMemory(PageMap + MemoryArea->SectionData.Segment->RawLength.QuadPart - SegOffset, - PAGE_SIZE - (MemoryArea->SectionData.Segment->RawLength.QuadPart - SegOffset)); - MiUnmapPageInHyperSpace(PsGetCurrentProcess(), PageMap, OldIrql); - } - - return Status; + return STATUS_SUCCESS; } -#else -NTSTATUS -NTAPI -MiReadPage(PMEMORY_AREA MemoryArea, - LONGLONG SegOffset, - PPFN_NUMBER Page) -/* - * FUNCTION: Read a page for a section backed memory area. - * PARAMETERS: - * MemoryArea - Memory area to read the page for. - * Offset - Offset of the page to read. - * Page - Variable that receives a page contains the read data. - */ -{ - MM_REQUIRED_RESOURCES Resources; - NTSTATUS Status; - - RtlZeroMemory(&Resources, sizeof(MM_REQUIRED_RESOURCES)); - - Resources.Context = MemoryArea->SectionData.Section->FileObject; - Resources.FileOffset.QuadPart = SegOffset + - MemoryArea->SectionData.Segment->Image.FileOffset; - Resources.Consumer = MC_USER; - Resources.Amount = PAGE_SIZE; - - DPRINT("%S, offset 0x%x, len 0x%x, page 0x%x\n", ((PFILE_OBJECT)Resources.Context)->FileName.Buffer, Resources.FileOffset.LowPart, Resources.Amount, Resources.Page[0]); - - Status = MiReadFilePage(MmGetKernelAddressSpace(), MemoryArea, &Resources); - *Page = Resources.Page[0]; - return Status; -} -#endif - static VOID MmAlterViewAttributes(PMMSUPPORT AddressSpace, PVOID BaseAddress, @@ -1610,85 +1718,57 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, if (Entry == 0) { - SWAPENTRY FakeSwapEntry; - /* - * If the entry is zero (and it can't change because we have - * locked the segment) then we need to load the page. + * If the entry is zero, then we need to load the page. */ - - /* - * Release all our locks and read in the page from disk - */ - MmSetPageEntrySectionSegment(Segment, &Offset, MAKE_SWAP_SSE(MM_WAIT_ENTRY)); - MmUnlockSectionSegment(Segment); - MmCreatePageFileMapping(Process, PAddress, MM_WAIT_ENTRY); - MmUnlockAddressSpace(AddressSpace); - if ((Offset.QuadPart >= (LONGLONG)PAGE_ROUND_UP(Segment->RawLength.QuadPart)) && (MemoryArea->VadNode.u.VadFlags.VadType == VadImageMap)) { + /* We are beyond the data which is on file. Just get a new page. */ MI_SET_USAGE(MI_USAGE_SECTION); if (Process) MI_SET_PROCESS2(Process->ImageFileName); if (!Process) MI_SET_PROCESS2("Kernel Section"); - Status = MmRequestPageMemoryConsumer(MC_USER, TRUE, &Page); - if (!NT_SUCCESS(Status)) - { - DPRINT1("MmRequestPageMemoryConsumer failed (Status %x)\n", Status); - } + MmRequestPageMemoryConsumer(MC_USER, FALSE, &Page); + MmSetPageEntrySectionSegment(Segment, &Offset, MAKE_SSE(Page << PAGE_SHIFT, 1)); + MmUnlockSectionSegment(Segment); - } - else - { - DPRINT("Getting fresh page for file %wZ at offset %I64d.\n", &Segment->FileObject->FileName, Offset.QuadPart); - Status = MiReadPage(MemoryArea, Offset.QuadPart, &Page, FALSE); + Status = MmCreateVirtualMapping(Process, PAddress, Attributes, &Page, 1); if (!NT_SUCCESS(Status)) { - DPRINT1("MiReadPage failed (Status %x)\n", Status); + DPRINT1("Unable to create virtual mapping\n"); + KeBugCheck(MEMORY_MANAGEMENT); } - } - if (!NT_SUCCESS(Status)) - { - /* - * FIXME: What do we know in this case? - */ - /* - * Cleanup and release locks - */ - MmLockAddressSpace(AddressSpace); + ASSERT(MmIsPagePresent(Process, PAddress)); + if (Process) + MmInsertRmap(Page, Process, Address); + MiSetPageEvent(Process, Address); DPRINT("Address 0x%p\n", Address); - return(Status); + return(STATUS_SUCCESS); } - /* Lock both segment and process address space while we proceed. */ - MmLockAddressSpace(AddressSpace); - MmLockSectionSegment(Segment); + MmUnlockSectionSegment(Segment); + MmUnlockAddressSpace(AddressSpace); - MmDeletePageFileMapping(Process, PAddress, &FakeSwapEntry); - DPRINT("CreateVirtualMapping Page %x Process %p PAddress %p Attributes %x\n", - Page, Process, PAddress, Attributes); - Status = MmCreateVirtualMapping(Process, - PAddress, - Attributes, - &Page, - 1); + /* The data must be paged in. Lock the file, so that the VDL doesn't get updated behind us. */ + FsRtlAcquireFileExclusive(Segment->FileObject); + + PFSRTL_COMMON_FCB_HEADER FcbHeader = Segment->FileObject->FsContext; + + Status = MmMakeSegmentResident(Segment, Offset.QuadPart, PAGE_SIZE, &FcbHeader->ValidDataLength); + + FsRtlReleaseFile(Segment->FileObject); + + /* Lock address space again */ + MmLockAddressSpace(AddressSpace); if (!NT_SUCCESS(Status)) { - DPRINT1("Unable to create virtual mapping\n"); - KeBugCheck(MEMORY_MANAGEMENT); + /* Damn */ + DPRINT1("Failed to page data in!\n"); + return STATUS_IN_PAGE_ERROR; } - ASSERT(MmIsPagePresent(Process, PAddress)); - if (Process) - MmInsertRmap(Page, Process, Address); - /* Set this section offset has being backed by our new page. */ - Entry = MAKE_SSE(Page << PAGE_SHIFT, 1); - MmSetPageEntrySectionSegment(Segment, &Offset, Entry); - MmUnlockSectionSegment(Segment); - - MiSetPageEvent(Process, Address); - DPRINT("Address 0x%p\n", Address); - return(STATUS_SUCCESS); + /* Everything went fine. Restart the operation */ + return STATUS_MM_RESTART_OPERATION; } else if (IS_SWAP_FROM_SSE(Entry)) { @@ -4458,103 +4538,6 @@ MmArePagesResident( return Ret; } -NTSTATUS -NTAPI -MmMakePagesResident( - _In_ PEPROCESS Process, - _In_ PVOID Address, - _In_ ULONG Length) -{ - PMEMORY_AREA MemoryArea; - PMM_SECTION_SEGMENT Segment; - LARGE_INTEGER SegmentOffset, RangeEnd; - PMMSUPPORT AddressSpace = Process ? &Process->Vm : MmGetKernelAddressSpace(); - - MmLockAddressSpace(AddressSpace); - - MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace, Address); - if (MemoryArea == NULL) - { - DPRINT1("Unable to find memory area at address %p.\n", Address); - MmUnlockAddressSpace(AddressSpace); - return STATUS_NOT_MAPPED_VIEW; - } - - /* Only supported in old Mm for now */ - ASSERT(MemoryArea->Type == MEMORY_AREA_SECTION_VIEW); - /* For file mappings */ - ASSERT(MemoryArea->VadNode.u.VadFlags.VadType != VadImageMap); - - Segment = MemoryArea->SectionData.Segment; - MmLockSectionSegment(Segment); - - SegmentOffset.QuadPart = PAGE_ROUND_DOWN(Address) - MA_GetStartingAddress(MemoryArea) - + MemoryArea->SectionData.ViewOffset.QuadPart; - RangeEnd.QuadPart = PAGE_ROUND_UP((ULONG_PTR)Address + Length) - MA_GetStartingAddress(MemoryArea) - + MemoryArea->SectionData.ViewOffset.QuadPart; - - DPRINT("MmMakePagesResident: Segment %p, 0x%I64x -> 0x%I64x\n", Segment, SegmentOffset.QuadPart, RangeEnd.QuadPart); - - while (SegmentOffset.QuadPart < RangeEnd.QuadPart) - { - ULONG_PTR Entry = MmGetPageEntrySectionSegment(Segment, &SegmentOffset); - - /* Let any pending read proceed */ - while (MM_IS_WAIT_PTE(Entry)) - { - MmUnlockSectionSegment(Segment); - MmUnlockAddressSpace(AddressSpace); - MiWaitForPageEvent(NULL, NULL); - MmLockAddressSpace(AddressSpace); - MmLockSectionSegment(Segment); - Entry = MmGetPageEntrySectionSegment(Segment, &SegmentOffset); - } - - /* We are called from Cc, this can't be backed by the page files */ - ASSERT(!IS_SWAP_FROM_SSE(Entry)); - - /* At this point, there may be a valid page there */ - if (Entry == 0) - { - PFN_NUMBER Page; - NTSTATUS Status; - - /* - * Release all our locks and read in the page from disk - */ - MmSetPageEntrySectionSegment(Segment, &SegmentOffset, MAKE_SWAP_SSE(MM_WAIT_ENTRY)); - MmUnlockSectionSegment(Segment); - MmUnlockAddressSpace(AddressSpace); - - /* FIXME: Read the whole range at once instead of one page at a time */ - /* Ignore file size, as Cc already checked on its side. */ - Status = MiReadPage(MemoryArea, SegmentOffset.QuadPart, &Page, TRUE); - if (!NT_SUCCESS(Status)) - { - /* Reset the Segment entry and fail */ - MmLockSectionSegment(Segment); - MmSetPageEntrySectionSegment(Segment, &SegmentOffset, 0); - MmUnlockSectionSegment(Segment); - MiSetPageEvent(Process, Address); - return Status; - } - - MmLockAddressSpace(AddressSpace); - MmLockSectionSegment(Segment); - - /* We set it with 0 ref count, nobody maps this page yet. */ - MmSetPageEntrySectionSegment(Segment, &SegmentOffset, MAKE_SSE(Page << PAGE_SHIFT, 0)); - MiSetPageEvent(Process, Address); - } - SegmentOffset.QuadPart += PAGE_SIZE; - } - - MmUnlockSectionSegment(Segment); - - MmUnlockAddressSpace(AddressSpace); - return STATUS_SUCCESS; -} - NTSTATUS NTAPI MmRosFlushVirtualMemory( @@ -4723,6 +4706,26 @@ MmPurgeSegment( return TRUE; } +NTSTATUS +NTAPI +MmMakeDataSectionResident( + _In_ PSECTION_OBJECT_POINTERS SectionObjectPointer, + _In_ LONGLONG Offset, + _In_ ULONG Length, + _In_ PLARGE_INTEGER ValidDataLength) +{ + PMM_SECTION_SEGMENT Segment = MiGrabDataSection(SectionObjectPointer); + + /* There must be a segment for this call */ + ASSERT(Segment); + + NTSTATUS Status = MmMakeSegmentResident(Segment, Offset, Length, ValidDataLength); + + MmDereferenceSegment(Segment); + + return Status; +} + NTSTATUS NTAPI MmFlushSegment( From 7f7abc983800d8d221fed2394e3c25528704f4c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Mon, 1 Feb 2021 12:02:53 +0100 Subject: [PATCH 95/99] [NTOS:MM] Simplify code & fix MSVC x64 build --- ntoskrnl/mm/rmap.c | 52 ++++++++++------------------------------------ 1 file changed, 11 insertions(+), 41 deletions(-) diff --git a/ntoskrnl/mm/rmap.c b/ntoskrnl/mm/rmap.c index e40d9554038..75bcc45a230 100644 --- a/ntoskrnl/mm/rmap.c +++ b/ntoskrnl/mm/rmap.c @@ -102,7 +102,7 @@ GetEntry: MmLockAddressSpace(AddressSpace); - if ((MmGetPfnForProcess(Process, Address) != Page) || MmIsPageAccessed(Process, Address)) + if (MmGetPfnForProcess(Process, Address) != Page) { /* This changed in the short window where we didn't have any locks */ MmUnlockAddressSpace(AddressSpace); @@ -152,31 +152,8 @@ GetEntry: /* Delete this virtual mapping in the process */ MmDeleteVirtualMapping(Process, Address, &Dirty, &MapPage); - /* There is a window betwwen the start of this function and now, - * where it's possible that the process changed its memory layout, - * because of copy-on-write, unmapping memory, or whatsoever. - * Just go away if that is the case */ - if (MapPage != Page) - { - PMM_REGION Region = MmFindRegion((PVOID)MA_GetStartingAddress(MemoryArea), - &MemoryArea->SectionData.RegionListHead, - Address, NULL); - /* Restore the mapping */ - MmCreateVirtualMapping(Process, Address, Region->Protect, &MapPage, 1); - if (Dirty) - MmSetDirtyPage(Process, Address); - - MmUnlockSectionSegment(Segment); - MmUnlockAddressSpace(AddressSpace); - if (Address < MmSystemRangeStart) - { - ExReleaseRundownProtection(&Process->RundownProtect); - ObDereferenceObject(Process); - } - - /* We can still try to flush it to disk, though */ - goto WriteSegment; - } + /* We checked this earlier */ + ASSERT(MapPage == Page); if (Page != PFN_FROM_SSE(Entry)) { @@ -232,11 +209,9 @@ GetEntry: MmSetDirtyPage(Process, Address); MmUnlockAddressSpace(AddressSpace); - if (Address < MmSystemRangeStart) - { - ExReleaseRundownProtection(&Process->RundownProtect); - ObDereferenceObject(Process); - } + ExReleaseRundownProtection(&Process->RundownProtect); + ObDereferenceObject(Process); + return STATUS_UNSUCCESSFUL; } } @@ -259,11 +234,9 @@ GetEntry: #endif MmReleasePageMemoryConsumer(MC_USER, Page); - if (Address < MmSystemRangeStart) - { - ExReleaseRundownProtection(&Process->RundownProtect); - ObDereferenceObject(Process); - } + ExReleaseRundownProtection(&Process->RundownProtect); + ObDereferenceObject(Process); + return STATUS_SUCCESS; } @@ -276,11 +249,8 @@ GetEntry: MmUnlockSectionSegment(Segment); MmUnlockAddressSpace(AddressSpace); - if (Address < MmSystemRangeStart) - { - ExReleaseRundownProtection(&Process->RundownProtect); - ObDereferenceObject(Process); - } + ExReleaseRundownProtection(&Process->RundownProtect); + ObDereferenceObject(Process); if (Released) return STATUS_SUCCESS; } From c0bf352069bd0c1b0d51da64c48fda70bb3ba92a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Mon, 1 Feb 2021 18:20:18 +0100 Subject: [PATCH 96/99] [NTOS:MM] Fix regression of MmCanFileBeTruncated --- ntoskrnl/mm/section.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index d848e337038..62da435122e 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -4115,7 +4115,7 @@ MmCanFileBeTruncated (IN PSECTION_OBJECT_POINTERS SectionObjectPointer, } MmLockSectionSegment(Segment); - if ((Segment->SectionCount == 1) && (SectionObjectPointer->SharedCacheMap != NULL)) + if ((Segment->SectionCount == 0) || ((Segment->SectionCount == 1) && (SectionObjectPointer->SharedCacheMap != NULL))) { /* If the cache is the only one holding a reference to the segment, then it's fine to resize */ Ret = TRUE; @@ -4124,6 +4124,12 @@ MmCanFileBeTruncated (IN PSECTION_OBJECT_POINTERS SectionObjectPointer, { /* We can't shrink, but we can extend */ Ret = NewFileSize->QuadPart >= Segment->RawLength.QuadPart; +#if DBG + if (!Ret) + { + DPRINT1("Cannot truncate data: New Size %I64d, Segment Size %I64d\n", NewFileSize->QuadPart, Segment->RawLength.QuadPart); + } +#endif } MmUnlockSectionSegment(Segment); MmDereferenceSegment(Segment); From b7eb0fddf3b675ab495c41e44861a924eded4730 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Tue, 2 Feb 2021 10:06:00 +0100 Subject: [PATCH 97/99] Address PR review --- ntoskrnl/cache/newcc.h | 4 - ntoskrnl/cc/cacheman.c | 1 - ntoskrnl/cc/view.c | 26 +------ ntoskrnl/include/internal/cc.h | 16 +--- ntoskrnl/include/internal/mm.h | 14 +--- ntoskrnl/mm/ARM3/section.c | 20 ++--- ntoskrnl/mm/rmap.c | 2 +- ntoskrnl/mm/section.c | 137 ++++++++++++++++----------------- 8 files changed, 85 insertions(+), 135 deletions(-) diff --git a/ntoskrnl/cache/newcc.h b/ntoskrnl/cache/newcc.h index 8768d7aebd2..801bdc08bb8 100644 --- a/ntoskrnl/cache/newcc.h +++ b/ntoskrnl/cache/newcc.h @@ -62,10 +62,6 @@ NTAPI CcpUnpinData(PNOCC_BCB Bcb, BOOLEAN ActuallyRelease); -BOOLEAN -NTAPI -CcInitializeCacheManager(VOID); - VOID NTAPI CcShutdownSystem(VOID); diff --git a/ntoskrnl/cc/cacheman.c b/ntoskrnl/cc/cacheman.c index fa519d43e7a..238fecb0f5a 100644 --- a/ntoskrnl/cc/cacheman.c +++ b/ntoskrnl/cc/cacheman.c @@ -42,7 +42,6 @@ CcPfInitializePrefetcher(VOID) CODE_SEG("INIT") BOOLEAN -NTAPI CcInitializeCacheManager(VOID) { ULONG Thread; diff --git a/ntoskrnl/cc/view.c b/ntoskrnl/cc/view.c index c0f1458287d..4c094f074d5 100644 --- a/ntoskrnl/cc/view.c +++ b/ntoskrnl/cc/view.c @@ -110,7 +110,6 @@ ULONG CcRosVacbGetRefCount_(PROS_VACB vacb, PCSTR file, INT line) /* FUNCTIONS *****************************************************************/ VOID -NTAPI CcRosTraceCacheMap ( PROS_SHARED_CACHE_MAP SharedCacheMap, BOOLEAN Trace ) @@ -157,14 +156,6 @@ CcRosTraceCacheMap ( } NTSTATUS -NTAPI -MmFlushVirtualMemory(IN PEPROCESS Process, - IN OUT PVOID *BaseAddress, - IN OUT PSIZE_T RegionSize, - OUT PIO_STATUS_BLOCK IoStatusBlock); - -NTSTATUS -NTAPI CcRosFlushVacb ( _In_ PROS_VACB Vacb, _In_ PIO_STATUS_BLOCK Iosb) @@ -210,7 +201,6 @@ quit: } NTSTATUS -NTAPI CcRosFlushDirtyPages ( ULONG Target, PULONG Count, @@ -349,7 +339,6 @@ CcRosFlushDirtyPages ( } NTSTATUS -NTAPI CcRosReleaseVacb ( PROS_SHARED_CACHE_MAP SharedCacheMap, PROS_VACB Vacb, @@ -382,7 +371,6 @@ CcRosReleaseVacb ( /* Returns with VACB Lock Held! */ PROS_VACB -NTAPI CcRosLookupVacb ( PROS_SHARED_CACHE_MAP SharedCacheMap, LONGLONG FileOffset) @@ -426,7 +414,6 @@ CcRosLookupVacb ( } VOID -NTAPI CcRosMarkDirtyVacb ( PROS_VACB Vacb) { @@ -463,7 +450,6 @@ CcRosMarkDirtyVacb ( } VOID -NTAPI CcRosUnmarkDirtyVacb ( PROS_VACB Vacb, BOOLEAN LockViews) @@ -617,7 +603,7 @@ CcRosCreateVacb ( Retried = FALSE; Retry: /* Map VACB in system space */ - Status = MmMapViewInSystemSpaceEx(SharedCacheMap->Section, ¤t->BaseAddress, &ViewSize, ¤t->FileOffset); + Status = MmMapViewInSystemSpaceEx(SharedCacheMap->Section, ¤t->BaseAddress, &ViewSize, ¤t->FileOffset, 0); if (!NT_SUCCESS(Status)) { @@ -735,7 +721,6 @@ Retry: } BOOLEAN -NTAPI CcRosEnsureVacbResident( _In_ PROS_VACB Vacb, _In_ BOOLEAN Wait, @@ -783,7 +768,6 @@ CcRosEnsureVacbResident( NTSTATUS -NTAPI CcRosGetVacb ( PROS_SHARED_CACHE_MAP SharedCacheMap, LONGLONG FileOffset, @@ -835,7 +819,6 @@ CcRosGetVacb ( } NTSTATUS -NTAPI CcRosRequestVacb ( PROS_SHARED_CACHE_MAP SharedCacheMap, LONGLONG FileOffset, @@ -1032,7 +1015,6 @@ quit: } NTSTATUS -NTAPI CcRosDeleteFileCache ( PFILE_OBJECT FileObject, PROS_SHARED_CACHE_MAP SharedCacheMap, @@ -1085,11 +1067,10 @@ CcRosDeleteFileCache ( /* Flush to disk, if needed */ if (Vacb->Dirty) { - SIZE_T FlushSize = VACB_MAPPING_GRANULARITY; IO_STATUS_BLOCK Iosb; NTSTATUS Status; - Status = MmFlushVirtualMemory(NULL, &Vacb->BaseAddress, &FlushSize, &Iosb); + Status = MmFlushSegment(FileObject->SectionObjectPointer, &Vacb->FileOffset, VACB_MAPPING_GRANULARITY, &Iosb); if (!NT_SUCCESS(Status)) { /* Complain. There's not much we can do */ @@ -1134,7 +1115,6 @@ CcRosDeleteFileCache ( } VOID -NTAPI CcRosReferenceCache ( PFILE_OBJECT FileObject) { @@ -1150,7 +1130,6 @@ CcRosReferenceCache ( } NTSTATUS -NTAPI CcRosReleaseFileCache ( PFILE_OBJECT FileObject) /* @@ -1207,7 +1186,6 @@ CcRosReleaseFileCache ( } NTSTATUS -NTAPI CcRosInitializeFileCache ( PFILE_OBJECT FileObject, PCC_FILE_SIZES FileSizes, diff --git a/ntoskrnl/include/internal/cc.h b/ntoskrnl/include/internal/cc.h index cf57319449b..52a7b19f0a6 100644 --- a/ntoskrnl/include/internal/cc.h +++ b/ntoskrnl/include/internal/cc.h @@ -3,7 +3,7 @@ // // Define this if you want debugging support // -#define _CC_DEBUG_ 0x0 +#define _CC_DEBUG_ 0x00 // // These define the Debug Masks Supported @@ -309,11 +309,9 @@ CcMdlWriteComplete2( ); NTSTATUS -NTAPI CcRosFlushVacb(PROS_VACB Vacb, PIO_STATUS_BLOCK Iosb); NTSTATUS -NTAPI CcRosGetVacb( PROS_SHARED_CACHE_MAP SharedCacheMap, LONGLONG FileOffset, @@ -321,7 +319,6 @@ CcRosGetVacb( ); BOOLEAN -NTAPI CcRosEnsureVacbResident( _In_ PROS_VACB Vacb, _In_ BOOLEAN Wait, @@ -339,11 +336,9 @@ NTAPI CcShutdownLazyWriter(VOID); BOOLEAN -NTAPI CcInitializeCacheManager(VOID); PROS_VACB -NTAPI CcRosLookupVacb( PROS_SHARED_CACHE_MAP SharedCacheMap, LONGLONG FileOffset @@ -354,18 +349,15 @@ NTAPI CcInitCacheZeroPage(VOID); VOID -NTAPI CcRosMarkDirtyVacb( PROS_VACB Vacb); VOID -NTAPI CcRosUnmarkDirtyVacb( PROS_VACB Vacb, BOOLEAN LockViews); NTSTATUS -NTAPI CcRosFlushDirtyPages( ULONG Target, PULONG Count, @@ -374,15 +366,12 @@ CcRosFlushDirtyPages( ); VOID -NTAPI CcRosDereferenceCache(PFILE_OBJECT FileObject); VOID -NTAPI CcRosReferenceCache(PFILE_OBJECT FileObject); NTSTATUS -NTAPI CcRosReleaseVacb( PROS_SHARED_CACHE_MAP SharedCacheMap, PROS_VACB Vacb, @@ -391,7 +380,6 @@ CcRosReleaseVacb( ); NTSTATUS -NTAPI CcRosRequestVacb( PROS_SHARED_CACHE_MAP SharedCacheMap, LONGLONG FileOffset, @@ -399,7 +387,6 @@ CcRosRequestVacb( ); NTSTATUS -NTAPI CcRosInitializeFileCache( PFILE_OBJECT FileObject, PCC_FILE_SIZES FileSizes, @@ -409,7 +396,6 @@ CcRosInitializeFileCache( ); NTSTATUS -NTAPI CcRosReleaseFileCache( PFILE_OBJECT FileObject ); diff --git a/ntoskrnl/include/internal/mm.h b/ntoskrnl/include/internal/mm.h index cec0d6e13d1..a661aa469f3 100644 --- a/ntoskrnl/include/internal/mm.h +++ b/ntoskrnl/include/internal/mm.h @@ -224,7 +224,7 @@ typedef struct _MEMORY_AREA struct { - LARGE_INTEGER ViewOffset; + LONGLONG ViewOffset; PMM_SECTION_SEGMENT Segment; LIST_ENTRY RegionListHead; } SectionData; @@ -770,15 +770,6 @@ MmAccessFault( IN PVOID TrapInformation ); -/* kmap.c ********************************************************************/ - -NTSTATUS -NTAPI -MiCopyFromUserPage( - PFN_NUMBER DestPage, - const VOID *SrcAddress -); - /* process.c *****************************************************************/ PVOID @@ -1359,7 +1350,8 @@ MmMapViewInSystemSpaceEx ( _In_ PVOID Section, _Outptr_result_bytebuffer_ (*ViewSize) PVOID *MappedBase, _Inout_ PSIZE_T ViewSize, - _Inout_ PLARGE_INTEGER SectionOffset + _Inout_ PLARGE_INTEGER SectionOffset, + _In_ ULONG_PTR Flags ); BOOLEAN diff --git a/ntoskrnl/mm/ARM3/section.c b/ntoskrnl/mm/ARM3/section.c index aeccb2adf28..3e6a25b44db 100644 --- a/ntoskrnl/mm/ARM3/section.c +++ b/ntoskrnl/mm/ARM3/section.c @@ -412,19 +412,19 @@ MiInsertInSystemSpace(IN PMMSESSION Session, return Base; } +static NTSTATUS -NTAPI MiAddMappedPtes(IN PMMPTE FirstPte, IN PFN_NUMBER PteCount, IN PCONTROL_AREA ControlArea, - IN PLARGE_INTEGER SectionOffset) + IN LONGLONG SectionOffset) { MMPTE TempPte; PMMPTE PointerPte, ProtoPte, LastProtoPte, LastPte; PSUBSECTION Subsection; /* Mapping at offset not supported yet */ - ASSERT(SectionOffset->QuadPart == 0); + ASSERT(SectionOffset == 0); /* ARM3 doesn't support this yet */ ASSERT(ControlArea->u.Flags.GlobalOnlyPerSession == 0); @@ -1054,12 +1054,12 @@ _WARN("MiSessionCommitPageTables halfplemented for amd64") } NTSTATUS -NTAPI -MiMapViewInSystemSpace(IN PVOID Section, - IN PMMSESSION Session, - OUT PVOID *MappedBase, - IN OUT PSIZE_T ViewSize, - IN PLARGE_INTEGER SectionOffset) +MiMapViewInSystemSpace( + _In_ PVOID Section, + _In_ PMMSESSION Session, + _Outptr_result_bytebuffer_ (*ViewSize) PVOID *MappedBase, + _Inout_ PSIZE_T ViewSize, + _Inout_ PLARGE_INTEGER SectionOffset) { PVOID Base; PCONTROL_AREA ControlArea; @@ -1156,7 +1156,7 @@ MiMapViewInSystemSpace(IN PVOID Section, Status = MiAddMappedPtes(MiAddressToPte(Base), BYTES_TO_PAGES(*ViewSize), ControlArea, - SectionOffset); + SectionOffset->QuadPart); ASSERT(NT_SUCCESS(Status)); /* Return the base adress of the mapping and success */ diff --git a/ntoskrnl/mm/rmap.c b/ntoskrnl/mm/rmap.c index 75bcc45a230..cb3f956ef35 100644 --- a/ntoskrnl/mm/rmap.c +++ b/ntoskrnl/mm/rmap.c @@ -128,7 +128,7 @@ GetEntry: LARGE_INTEGER Offset; BOOLEAN Released; - Offset.QuadPart = MemoryArea->SectionData.ViewOffset.QuadPart + + Offset.QuadPart = MemoryArea->SectionData.ViewOffset + ((ULONG_PTR)Address - MA_GetStartingAddress(MemoryArea)); Segment = MemoryArea->SectionData.Segment; diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index 62da435122e..3f364b49f4d 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -125,7 +125,6 @@ PMM_IMAGE_SECTION_OBJECT ImageSectionObjectFromSegment(PMM_SECTION_SEGMENT Segme } NTSTATUS -NTAPI MiMapViewInSystemSpace(IN PVOID Section, IN PVOID Session, OUT PVOID *MappedBase, @@ -1160,8 +1159,8 @@ MmUnsharePageEntrySectionSegment(PMEMORY_AREA MemoryArea, return TRUE; } +static NTSTATUS -NTAPI MiCopyFromUserPage(PFN_NUMBER DestPage, const VOID *SrcAddress) { PEPROCESS Process; @@ -1172,13 +1171,13 @@ MiCopyFromUserPage(PFN_NUMBER DestPage, const VOID *SrcAddress) DestAddress = MiMapPageInHyperSpace(Process, DestPage, &Irql); if (DestAddress == NULL) { - return(STATUS_NO_MEMORY); + return STATUS_NO_MEMORY; } ASSERT((ULONG_PTR)DestAddress % PAGE_SIZE == 0); ASSERT((ULONG_PTR)SrcAddress % PAGE_SIZE == 0); RtlCopyMemory(DestAddress, SrcAddress, PAGE_SIZE); MiUnmapPageInHyperSpace(Process, DestAddress, Irql); - return(STATUS_SUCCESS); + return STATUS_SUCCESS; } static @@ -1465,7 +1464,7 @@ MmAlterViewAttributes(PMMSUPPORT AddressSpace, PFN_NUMBER Page; Offset.QuadPart = (ULONG_PTR)Address - MA_GetStartingAddress(MemoryArea) - + MemoryArea->SectionData.ViewOffset.QuadPart; + + MemoryArea->SectionData.ViewOffset; Entry = MmGetPageEntrySectionSegment(Segment, &Offset); /* * An MM_WAIT_ENTRY is ok in this case... It'll just count as @@ -1518,12 +1517,12 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, */ if (MmIsPagePresent(Process, Address)) { - return(STATUS_SUCCESS); + return STATUS_SUCCESS; } if (MmIsDisabledPage(Process, Address)) { - return(STATUS_ACCESS_VIOLATION); + return STATUS_ACCESS_VIOLATION; } /* @@ -1531,12 +1530,12 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, */ if (MemoryArea->DeleteInProgress) { - return(STATUS_UNSUCCESSFUL); + return STATUS_UNSUCCESSFUL; } PAddress = MM_ROUND_DOWN(Address, PAGE_SIZE); Offset.QuadPart = (ULONG_PTR)PAddress - MA_GetStartingAddress(MemoryArea) - + MemoryArea->SectionData.ViewOffset.QuadPart; + + MemoryArea->SectionData.ViewOffset; Segment = MemoryArea->SectionData.Segment; Region = MmFindRegion((PVOID)MA_GetStartingAddress(MemoryArea), @@ -1596,7 +1595,7 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, MiWaitForPageEvent(NULL, NULL); MmLockAddressSpace(AddressSpace); DPRINT("Address 0x%p\n", Address); - return(STATUS_MM_RESTART_OPERATION); + return STATUS_MM_RESTART_OPERATION; } HasSwapEntry = MmIsPageSwapEntry(Process, Address); @@ -1660,7 +1659,7 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, { DPRINT("MmCreateVirtualMapping failed, not out of memory\n"); KeBugCheck(MEMORY_MANAGEMENT); - return(Status); + return Status; } /* @@ -1678,7 +1677,7 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, */ MiSetPageEvent(Process, Address); DPRINT("Address 0x%p\n", Address); - return(STATUS_SUCCESS); + return STATUS_SUCCESS; } /* @@ -1700,7 +1699,7 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, { DPRINT("MmCreateVirtualMappingUnsafe failed, not out of memory\n"); KeBugCheck(MEMORY_MANAGEMENT); - return(Status); + return Status; } /* @@ -1708,7 +1707,7 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, */ MiSetPageEvent(Process, Address); DPRINT("Address 0x%p\n", Address); - return(STATUS_SUCCESS); + return STATUS_SUCCESS; } /* @@ -1743,7 +1742,7 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, MiSetPageEvent(Process, Address); DPRINT("Address 0x%p\n", Address); - return(STATUS_SUCCESS); + return STATUS_SUCCESS; } MmUnlockSectionSegment(Segment); @@ -1853,7 +1852,7 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, MiSetPageEvent(Process, Address); DPRINT("Address 0x%p\n", Address); - return(STATUS_SUCCESS); + return STATUS_SUCCESS; } else { @@ -1880,7 +1879,7 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, MiSetPageEvent(Process, Address); DPRINT("Address 0x%p\n", Address); - return(STATUS_SUCCESS); + return STATUS_SUCCESS; } } @@ -1916,7 +1915,7 @@ MmAccessFaultSectionView(PMMSUPPORT AddressSpace, if (MmGetPageProtect(Process, Address) & PAGE_READWRITE) { DPRINT("Address 0x%p\n", Address); - return(STATUS_SUCCESS); + return STATUS_SUCCESS; } /* @@ -1924,7 +1923,7 @@ MmAccessFaultSectionView(PMMSUPPORT AddressSpace, */ PAddress = MM_ROUND_DOWN(Address, PAGE_SIZE); Offset.QuadPart = (ULONG_PTR)PAddress - MA_GetStartingAddress(MemoryArea) - + MemoryArea->SectionData.ViewOffset.QuadPart; + + MemoryArea->SectionData.ViewOffset; Segment = MemoryArea->SectionData.Segment; Region = MmFindRegion((PVOID)MA_GetStartingAddress(MemoryArea), @@ -1940,7 +1939,7 @@ MmAccessFaultSectionView(PMMSUPPORT AddressSpace, Region->Protect == PAGE_EXECUTE_READWRITE))) { DPRINT("Address 0x%p\n", Address); - return(STATUS_ACCESS_VIOLATION); + return STATUS_ACCESS_VIOLATION; } /* Get the page mapping this section offset. */ @@ -1958,7 +1957,7 @@ MmAccessFaultSectionView(PMMSUPPORT AddressSpace, MmUnlockSectionSegment(Segment); /* This is a private page. We must only change the page protection. */ MmSetPageProtect(Process, PAddress, Region->Protect); - return(STATUS_SUCCESS); + return STATUS_SUCCESS; } /* @@ -2000,7 +1999,7 @@ MmAccessFaultSectionView(PMMSUPPORT AddressSpace, { DPRINT1("MmCreateVirtualMapping failed, unable to create virtual mapping, not out of memory\n"); KeBugCheck(MEMORY_MANAGEMENT); - return(Status); + return Status; } if (Process) @@ -2008,7 +2007,7 @@ MmAccessFaultSectionView(PMMSUPPORT AddressSpace, MiSetPageEvent(Process, Address); DPRINT("Address 0x%p\n", Address); - return(STATUS_SUCCESS); + return STATUS_SUCCESS; } NTSTATUS @@ -2045,7 +2044,7 @@ MmProtectSectionView(PMMSUPPORT AddressSpace, BaseAddress, Length, Region->Type, Protect, MmAlterViewAttributes); - return(Status); + return Status; } NTSTATUS NTAPI @@ -2084,7 +2083,7 @@ MmQuerySectionView(PMEMORY_AREA MemoryArea, Info->Protect = Region->Protect; *ResultLength = sizeof(MEMORY_BASIC_INFORMATION); - return(STATUS_SUCCESS); + return STATUS_SUCCESS; } VOID NTAPI @@ -2180,7 +2179,7 @@ MmCreatePhysicalMemorySection(VOID) if (!NT_SUCCESS(Status)) { DPRINT1("MmCreatePhysicalMemorySection: failed to create object (0x%lx)\n", Status); - return(Status); + return Status; } /* @@ -2198,7 +2197,7 @@ MmCreatePhysicalMemorySection(VOID) if (Segment == NULL) { ObDereferenceObject(PhysSection); - return(STATUS_NO_MEMORY); + return STATUS_NO_MEMORY; } RtlZeroMemory(Segment, sizeof(MM_SECTION_SEGMENT)); PhysSection->Segment = (PSEGMENT)Segment; @@ -2231,7 +2230,7 @@ MmCreatePhysicalMemorySection(VOID) } ObCloseHandle(Handle, KernelMode); - return(STATUS_SUCCESS); + return STATUS_SUCCESS; } CODE_SEG("INIT") @@ -2264,7 +2263,7 @@ MmInitSectionImplementation(VOID) MmCreatePhysicalMemorySection(); - return(STATUS_SUCCESS); + return STATUS_SUCCESS; } static @@ -2302,7 +2301,7 @@ MmCreateDataFileSection(PSECTION *SectionObject, (PVOID*)&Section); if (!NT_SUCCESS(Status)) { - return(Status); + return Status; } /* * Initialize it @@ -2363,7 +2362,7 @@ MmCreateDataFileSection(PSECTION *SectionObject, if (!NT_SUCCESS(Status)) { ObDereferenceObject(Section); - return(STATUS_SECTION_NOT_EXTENDED); + return STATUS_SECTION_NOT_EXTENDED; } } } @@ -2381,7 +2380,7 @@ MmCreateDataFileSection(PSECTION *SectionObject, if (Status != STATUS_SUCCESS) { ObDereferenceObject(Section); - return(Status); + return Status; } /* Lock the PFN lock while messing with Section Object pointers */ @@ -2390,10 +2389,8 @@ MmCreateDataFileSection(PSECTION *SectionObject, while (Segment && (Segment->SegFlags & (MM_SEGMENT_INDELETE | MM_SEGMENT_INCREATE))) { - LARGE_INTEGER ShortTime = {{-10 * 100 * 1000, -1}}; - MiReleasePfnLock(OldIrql); - KeDelayExecutionThread(KernelMode, FALSE, &ShortTime); + KeDelayExecutionThread(KernelMode, FALSE, &TinyTime); OldIrql = MiAcquirePfnLock(); Segment = FileObject->SectionObjectPointer->DataSectionObject; } @@ -2411,7 +2408,7 @@ MmCreateDataFileSection(PSECTION *SectionObject, //KeSetEvent((PVOID)&FileObject->Lock, IO_NO_INCREMENT, FALSE); MiReleasePfnLock(OldIrql); ObDereferenceObject(Section); - return(STATUS_NO_MEMORY); + return STATUS_NO_MEMORY; } /* We are creating it */ @@ -2481,7 +2478,7 @@ MmCreateDataFileSection(PSECTION *SectionObject, //KeSetEvent((PVOID)&FileObject->Lock, IO_NO_INCREMENT, FALSE); *SectionObject = Section; - return(STATUS_SUCCESS); + return STATUS_SUCCESS; } /* @@ -3145,7 +3142,7 @@ MmCreateImageSection(PSECTION *SectionObject, (PVOID*)(PVOID)&Section); if (!NT_SUCCESS(Status)) { - return(Status); + return Status; } /* @@ -3168,12 +3165,9 @@ MmCreateImageSection(PSECTION *SectionObject, ImageSectionObject = FileObject->SectionObjectPointer->ImageSectionObject; while(ImageSectionObject && (ImageSectionObject->SegFlags & (MM_SEGMENT_INDELETE | MM_SEGMENT_INCREATE))) { - LARGE_INTEGER ShortTime; - MiReleasePfnLock(OldIrql); - ShortTime.QuadPart = - 10 * 100 * 1000; - KeDelayExecutionThread(KernelMode, FALSE, &ShortTime); + KeDelayExecutionThread(KernelMode, FALSE, &TinyTime); OldIrql = MiAcquirePfnLock(); ImageSectionObject = FileObject->SectionObjectPointer->ImageSectionObject; @@ -3188,7 +3182,7 @@ MmCreateImageSection(PSECTION *SectionObject, { MiReleasePfnLock(OldIrql); ObDereferenceObject(Section); - return(STATUS_NO_MEMORY); + return STATUS_NO_MEMORY; } ImageSectionObject->SegFlags = MM_SEGMENT_INCREATE; @@ -3223,7 +3217,7 @@ MmCreateImageSection(PSECTION *SectionObject, ExFreePoolWithTag(ImageSectionObject, TAG_MM_SECTION_SEGMENT); ObDereferenceObject(Section); - return(Status); + return Status; } Section->Segment = (PSEGMENT)ImageSectionObject; @@ -3244,7 +3238,7 @@ MmCreateImageSection(PSECTION *SectionObject, ExFreePool(ImageSectionObject->Segments); ExFreePool(ImageSectionObject); ObDereferenceObject(Section); - return(Status); + return Status; } OldIrql = MiAcquirePfnLock(); @@ -3272,7 +3266,7 @@ MmCreateImageSection(PSECTION *SectionObject, *SectionObject = Section; ASSERT(ImageSectionObject->RefCount > 0); - return(Status); + return Status; } @@ -3337,13 +3331,13 @@ MmMapViewOfSegment( { DPRINT1("Mapping between 0x%p and 0x%p failed (%X).\n", (*BaseAddress), (char*)(*BaseAddress) + ViewSize, Status); - return(Status); + return Status; } InterlockedIncrement64(Segment->ReferenceCount); MArea->SectionData.Segment = Segment; - MArea->SectionData.ViewOffset.QuadPart = ViewOffset; + MArea->SectionData.ViewOffset = ViewOffset; if (AsImage) { MArea->VadNode.u.VadFlags.VadType = VadImageMap; @@ -3352,7 +3346,7 @@ MmMapViewOfSegment( MmInitializeRegion(&MArea->SectionData.RegionListHead, ViewSize, 0, Protect); - return(STATUS_SUCCESS); + return STATUS_SUCCESS; } @@ -3373,7 +3367,7 @@ MmFreeSectionPage(PVOID Context, MEMORY_AREA* MemoryArea, PVOID Address, Address = (PVOID)PAGE_ROUND_DOWN(Address); Offset.QuadPart = ((ULONG_PTR)Address - MA_GetStartingAddress(MemoryArea)) + - MemoryArea->SectionData.ViewOffset.QuadPart; + MemoryArea->SectionData.ViewOffset; Segment = MemoryArea->SectionData.Segment; @@ -3455,7 +3449,7 @@ MmUnmapViewOfSegment(PMMSUPPORT AddressSpace, BaseAddress); if (MemoryArea == NULL) { - return(STATUS_UNSUCCESSFUL); + return STATUS_UNSUCCESSFUL; } Segment = MemoryArea->SectionData.Segment; @@ -3499,7 +3493,7 @@ MmUnmapViewOfSegment(PMMSUPPORT AddressSpace, } MmUnlockSectionSegment(Segment); MmDereferenceSegment(Segment); - return(Status); + return Status; } /* This functions must be called with a locked address space */ @@ -3596,7 +3590,7 @@ MiRosUnmapViewOfSection(IN PEPROCESS Process, /* Notify debugger */ if (ImageBaseAddress && !SkipDebuggerNotify) DbgkUnMapViewOfSection(ImageBaseAddress); - return(STATUS_SUCCESS); + return STATUS_SUCCESS; } @@ -3801,7 +3795,7 @@ NtQuerySection( ObDereferenceObject(Section); - return(Status); + return Status; } /********************************************************************** @@ -3959,14 +3953,14 @@ MmMapViewOfSection(IN PVOID SectionObject, if ((*BaseAddress) != NULL) { MmUnlockAddressSpace(AddressSpace); - return(STATUS_CONFLICTING_ADDRESSES); + return STATUS_CONFLICTING_ADDRESSES; } /* Otherwise find a gap to map the image. */ ImageBase = (ULONG_PTR)MmFindGap(AddressSpace, PAGE_ROUND_UP(ImageSize), MM_VIRTMEM_GRANULARITY, FALSE); if (ImageBase == 0) { MmUnlockAddressSpace(AddressSpace); - return(STATUS_CONFLICTING_ADDRESSES); + return STATUS_CONFLICTING_ADDRESSES; } /* Remember that we loaded image at a different base address */ NotAtBase = TRUE; @@ -3998,7 +3992,7 @@ MmMapViewOfSection(IN PVOID SectionObject, } MmUnlockAddressSpace(AddressSpace); - return(Status); + return Status; } } @@ -4046,7 +4040,7 @@ MmMapViewOfSection(IN PVOID SectionObject, if ((ViewOffset % PAGE_SIZE) != 0) { MmUnlockAddressSpace(AddressSpace); - return(STATUS_MAPPED_ALIGNMENT); + return STATUS_MAPPED_ALIGNMENT; } if ((*ViewSize) == 0) @@ -4076,7 +4070,7 @@ MmMapViewOfSection(IN PVOID SectionObject, if (!NT_SUCCESS(Status)) { MmUnlockAddressSpace(AddressSpace); - return(Status); + return Status; } } @@ -4115,7 +4109,8 @@ MmCanFileBeTruncated (IN PSECTION_OBJECT_POINTERS SectionObjectPointer, } MmLockSectionSegment(Segment); - if ((Segment->SectionCount == 0) || ((Segment->SectionCount == 1) && (SectionObjectPointer->SharedCacheMap != NULL))) + if ((Segment->SectionCount == 0) || + ((Segment->SectionCount == 1) && (SectionObjectPointer->SharedCacheMap != NULL))) { /* If the cache is the only one holding a reference to the segment, then it's fine to resize */ Ret = TRUE; @@ -4171,7 +4166,8 @@ MmFlushImageSection (IN PSECTION_OBJECT_POINTERS SectionObjectPointer, /* * @implemented */ -NTSTATUS NTAPI +NTSTATUS +NTAPI MmMapViewInSystemSpace (IN PVOID SectionObject, OUT PVOID * MappedBase, IN OUT PSIZE_T ViewSize) @@ -4180,7 +4176,7 @@ MmMapViewInSystemSpace (IN PVOID SectionObject, SectionOffset.QuadPart = 0; - return MmMapViewInSystemSpaceEx(SectionObject, MappedBase, ViewSize, &SectionOffset); + return MmMapViewInSystemSpaceEx(SectionObject, MappedBase, ViewSize, &SectionOffset, 0); } NTSTATUS @@ -4189,7 +4185,8 @@ MmMapViewInSystemSpaceEx ( _In_ PVOID SectionObject, _Outptr_result_bytebuffer_ (*ViewSize) PVOID *MappedBase, _Inout_ PSIZE_T ViewSize, - _Inout_ PLARGE_INTEGER SectionOffset + _Inout_ PLARGE_INTEGER SectionOffset, + _In_ ULONG_PTR Flags ) { PSECTION Section = SectionObject; @@ -4197,6 +4194,8 @@ MmMapViewInSystemSpaceEx ( PMMSUPPORT AddressSpace; NTSTATUS Status; + UNREFERENCED_PARAMETER(Flags); + PAGED_CODE(); if (MiIsRosSectionObject(SectionObject) == FALSE) @@ -4523,9 +4522,9 @@ MmArePagesResident( MmLockSectionSegment(Segment); SegmentOffset.QuadPart = PAGE_ROUND_DOWN(Address) - MA_GetStartingAddress(MemoryArea) - + MemoryArea->SectionData.ViewOffset.QuadPart; + + MemoryArea->SectionData.ViewOffset; RangeEnd.QuadPart = PAGE_ROUND_UP((ULONG_PTR)Address + Length) - MA_GetStartingAddress(MemoryArea) - + MemoryArea->SectionData.ViewOffset.QuadPart; + + MemoryArea->SectionData.ViewOffset; while (SegmentOffset.QuadPart < RangeEnd.QuadPart) { @@ -4576,9 +4575,9 @@ MmRosFlushVirtualMemory( Segment = MemoryArea->SectionData.Segment; SegmentOffset.QuadPart = PAGE_ROUND_DOWN(*Address) - MA_GetStartingAddress(MemoryArea) - + MemoryArea->SectionData.ViewOffset.QuadPart; + + MemoryArea->SectionData.ViewOffset; RangeEnd.QuadPart = PAGE_ROUND_UP((ULONG_PTR)*Address + *Length) - MA_GetStartingAddress(MemoryArea) - + MemoryArea->SectionData.ViewOffset.QuadPart; + + MemoryArea->SectionData.ViewOffset; CurrentAddress = *Address; @@ -4990,9 +4989,9 @@ MmMakePagesDirty( MmLockSectionSegment(Segment); SegmentOffset.QuadPart = PAGE_ROUND_DOWN(Address) - MA_GetStartingAddress(MemoryArea) - + MemoryArea->SectionData.ViewOffset.QuadPart; + + MemoryArea->SectionData.ViewOffset; RangeEnd.QuadPart = PAGE_ROUND_UP((ULONG_PTR)Address + Length) - MA_GetStartingAddress(MemoryArea) - + MemoryArea->SectionData.ViewOffset.QuadPart; + + MemoryArea->SectionData.ViewOffset; DPRINT("MmMakePagesResident: Segment %p, 0x%I64x -> 0x%I64x\n", Segment, SegmentOffset.QuadPart, RangeEnd.QuadPart); From 14077e3f4e3e6048484a2e32e7625d869b3458cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Wed, 3 Feb 2021 09:41:04 +0100 Subject: [PATCH 98/99] [NTOS:MM] Simplify a check --- ntoskrnl/cache/section/sptab.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ntoskrnl/cache/section/sptab.c b/ntoskrnl/cache/section/sptab.c index 7d960a1ff08..0877fb085f6 100644 --- a/ntoskrnl/cache/section/sptab.c +++ b/ntoskrnl/cache/section/sptab.c @@ -225,7 +225,7 @@ _MmSetPageEntrySectionSegment(PMM_SECTION_SEGMENT Segment, MmSetSectionAssociation(PFN_FROM_SSE(Entry), Segment, Offset); InterlockedIncrement64(Segment->ReferenceCount); - if ((Offset->QuadPart > (Segment->LastPage << PAGE_SHIFT)) || !Segment->LastPage) + if (Offset->QuadPart >= (Segment->LastPage << PAGE_SHIFT)) Segment->LastPage = (Offset->QuadPart >> PAGE_SHIFT) + 1; } } From 0699dcb50cefd64015392ba2b74754d2b3f0c573 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Wed, 3 Feb 2021 10:29:28 +0100 Subject: [PATCH 99/99] [NTOS:EX] Restore sysinfo --- ntoskrnl/ex/sysinfo.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ntoskrnl/ex/sysinfo.c b/ntoskrnl/ex/sysinfo.c index f595623b4c8..ffb8ddc906d 100644 --- a/ntoskrnl/ex/sysinfo.c +++ b/ntoskrnl/ex/sysinfo.c @@ -278,7 +278,7 @@ ExpGetRawSMBiosTable( DPRINT1("IoWMIOpenBlock failed: 0x%08lx\n", Status); return Status; } - + AllData = ExAllocatePoolWithTag(PagedPool, WMIBufSize, 'itfS'); if (AllData == NULL) { @@ -766,7 +766,7 @@ QSI_DEF(SystemPerformanceInformation) Spi->TotalSystemDriverPages = 0; /* FIXME */ Spi->Spare3Count = 0; /* FIXME */ - Spi->ResidentSystemCachePage = 0; /* FIXME */ + Spi->ResidentSystemCachePage = MiMemoryConsumers[MC_USER].PagesUsed; /* FIXME */ Spi->ResidentPagedPoolPage = 0; /* FIXME */ Spi->ResidentSystemDriverPage = 0; /* FIXME */ @@ -1476,10 +1476,10 @@ QSI_DEF(SystemFileCacheInformation) RtlZeroMemory(Sci, sizeof(SYSTEM_FILECACHE_INFORMATION)); /* Return the Byte size not the page size. */ - Sci->CurrentSize = 0; /* FIXME */ - Sci->PeakSize = 0; /* FIXME */ + Sci->CurrentSize = MiMemoryConsumers[MC_USER].PagesUsed; /* FIXME */ + Sci->PeakSize = MiMemoryConsumers[MC_USER].PagesUsed; /* FIXME */ /* Taskmgr multiplies this one by page size right away */ - Sci->CurrentSizeIncludingTransitionInPages = 0; /* FIXME: Should be */ + Sci->CurrentSizeIncludingTransitionInPages = MiMemoryConsumers[MC_USER].PagesUsed; /* FIXME: Should be */ /* system working set and standby pages. */ Sci->PageFaultCount = 0; /* FIXME */ Sci->MinimumWorkingSet = 0; /* FIXME */