diff --git a/base/setup/lib/bootsup.c b/base/setup/lib/bootsup.c index 37f7bdb43ab..b1e11d108d4 100644 --- a/base/setup/lib/bootsup.c +++ b/base/setup/lib/bootsup.c @@ -555,7 +555,7 @@ InstallBootCodeToDisk( IN PCWSTR RootPath, IN PFS_INSTALL_BOOTCODE InstallBootCode) { - NTSTATUS Status; + NTSTATUS Status, LockStatus; UNICODE_STRING Name; OBJECT_ATTRIBUTES ObjectAttributes; IO_STATUS_BLOCK IoStatusBlock; @@ -584,9 +584,32 @@ InstallBootCodeToDisk( if (!NT_SUCCESS(Status)) return Status; + /* Lock the volume */ + LockStatus = NtFsControlFile(PartitionHandle, NULL, NULL, NULL, &IoStatusBlock, FSCTL_LOCK_VOLUME, NULL, 0, NULL, 0); + if (!NT_SUCCESS(LockStatus)) + { + DPRINT1("Unable to lock the volume before installing boot code. Status 0x%08x. Expect problems.\n", LockStatus); + } + /* Install the bootcode (MBR, VBR) */ Status = InstallBootCode(SrcPath, PartitionHandle, PartitionHandle); + /* dismount & Unlock the volume */ + if (NT_SUCCESS(LockStatus)) + { + LockStatus = NtFsControlFile(PartitionHandle, NULL, NULL, NULL, &IoStatusBlock, FSCTL_DISMOUNT_VOLUME, NULL, 0, NULL, 0); + if (!NT_SUCCESS(LockStatus)) + { + DPRINT1("Unable to unlock the volume after installing boot code. Status 0x%08x. Expect problems.\n", LockStatus); + } + + LockStatus = NtFsControlFile(PartitionHandle, NULL, NULL, NULL, &IoStatusBlock, FSCTL_UNLOCK_VOLUME, NULL, 0, NULL, 0); + if (!NT_SUCCESS(LockStatus)) + { + DPRINT1("Unable to unlock the volume after installing boot code. Status 0x%08x. Expect problems.\n", LockStatus); + } + } + /* Close the partition */ NtClose(PartitionHandle); @@ -1076,6 +1099,7 @@ InstallFatBootcodeToPartition( DPRINT1("Install FAT32 bootcode: %S ==> %S\n", SrcPath, SystemRootPath->Buffer); Status = InstallBootCodeToDisk(SrcPath, SystemRootPath->Buffer, InstallFat32BootCode); + DPRINT1("Status: 0x%08X\n", Status); if (!NT_SUCCESS(Status)) { DPRINT1("InstallBootCodeToDisk(FAT32) failed (Status %lx)\n", Status); diff --git a/drivers/filesystems/CMakeLists.txt b/drivers/filesystems/CMakeLists.txt index 7485d433b5a..3d5277dfa74 100644 --- a/drivers/filesystems/CMakeLists.txt +++ b/drivers/filesystems/CMakeLists.txt @@ -2,7 +2,8 @@ add_subdirectory(btrfs) add_subdirectory(cdfs) add_subdirectory(ext2) -add_subdirectory(fastfat) +#add_subdirectory(fastfat) +add_subdirectory(fastfat_new) add_subdirectory(ffs) add_subdirectory(fs_rec) add_subdirectory(msfs) diff --git a/ntoskrnl/cache/newcc.h b/ntoskrnl/cache/newcc.h index adbf1c5d926..801bdc08bb8 100644 --- a/ntoskrnl/cache/newcc.h +++ b/ntoskrnl/cache/newcc.h @@ -6,7 +6,7 @@ typedef struct _NOCC_BCB PUBLIC_BCB Bcb; struct _NOCC_CACHE_MAP *Map; - PROS_SECTION_OBJECT SectionObject; + PSECTION SectionObject; LARGE_INTEGER FileOffset; ULONG Length; PVOID BaseAddress; @@ -62,10 +62,6 @@ NTAPI CcpUnpinData(PNOCC_BCB Bcb, BOOLEAN ActuallyRelease); -BOOLEAN -NTAPI -CcInitializeCacheManager(VOID); - VOID NTAPI CcShutdownSystem(VOID); diff --git a/ntoskrnl/cache/section/data.c b/ntoskrnl/cache/section/data.c index 1fb0fc20c56..fa83080eddb 100644 --- a/ntoskrnl/cache/section/data.c +++ b/ntoskrnl/cache/section/data.c @@ -112,6 +112,7 @@ _MmUnlockSectionSegment(PMM_SECTION_SEGMENT Segment, const char *file, int line) //DPRINT("MmUnlockSectionSegment(%p,%s:%d)\n", Segment, file, line); } +#ifdef NEWCC /* MiFlushMappedSection @@ -265,7 +266,6 @@ This deletes a segment entirely including its page map. It must have been unmapped in every address space. */ - VOID NTAPI MmFinalizeSegment(PMM_SECTION_SEGMENT Segment) @@ -603,6 +603,7 @@ _MiMapViewOfSegment(PMMSUPPORT AddressSpace, return STATUS_SUCCESS; } +#endif /* @@ -708,6 +709,7 @@ MmFreeCacheSectionPage(PVOID Context, } } +#ifdef NEWCC NTSTATUS NTAPI MmUnmapViewOfCacheSegment(PMMSUPPORT AddressSpace, @@ -840,5 +842,6 @@ MmUnmapCacheViewInSystemSpace (IN PVOID MappedBase) return Status; } +#endif /* NEWCC */ /* EOF */ diff --git a/ntoskrnl/cache/section/fault.c b/ntoskrnl/cache/section/fault.c index c70805a71ae..e869dac88fb 100644 --- a/ntoskrnl/cache/section/fault.c +++ b/ntoskrnl/cache/section/fault.c @@ -82,6 +82,8 @@ rmaps, so each mapping should be immediately followed by an rmap addition. #define DPRINTC DPRINT extern KEVENT MmWaitPageEvent; + +#ifdef NEWCC extern PMMWSL MmWorkingSetList; /* @@ -150,6 +152,7 @@ MmNotPresentFaultCachePage ( if (Segment->FileObject) { + __debugbreak(); DPRINT("FileName %wZ\n", &Segment->FileObject->FileName); } @@ -453,9 +456,11 @@ MiCowCacheSectionPage ( DPRINT("Address 0x%p\n", Address); return STATUS_SUCCESS; } +#endif KEVENT MmWaitPageEvent; +#ifdef NEWCC typedef struct _WORK_QUEUE_WITH_CONTEXT { WORK_QUEUE_ITEM WorkItem; @@ -960,3 +965,4 @@ MmNotPresentFaultCacheSection(KPROCESSOR_MODE Mode, return Status; } +#endif diff --git a/ntoskrnl/cache/section/io.c b/ntoskrnl/cache/section/io.c index 20454abb905..8ac93b06134 100644 --- a/ntoskrnl/cache/section/io.c +++ b/ntoskrnl/cache/section/io.c @@ -188,6 +188,7 @@ MiSimpleRead(PFILE_OBJECT FileObject, return Status; } +#ifdef NEWCC /* Convenience function for writing from kernel space. This issues a paging @@ -334,3 +335,4 @@ _MiWriteBackPage(PFILE_OBJECT FileObject, return Status; } +#endif diff --git a/ntoskrnl/cache/section/newmm.h b/ntoskrnl/cache/section/newmm.h index d5f9998993c..2efcd9cf7d5 100644 --- a/ntoskrnl/cache/section/newmm.h +++ b/ntoskrnl/cache/section/newmm.h @@ -3,26 +3,8 @@ #include /* TYPES *********************************************************************/ - -#define PFN_FROM_SSE(E) ((PFN_NUMBER)((E) >> PAGE_SHIFT)) -#define IS_SWAP_FROM_SSE(E) ((E) & 0x00000001) -#define MM_IS_WAIT_PTE(E) \ - (IS_SWAP_FROM_SSE(E) && SWAPENTRY_FROM_SSE(E) == MM_WAIT_ENTRY) -#define MAKE_PFN_SSE(P) ((ULONG_PTR)((P) << PAGE_SHIFT)) -#define SWAPENTRY_FROM_SSE(E) ((E) >> 1) -#define MAKE_SWAP_SSE(S) (((ULONG_PTR)(S) << 1) | 0x1) -#define DIRTY_SSE(E) ((E) | 2) -#define CLEAN_SSE(E) ((E) & ~2) -#define IS_DIRTY_SSE(E) ((E) & 2) -#define PAGE_FROM_SSE(E) ((E) & 0xFFFFF000) -#define SHARE_COUNT_FROM_SSE(E) (((E) & 0x00000FFC) >> 2) -#define MAX_SHARE_COUNT 0x3FF -#define MAKE_SSE(P, C) ((ULONG_PTR)((P) | ((C) << 2))) - #define MM_SEGMENT_FINALIZE (0x40000000) -#define RMAP_SEGMENT_MASK ~((ULONG_PTR)0xff) -#define RMAP_IS_SEGMENT(x) (((ULONG_PTR)(x) & RMAP_SEGMENT_MASK) == RMAP_SEGMENT_MASK) #define MIN(x,y) (((x)<(y))?(x):(y)) #define MAX(x,y) (((x)>(y))?(x):(y)) @@ -96,7 +78,7 @@ typedef struct _MM_REQUIRED_RESOURCES NTSTATUS NTAPI -MmCreateCacheSection(PROS_SECTION_OBJECT *SectionObject, +MmCreateCacheSection(PSECTION *SectionObject, ACCESS_MASK DesiredAccess, POBJECT_ATTRIBUTES ObjectAttributes, PLARGE_INTEGER UMaximumSize, @@ -123,25 +105,6 @@ VOID NTAPI MiInitializeSectionPageTable(PMM_SECTION_SEGMENT Segment); -NTSTATUS -NTAPI -_MmSetPageEntrySectionSegment(PMM_SECTION_SEGMENT Segment, - PLARGE_INTEGER Offset, - ULONG_PTR Entry, - const char *file, - int line); - -ULONG_PTR -NTAPI -_MmGetPageEntrySectionSegment(PMM_SECTION_SEGMENT Segment, - PLARGE_INTEGER Offset, - const char *file, - int line); - -#define MmSetPageEntrySectionSegment(S,O,E) _MmSetPageEntrySectionSegment(S,O,E,__FILE__,__LINE__) - -#define MmGetPageEntrySectionSegment(S,O) _MmGetPageEntrySectionSegment(S,O,__FILE__,__LINE__) - typedef VOID (NTAPI *FREE_SECTION_PAGE_FUN)( PMM_SECTION_SEGMENT Segment, PLARGE_INTEGER Offset); @@ -151,12 +114,6 @@ NTAPI MmFreePageTablesSectionSegment(PMM_SECTION_SEGMENT Segment, FREE_SECTION_PAGE_FUN FreePage); -/* Yields a lock */ -PMM_SECTION_SEGMENT -NTAPI -MmGetSectionAssociation(PFN_NUMBER Page, - PLARGE_INTEGER Offset); - NTSTATUS NTAPI MmSetSectionAssociation(PFN_NUMBER Page, @@ -267,22 +224,6 @@ MmPageOutDeleteMapping(PVOID Context, PEPROCESS Process, PVOID Address); -VOID -NTAPI -_MmLockSectionSegment(PMM_SECTION_SEGMENT Segment, - const char *file, - int line); - -#define MmLockSectionSegment(x) _MmLockSectionSegment(x,__FILE__,__LINE__) - -VOID -NTAPI -_MmUnlockSectionSegment(PMM_SECTION_SEGMENT Segment, - const char *file, - int line); - -#define MmUnlockSectionSegment(x) _MmUnlockSectionSegment(x,__FILE__,__LINE__) - VOID MmFreeCacheSectionPage(PVOID Context, MEMORY_AREA* MemoryArea, @@ -386,7 +327,7 @@ MiSwapInSectionPage(PMMSUPPORT AddressSpace, NTSTATUS NTAPI -MmExtendCacheSection(PROS_SECTION_OBJECT Section, +MmExtendCacheSection(PSECTION Section, PLARGE_INTEGER NewSize, BOOLEAN ExtendFile); diff --git a/ntoskrnl/cache/section/sptab.c b/ntoskrnl/cache/section/sptab.c index 454fb25af42..0877fb085f6 100644 --- a/ntoskrnl/cache/section/sptab.c +++ b/ntoskrnl/cache/section/sptab.c @@ -187,9 +187,6 @@ _MmSetPageEntrySectionSegment(PMM_SECTION_SEGMENT Segment, ASSERT(Segment->Locked); ASSERT(!IS_SWAP_FROM_SSE(Entry) || !IS_DIRTY_SSE(Entry)); - if (Entry && !IS_SWAP_FROM_SSE(Entry)) - MmGetRmapListHeadPage(PFN_FROM_SSE(Entry)); - PageTable = MiSectionPageTableGetOrAllocate(&Segment->PageTable, Offset); if (!PageTable) return STATUS_NO_MEMORY; @@ -207,27 +204,51 @@ _MmSetPageEntrySectionSegment(PMM_SECTION_SEGMENT Segment, OldEntry, Entry); - if (PFN_FROM_SSE(Entry) == PFN_FROM_SSE(OldEntry)) { - /* Nothing */ - } else if (Entry && !IS_SWAP_FROM_SSE(Entry)) { - ASSERT(!OldEntry || IS_SWAP_FROM_SSE(OldEntry)); - MmSetSectionAssociation(PFN_FROM_SSE(Entry), Segment, Offset); - } else if (OldEntry && !IS_SWAP_FROM_SSE(OldEntry)) { - ASSERT(!Entry || IS_SWAP_FROM_SSE(Entry)); - MmDeleteSectionAssociation(PFN_FROM_SSE(OldEntry)); - } else if (IS_SWAP_FROM_SSE(Entry)) { - ASSERT(!IS_SWAP_FROM_SSE(OldEntry) || - SWAPENTRY_FROM_SSE(OldEntry) == MM_WAIT_ENTRY); - if (OldEntry && SWAPENTRY_FROM_SSE(OldEntry) != MM_WAIT_ENTRY) - MmDeleteSectionAssociation(PFN_FROM_SSE(OldEntry)); - } else if (IS_SWAP_FROM_SSE(OldEntry)) { - ASSERT(!IS_SWAP_FROM_SSE(Entry)); - if (Entry) - MmSetSectionAssociation(PFN_FROM_SSE(OldEntry), Segment, Offset); - } else { - /* We should not be replacing a page like this */ - ASSERT(FALSE); + if (Entry && !IS_SWAP_FROM_SSE(Entry)) + { + /* We have a valid entry. See if we must do something */ + if (OldEntry && !IS_SWAP_FROM_SSE(OldEntry)) + { + /* The previous entry was valid. Shall we swap the Rmaps ? */ + if (PFN_FROM_SSE(Entry) != PFN_FROM_SSE(OldEntry)) + { + MmDeleteSectionAssociation(PFN_FROM_SSE(OldEntry)); + MmSetSectionAssociation(PFN_FROM_SSE(Entry), Segment, Offset); + } + } + else + { + /* + * We're switching to a valid entry from an invalid one. + * Add the Rmap and take a ref on the segment. + */ + MmSetSectionAssociation(PFN_FROM_SSE(Entry), Segment, Offset); + InterlockedIncrement64(Segment->ReferenceCount); + + if (Offset->QuadPart >= (Segment->LastPage << PAGE_SHIFT)) + Segment->LastPage = (Offset->QuadPart >> PAGE_SHIFT) + 1; + } } + else if (OldEntry && !IS_SWAP_FROM_SSE(OldEntry)) + { + /* We're switching to an invalid entry from a valid one */ + MmDeleteSectionAssociation(PFN_FROM_SSE(OldEntry)); + MmDereferenceSegment(Segment); + + if (Offset->QuadPart == ((Segment->LastPage - 1ULL) << PAGE_SHIFT)) + { + /* We are unsetting the last page */ + while (--Segment->LastPage) + { + LARGE_INTEGER CheckOffset; + CheckOffset.QuadPart = (Segment->LastPage - 1) << PAGE_SHIFT; + ULONG_PTR Entry = MmGetPageEntrySectionSegment(Segment, &CheckOffset); + if ((Entry != 0) && !IS_SWAP_FROM_SSE(Entry)) + break; + } + } + } + PageTable->PageEntries[PageIndex] = Entry; return STATUS_SUCCESS; } @@ -335,13 +356,13 @@ MmGetSectionAssociation(PFN_NUMBER Page, PMM_SECTION_SEGMENT Segment = NULL; PCACHE_SECTION_PAGE_TABLE PageTable; - PageTable = (PCACHE_SECTION_PAGE_TABLE)MmGetSegmentRmap(Page, - &RawOffset); + PageTable = MmGetSegmentRmap(Page, &RawOffset); if (PageTable) { Segment = PageTable->Segment; Offset->QuadPart = PageTable->FileOffset.QuadPart + ((ULONG64)RawOffset << PAGE_SHIFT); + ASSERT(PFN_FROM_SSE(PageTable->PageEntries[RawOffset]) == Page); } return Segment; diff --git a/ntoskrnl/cc/cacheman.c b/ntoskrnl/cc/cacheman.c index 6086360db25..238fecb0f5a 100644 --- a/ntoskrnl/cc/cacheman.c +++ b/ntoskrnl/cc/cacheman.c @@ -42,7 +42,6 @@ CcPfInitializePrefetcher(VOID) CODE_SEG("INIT") BOOLEAN -NTAPI CcInitializeCacheManager(VOID) { ULONG Thread; @@ -308,7 +307,7 @@ CcSetBcbOwnerPointer ( IN PVOID Owner ) { - PINTERNAL_BCB iBcb = Bcb; + PINTERNAL_BCB iBcb = CONTAINING_RECORD(Bcb, INTERNAL_BCB, PFCB); CCTRACE(CC_API_DEBUG, "Bcb=%p Owner=%p\n", Bcb, Owner); diff --git a/ntoskrnl/cc/copy.c b/ntoskrnl/cc/copy.c index 1f731716b1f..cdab94c8313 100644 --- a/ntoskrnl/cc/copy.c +++ b/ntoskrnl/cc/copy.c @@ -20,13 +20,6 @@ static PFN_NUMBER CcZeroPage = 0; #define MAX_ZERO_LENGTH (256 * 1024) -typedef enum _CC_COPY_OPERATION -{ - CcOperationRead, - CcOperationWrite, - CcOperationZero -} CC_COPY_OPERATION; - typedef enum _CC_CAN_WRITE_RETRY { FirstTry = 0, @@ -35,7 +28,7 @@ typedef enum _CC_CAN_WRITE_RETRY RetryMasterLocked = 255, } CC_CAN_WRITE_RETRY; -ULONG CcRosTraceLevel = 0; +ULONG CcRosTraceLevel = CC_API_DEBUG; ULONG CcFastMdlReadWait; ULONG CcFastMdlReadNotPossible; ULONG CcFastReadNotPossible; @@ -76,338 +69,6 @@ CcInitCacheZeroPage ( MiZeroPhysicalPage(CcZeroPage); } -NTSTATUS -NTAPI -CcReadVirtualAddress ( - PROS_VACB Vacb) -{ - ULONG Size; - PMDL Mdl; - NTSTATUS Status; - IO_STATUS_BLOCK IoStatus; - KEVENT Event; - ULARGE_INTEGER LargeSize; - - LargeSize.QuadPart = Vacb->SharedCacheMap->SectionSize.QuadPart - Vacb->FileOffset.QuadPart; - if (LargeSize.QuadPart > VACB_MAPPING_GRANULARITY) - { - LargeSize.QuadPart = VACB_MAPPING_GRANULARITY; - } - Size = LargeSize.LowPart; - - Size = ROUND_TO_PAGES(Size); - ASSERT(Size <= VACB_MAPPING_GRANULARITY); - ASSERT(Size > 0); - - Mdl = IoAllocateMdl(Vacb->BaseAddress, Size, FALSE, FALSE, NULL); - if (!Mdl) - { - return STATUS_INSUFFICIENT_RESOURCES; - } - - Status = STATUS_SUCCESS; - _SEH2_TRY - { - MmProbeAndLockPages(Mdl, KernelMode, IoWriteAccess); - } - _SEH2_EXCEPT (EXCEPTION_EXECUTE_HANDLER) - { - Status = _SEH2_GetExceptionCode(); - DPRINT1("MmProbeAndLockPages failed with: %lx for %p (%p, %p)\n", Status, Mdl, Vacb, Vacb->BaseAddress); - KeBugCheck(CACHE_MANAGER); - } _SEH2_END; - - if (NT_SUCCESS(Status)) - { - Mdl->MdlFlags |= MDL_IO_PAGE_READ; - KeInitializeEvent(&Event, NotificationEvent, FALSE); - Status = IoPageRead(Vacb->SharedCacheMap->FileObject, Mdl, &Vacb->FileOffset, &Event, &IoStatus); - if (Status == STATUS_PENDING) - { - KeWaitForSingleObject(&Event, Executive, KernelMode, FALSE, NULL); - Status = IoStatus.Status; - } - - MmUnlockPages(Mdl); - } - - IoFreeMdl(Mdl); - - if (!NT_SUCCESS(Status) && (Status != STATUS_END_OF_FILE)) - { - DPRINT1("IoPageRead failed, Status %x\n", Status); - return Status; - } - - if (Size < VACB_MAPPING_GRANULARITY) - { - RtlZeroMemory((char*)Vacb->BaseAddress + Size, - VACB_MAPPING_GRANULARITY - Size); - } - - return STATUS_SUCCESS; -} - -NTSTATUS -NTAPI -CcWriteVirtualAddress ( - PROS_VACB Vacb) -{ - ULONG Size; - PMDL Mdl; - NTSTATUS Status; - IO_STATUS_BLOCK IoStatus; - KEVENT Event; - ULARGE_INTEGER LargeSize; - - LargeSize.QuadPart = Vacb->SharedCacheMap->SectionSize.QuadPart - Vacb->FileOffset.QuadPart; - if (LargeSize.QuadPart > VACB_MAPPING_GRANULARITY) - { - LargeSize.QuadPart = VACB_MAPPING_GRANULARITY; - } - Size = LargeSize.LowPart; - // - // Nonpaged pool PDEs in ReactOS must actually be synchronized between the - // MmGlobalPageDirectory and the real system PDE directory. What a mess... - // - { - ULONG i = 0; - do - { - MmGetPfnForProcess(NULL, (PVOID)((ULONG_PTR)Vacb->BaseAddress + (i << PAGE_SHIFT))); - } while (++i < (Size >> PAGE_SHIFT)); - } - - ASSERT(Size <= VACB_MAPPING_GRANULARITY); - ASSERT(Size > 0); - - Mdl = IoAllocateMdl(Vacb->BaseAddress, Size, FALSE, FALSE, NULL); - if (!Mdl) - { - return STATUS_INSUFFICIENT_RESOURCES; - } - - Status = STATUS_SUCCESS; - _SEH2_TRY - { - MmProbeAndLockPages(Mdl, KernelMode, IoReadAccess); - } - _SEH2_EXCEPT (EXCEPTION_EXECUTE_HANDLER) - { - Status = _SEH2_GetExceptionCode(); - DPRINT1("MmProbeAndLockPages failed with: %lx for %p (%p, %p)\n", Status, Mdl, Vacb, Vacb->BaseAddress); - KeBugCheck(CACHE_MANAGER); - } _SEH2_END; - - if (NT_SUCCESS(Status)) - { - KeInitializeEvent(&Event, NotificationEvent, FALSE); - Status = IoSynchronousPageWrite(Vacb->SharedCacheMap->FileObject, Mdl, &Vacb->FileOffset, &Event, &IoStatus); - if (Status == STATUS_PENDING) - { - KeWaitForSingleObject(&Event, Executive, KernelMode, FALSE, NULL); - Status = IoStatus.Status; - } - - MmUnlockPages(Mdl); - } - IoFreeMdl(Mdl); - if (!NT_SUCCESS(Status) && (Status != STATUS_END_OF_FILE)) - { - DPRINT1("IoPageWrite failed, Status %x\n", Status); - return Status; - } - - return STATUS_SUCCESS; -} - -NTSTATUS -ReadWriteOrZero( - _Inout_ PVOID BaseAddress, - _Inout_opt_ PVOID Buffer, - _In_ ULONG Length, - _In_ CC_COPY_OPERATION Operation) -{ - NTSTATUS Status = STATUS_SUCCESS; - - if (Operation == CcOperationZero) - { - /* Zero */ - RtlZeroMemory(BaseAddress, Length); - } - else - { - _SEH2_TRY - { - if (Operation == CcOperationWrite) - RtlCopyMemory(BaseAddress, Buffer, Length); - else - RtlCopyMemory(Buffer, BaseAddress, Length); - } - _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER) - { - Status = _SEH2_GetExceptionCode(); - } - _SEH2_END; - } - return Status; -} - -BOOLEAN -CcCopyData ( - _In_ PFILE_OBJECT FileObject, - _In_ LONGLONG FileOffset, - _Inout_ PVOID Buffer, - _In_ LONGLONG Length, - _In_ CC_COPY_OPERATION Operation, - _In_ BOOLEAN Wait, - _Out_ PIO_STATUS_BLOCK IoStatus) -{ - NTSTATUS Status; - LONGLONG CurrentOffset; - ULONG BytesCopied; - KIRQL OldIrql; - PROS_SHARED_CACHE_MAP SharedCacheMap; - PLIST_ENTRY ListEntry; - PROS_VACB Vacb; - ULONG PartialLength; - PVOID BaseAddress; - BOOLEAN Valid; - PPRIVATE_CACHE_MAP PrivateCacheMap; - - SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; - PrivateCacheMap = FileObject->PrivateCacheMap; - CurrentOffset = FileOffset; - BytesCopied = 0; - - if (!Wait) - { - /* test if the requested data is available */ - KeAcquireSpinLock(&SharedCacheMap->CacheMapLock, &OldIrql); - /* FIXME: this loop doesn't take into account areas that don't have - * a VACB in the list yet */ - ListEntry = SharedCacheMap->CacheMapVacbListHead.Flink; - while (ListEntry != &SharedCacheMap->CacheMapVacbListHead) - { - Vacb = CONTAINING_RECORD(ListEntry, - ROS_VACB, - CacheMapVacbListEntry); - ListEntry = ListEntry->Flink; - if (!Vacb->Valid && - DoRangesIntersect(Vacb->FileOffset.QuadPart, - VACB_MAPPING_GRANULARITY, - CurrentOffset, Length)) - { - KeReleaseSpinLock(&SharedCacheMap->CacheMapLock, OldIrql); - /* data not available */ - return FALSE; - } - if (Vacb->FileOffset.QuadPart >= CurrentOffset + Length) - break; - } - KeReleaseSpinLock(&SharedCacheMap->CacheMapLock, OldIrql); - } - - PartialLength = CurrentOffset % VACB_MAPPING_GRANULARITY; - if (PartialLength != 0) - { - PartialLength = min(Length, VACB_MAPPING_GRANULARITY - PartialLength); - Status = CcRosRequestVacb(SharedCacheMap, - ROUND_DOWN(CurrentOffset, - VACB_MAPPING_GRANULARITY), - &BaseAddress, - &Valid, - &Vacb); - if (!NT_SUCCESS(Status)) - ExRaiseStatus(Status); - if (!Valid) - { - Status = CcReadVirtualAddress(Vacb); - if (!NT_SUCCESS(Status)) - { - CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE, FALSE); - ExRaiseStatus(Status); - } - } - Status = ReadWriteOrZero((PUCHAR)BaseAddress + CurrentOffset % VACB_MAPPING_GRANULARITY, - Buffer, - PartialLength, - Operation); - - CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, Operation != CcOperationRead, FALSE); - - if (!NT_SUCCESS(Status)) - ExRaiseStatus(STATUS_INVALID_USER_BUFFER); - - Length -= PartialLength; - CurrentOffset += PartialLength; - BytesCopied += PartialLength; - - if (Operation != CcOperationZero) - Buffer = (PVOID)((ULONG_PTR)Buffer + PartialLength); - } - - while (Length > 0) - { - ASSERT(CurrentOffset % VACB_MAPPING_GRANULARITY == 0); - PartialLength = min(VACB_MAPPING_GRANULARITY, Length); - Status = CcRosRequestVacb(SharedCacheMap, - CurrentOffset, - &BaseAddress, - &Valid, - &Vacb); - if (!NT_SUCCESS(Status)) - ExRaiseStatus(Status); - if (!Valid && - (Operation == CcOperationRead || - PartialLength < VACB_MAPPING_GRANULARITY)) - { - Status = CcReadVirtualAddress(Vacb); - if (!NT_SUCCESS(Status)) - { - CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE, FALSE); - ExRaiseStatus(Status); - } - } - Status = ReadWriteOrZero(BaseAddress, Buffer, PartialLength, Operation); - - CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, Operation != CcOperationRead, FALSE); - - if (!NT_SUCCESS(Status)) - ExRaiseStatus(STATUS_INVALID_USER_BUFFER); - - Length -= PartialLength; - CurrentOffset += PartialLength; - BytesCopied += PartialLength; - - if (Operation != CcOperationZero) - Buffer = (PVOID)((ULONG_PTR)Buffer + PartialLength); - } - - /* If that was a successful sync read operation, let's handle read ahead */ - if (Operation == CcOperationRead && Length == 0 && Wait) - { - /* If file isn't random access and next read may get us cross VACB boundary, - * schedule next read - */ - if (!BooleanFlagOn(FileObject->Flags, FO_RANDOM_ACCESS) && - (CurrentOffset - 1) / VACB_MAPPING_GRANULARITY != (CurrentOffset + BytesCopied - 1) / VACB_MAPPING_GRANULARITY) - { - CcScheduleReadAhead(FileObject, (PLARGE_INTEGER)&FileOffset, BytesCopied); - } - - /* And update read history in private cache map */ - PrivateCacheMap->FileOffset1.QuadPart = PrivateCacheMap->FileOffset2.QuadPart; - PrivateCacheMap->BeyondLastByte1.QuadPart = PrivateCacheMap->BeyondLastByte2.QuadPart; - PrivateCacheMap->FileOffset2.QuadPart = FileOffset; - PrivateCacheMap->BeyondLastByte2.QuadPart = FileOffset + BytesCopied; - } - - IoStatus->Status = STATUS_SUCCESS; - IoStatus->Information = BytesCopied; - return TRUE; -} - VOID CcPostDeferredWrites(VOID) { @@ -492,8 +153,6 @@ CcPerformReadAhead( PROS_SHARED_CACHE_MAP SharedCacheMap; PROS_VACB Vacb; ULONG PartialLength; - PVOID BaseAddress; - BOOLEAN Valid; ULONG Length; PPRIVATE_CACHE_MAP PrivateCacheMap; BOOLEAN Locked; @@ -556,10 +215,7 @@ CcPerformReadAhead( { PartialLength = min(Length, VACB_MAPPING_GRANULARITY - PartialLength); Status = CcRosRequestVacb(SharedCacheMap, - ROUND_DOWN(CurrentOffset, - VACB_MAPPING_GRANULARITY), - &BaseAddress, - &Valid, + ROUND_DOWN(CurrentOffset, VACB_MAPPING_GRANULARITY), &Vacb); if (!NT_SUCCESS(Status)) { @@ -567,18 +223,16 @@ CcPerformReadAhead( goto Clear; } - if (!Valid) + Status = CcRosEnsureVacbResident(Vacb, TRUE, FALSE, + CurrentOffset % VACB_MAPPING_GRANULARITY, PartialLength); + if (!NT_SUCCESS(Status)) { - Status = CcReadVirtualAddress(Vacb); - if (!NT_SUCCESS(Status)) - { - CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE, FALSE); - DPRINT1("Failed to read data: %lx!\n", Status); - goto Clear; - } + CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE); + DPRINT1("Failed to read data: %lx!\n", Status); + goto Clear; } - CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); + CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE); Length -= PartialLength; CurrentOffset += PartialLength; @@ -590,8 +244,6 @@ CcPerformReadAhead( PartialLength = min(VACB_MAPPING_GRANULARITY, Length); Status = CcRosRequestVacb(SharedCacheMap, CurrentOffset, - &BaseAddress, - &Valid, &Vacb); if (!NT_SUCCESS(Status)) { @@ -599,18 +251,15 @@ CcPerformReadAhead( goto Clear; } - if (!Valid) + Status = CcRosEnsureVacbResident(Vacb, TRUE, FALSE, 0, PartialLength); + if (!NT_SUCCESS(Status)) { - Status = CcReadVirtualAddress(Vacb); - if (!NT_SUCCESS(Status)) - { - CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE, FALSE); - DPRINT1("Failed to read data: %lx!\n", Status); - goto Clear; - } + CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE); + DPRINT1("Failed to read data: %lx!\n", Status); + goto Clear; } - CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); + CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE); Length -= PartialLength; CurrentOffset += PartialLength; @@ -798,6 +447,26 @@ CcCanIWrite ( return TRUE; } +static +int +CcpCheckInvalidUserBuffer(PEXCEPTION_POINTERS Except, PVOID Buffer, ULONG Length) +{ + ULONG_PTR ExceptionAddress; + ULONG_PTR BeginAddress = (ULONG_PTR)Buffer; + ULONG_PTR EndAddress = (ULONG_PTR)Buffer + Length; + + if (Except->ExceptionRecord->ExceptionCode != STATUS_ACCESS_VIOLATION) + return EXCEPTION_CONTINUE_SEARCH; + if (Except->ExceptionRecord->NumberParameters < 2) + return EXCEPTION_CONTINUE_SEARCH; + + ExceptionAddress = Except->ExceptionRecord->ExceptionInformation[1]; + if ((ExceptionAddress >= BeginAddress) && (ExceptionAddress < EndAddress)) + return EXCEPTION_EXECUTE_HANDLER; + + return EXCEPTION_CONTINUE_SEARCH; +} + /* * @implemented */ @@ -811,6 +480,13 @@ CcCopyRead ( OUT PVOID Buffer, OUT PIO_STATUS_BLOCK IoStatus) { + PROS_VACB Vacb; + PROS_SHARED_CACHE_MAP SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; + NTSTATUS Status; + LONGLONG CurrentOffset; + LONGLONG ReadEnd = FileOffset->QuadPart + Length; + ULONG ReadLength = 0; + CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%I64d Length=%lu Wait=%d\n", FileObject, FileOffset->QuadPart, Length, Wait); @@ -819,13 +495,81 @@ CcCopyRead ( FileObject, FileOffset->QuadPart, Length, Wait, Buffer, IoStatus); - return CcCopyData(FileObject, - FileOffset->QuadPart, - Buffer, - Length, - CcOperationRead, - Wait, - IoStatus); + if (!SharedCacheMap) + return FALSE; + + /* Documented to ASSERT, but KMTests test this case... */ + // ASSERT((FileOffset->QuadPart + Length) <= SharedCacheMap->FileSize.QuadPart); + + CurrentOffset = FileOffset->QuadPart; + while(CurrentOffset < ReadEnd) + { + Status = CcRosGetVacb(SharedCacheMap, CurrentOffset, &Vacb); + if (!NT_SUCCESS(Status)) + { + ExRaiseStatus(Status); + return FALSE; + } + + _SEH2_TRY + { + ULONG VacbOffset = CurrentOffset % VACB_MAPPING_GRANULARITY; + ULONG VacbLength = min(Length, VACB_MAPPING_GRANULARITY - VacbOffset); + SIZE_T CopyLength = VacbLength; + + if (!CcRosEnsureVacbResident(Vacb, Wait, FALSE, VacbOffset, VacbLength)) + return FALSE; + + _SEH2_TRY + { + RtlCopyMemory(Buffer, (PUCHAR)Vacb->BaseAddress + VacbOffset, CopyLength); + } + _SEH2_EXCEPT(CcpCheckInvalidUserBuffer(_SEH2_GetExceptionInformation(), Buffer, VacbLength)) + { + ExRaiseStatus(STATUS_INVALID_USER_BUFFER); + } + _SEH2_END; + + ReadLength += VacbLength; + + Buffer = (PVOID)((ULONG_PTR)Buffer + VacbLength); + CurrentOffset += VacbLength; + Length -= VacbLength; + } + _SEH2_FINALLY + { + CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE); + } + _SEH2_END; + } + + IoStatus->Status = STATUS_SUCCESS; + IoStatus->Information = ReadLength; + +#if 0 + /* If that was a successful sync read operation, let's handle read ahead */ + if (Length == 0 && Wait) + { + PPRIVATE_CACHE_MAP PrivateCacheMap = FileObject->PrivateCacheMap; + + /* If file isn't random access and next read may get us cross VACB boundary, + * schedule next read + */ + if (!BooleanFlagOn(FileObject->Flags, FO_RANDOM_ACCESS) && + (CurrentOffset - 1) / VACB_MAPPING_GRANULARITY != (CurrentOffset + ReadLength - 1) / VACB_MAPPING_GRANULARITY) + { + CcScheduleReadAhead(FileObject, FileOffset, ReadLength); + } + + /* And update read history in private cache map */ + PrivateCacheMap->FileOffset1.QuadPart = PrivateCacheMap->FileOffset2.QuadPart; + PrivateCacheMap->BeyondLastByte1.QuadPart = PrivateCacheMap->BeyondLastByte2.QuadPart; + PrivateCacheMap->FileOffset2.QuadPart = FileOffset->QuadPart; + PrivateCacheMap->BeyondLastByte2.QuadPart = FileOffset->QuadPart + ReadLength; + } +#endif + + return TRUE; } /* @@ -840,7 +584,11 @@ CcCopyWrite ( IN BOOLEAN Wait, IN PVOID Buffer) { - IO_STATUS_BLOCK IoStatus; + PROS_VACB Vacb; + PROS_SHARED_CACHE_MAP SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; + NTSTATUS Status; + LONGLONG CurrentOffset; + LONGLONG WriteEnd; CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%I64d Length=%lu Wait=%d Buffer=%p\n", FileObject, FileOffset->QuadPart, Length, Wait, Buffer); @@ -849,13 +597,66 @@ CcCopyWrite ( "Length %lu, Wait %u, Buffer 0x%p)\n", FileObject, FileOffset->QuadPart, Length, Wait, Buffer); - return CcCopyData(FileObject, - FileOffset->QuadPart, - Buffer, - Length, - CcOperationWrite, - Wait, - &IoStatus); + if (!SharedCacheMap) + return FALSE; + + Status = RtlLongLongAdd(FileOffset->QuadPart, Length, &WriteEnd); + if (!NT_SUCCESS(Status)) + ExRaiseStatus(Status); + + ASSERT(WriteEnd <= SharedCacheMap->SectionSize.QuadPart); + + CurrentOffset = FileOffset->QuadPart; + while(CurrentOffset < WriteEnd) + { + ULONG VacbOffset = CurrentOffset % VACB_MAPPING_GRANULARITY; + ULONG VacbLength = min(WriteEnd - CurrentOffset, VACB_MAPPING_GRANULARITY - VacbOffset); + + Status = CcRosGetVacb(SharedCacheMap, CurrentOffset, &Vacb); + if (!NT_SUCCESS(Status)) + { + ExRaiseStatus(Status); + return FALSE; + } + + _SEH2_TRY + { + if (!CcRosEnsureVacbResident(Vacb, Wait, FALSE, VacbOffset, VacbLength)) + { + return FALSE; + } + + _SEH2_TRY + { + RtlCopyMemory((PVOID)((ULONG_PTR)Vacb->BaseAddress + VacbOffset), Buffer, VacbLength); + } + _SEH2_EXCEPT(CcpCheckInvalidUserBuffer(_SEH2_GetExceptionInformation(), Buffer, VacbLength)) + { + ExRaiseStatus(STATUS_INVALID_USER_BUFFER); + } + _SEH2_END; + + Buffer = (PVOID)((ULONG_PTR)Buffer + VacbLength); + CurrentOffset += VacbLength; + + /* Tell Mm */ + Status = MmMakePagesDirty(NULL, Add2Ptr(Vacb->BaseAddress, VacbOffset), VacbLength); + if (!NT_SUCCESS(Status)) + ExRaiseStatus(Status); + } + _SEH2_FINALLY + { + /* Do not mark the VACB as dirty if an exception was raised */ + CcRosReleaseVacb(SharedCacheMap, Vacb, !_SEH2_AbnormalTermination(), FALSE); + } + _SEH2_END; + } + + /* Flush if needed */ + if (FileObject->Flags & FO_WRITE_THROUGH) + CcFlushCache(FileObject->SectionObjectPointer, FileOffset, Length, NULL); + + return TRUE; } /* @@ -999,11 +800,8 @@ CcZeroData ( NTSTATUS Status; LARGE_INTEGER WriteOffset; LONGLONG Length; - ULONG CurrentLength; - PMDL Mdl; - ULONG i; - IO_STATUS_BLOCK Iosb; - KEVENT Event; + PROS_VACB Vacb; + PROS_SHARED_CACHE_MAP SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; CCTRACE(CC_API_DEBUG, "FileObject=%p StartOffset=%I64u EndOffset=%I64u Wait=%d\n", FileObject, StartOffset->QuadPart, EndOffset->QuadPart, Wait); @@ -1015,28 +813,33 @@ CcZeroData ( Length = EndOffset->QuadPart - StartOffset->QuadPart; WriteOffset.QuadPart = StartOffset->QuadPart; - if (FileObject->SectionObjectPointer->SharedCacheMap == NULL) + if (!SharedCacheMap) { - /* File is not cached */ + /* Make this a non-cached write */ + IO_STATUS_BLOCK Iosb; + KEVENT Event; + PMDL Mdl; + ULONG i; + ULONG CurrentLength; + PPFN_NUMBER PfnArray; - Mdl = _alloca(MmSizeOfMdl(NULL, MAX_ZERO_LENGTH)); + /* Setup our Mdl */ + Mdl = IoAllocateMdl(NULL, min(Length, MAX_ZERO_LENGTH), FALSE, FALSE, NULL); + if (!Mdl) + ExRaiseStatus(STATUS_INSUFFICIENT_RESOURCES); + PfnArray = MmGetMdlPfnArray(Mdl); + for (i = 0; i < BYTES_TO_PAGES(Mdl->ByteCount); i++) + PfnArray[i] = CcZeroPage; + Mdl->MdlFlags |= MDL_PAGES_LOCKED; + + /* Perform the write sequencially */ while (Length > 0) { - if (Length + WriteOffset.QuadPart % PAGE_SIZE > MAX_ZERO_LENGTH) - { - CurrentLength = MAX_ZERO_LENGTH - WriteOffset.QuadPart % PAGE_SIZE; - } - else - { - CurrentLength = Length; - } - MmInitializeMdl(Mdl, (PVOID)(ULONG_PTR)WriteOffset.QuadPart, CurrentLength); - Mdl->MdlFlags |= (MDL_PAGES_LOCKED | MDL_IO_PAGE_READ); - for (i = 0; i < ((Mdl->Size - sizeof(MDL)) / sizeof(ULONG)); i++) - { - ((PPFN_NUMBER)(Mdl + 1))[i] = CcZeroPage; - } + CurrentLength = min(Length, MAX_ZERO_LENGTH); + + Mdl->ByteCount = CurrentLength; + KeInitializeEvent(&Event, NotificationEvent, FALSE); Status = IoSynchronousPageWrite(FileObject, Mdl, &WriteOffset, &Event, &Iosb); if (Status == STATUS_PENDING) @@ -1050,24 +853,68 @@ CcZeroData ( } if (!NT_SUCCESS(Status)) { - return FALSE; + IoFreeMdl(Mdl); + ExRaiseStatus(Status); } WriteOffset.QuadPart += CurrentLength; Length -= CurrentLength; } - } - else - { - IO_STATUS_BLOCK IoStatus; - return CcCopyData(FileObject, - WriteOffset.QuadPart, - NULL, - Length, - CcOperationZero, - Wait, - &IoStatus); + IoFreeMdl(Mdl); + + return TRUE; } + /* See if we should simply truncate the valid data length */ + if ((StartOffset->QuadPart < SharedCacheMap->ValidDataLength.QuadPart) && (EndOffset->QuadPart >= SharedCacheMap->ValidDataLength.QuadPart)) + { + DPRINT1("Truncating VDL.\n"); + SharedCacheMap->ValidDataLength = *StartOffset; + return TRUE; + } + + ASSERT(EndOffset->QuadPart <= SharedCacheMap->SectionSize.QuadPart); + + while(WriteOffset.QuadPart < EndOffset->QuadPart) + { + ULONG VacbOffset = WriteOffset.QuadPart % VACB_MAPPING_GRANULARITY; + ULONG VacbLength = min(Length, VACB_MAPPING_GRANULARITY - VacbOffset); + + Status = CcRosGetVacb(SharedCacheMap, WriteOffset.QuadPart, &Vacb); + if (!NT_SUCCESS(Status)) + { + ExRaiseStatus(Status); + return FALSE; + } + + _SEH2_TRY + { + if (!CcRosEnsureVacbResident(Vacb, Wait, FALSE, VacbOffset, VacbLength)) + { + return FALSE; + } + + RtlZeroMemory((PVOID)((ULONG_PTR)Vacb->BaseAddress + VacbOffset), VacbLength); + + WriteOffset.QuadPart += VacbLength; + Length -= VacbLength; + + /* Tell Mm */ + Status = MmMakePagesDirty(NULL, Add2Ptr(Vacb->BaseAddress, VacbOffset), VacbLength); + if (!NT_SUCCESS(Status)) + ExRaiseStatus(Status); + } + _SEH2_FINALLY + { + /* Do not mark the VACB as dirty if an exception was raised */ + CcRosReleaseVacb(SharedCacheMap, Vacb, !_SEH2_AbnormalTermination(), FALSE); + } + _SEH2_END; + } + + /* Flush if needed */ + if (FileObject->Flags & FO_WRITE_THROUGH) + CcFlushCache(FileObject->SectionObjectPointer, StartOffset, EndOffset->QuadPart - StartOffset->QuadPart, NULL); + return TRUE; } diff --git a/ntoskrnl/cc/fs.c b/ntoskrnl/cc/fs.c index 5798f56a91a..bef6502f73f 100644 --- a/ntoskrnl/cc/fs.c +++ b/ntoskrnl/cc/fs.c @@ -10,13 +10,10 @@ /* INCLUDES ******************************************************************/ #include + #define NDEBUG #include -/* GLOBALS *****************************************************************/ - -NTSTATUS CcRosInternalFreeVacb(PROS_VACB Vacb); - /* FUNCTIONS *****************************************************************/ /* @@ -48,7 +45,7 @@ NTAPI CcGetFileObjectFromBcb ( IN PVOID Bcb) { - PINTERNAL_BCB iBcb = (PINTERNAL_BCB)Bcb; + PINTERNAL_BCB iBcb = CONTAINING_RECORD(Bcb, INTERNAL_BCB, PFCB); CCTRACE(CC_API_DEBUG, "Bcb=%p\n", Bcb); @@ -183,7 +180,10 @@ CcPurgeCacheSection ( SharedCacheMap = SectionObjectPointer->SharedCacheMap; if (!SharedCacheMap) - return FALSE; + { + Success = TRUE; + goto purgeMm; + } StartOffset = FileOffset != NULL ? FileOffset->QuadPart : 0; if (Length == 0 || FileOffset == NULL) @@ -260,6 +260,11 @@ CcPurgeCacheSection ( ASSERT(Refs == 0); } + /* Now make sure that Mm doesn't hold some pages here. */ +purgeMm: + if (Success) + Success = MmPurgeSegment(SectionObjectPointer, FileOffset, Length); + return Success; } @@ -272,8 +277,9 @@ CcSetFileSizes ( IN PFILE_OBJECT FileObject, IN PCC_FILE_SIZES FileSizes) { - KIRQL oldirql; + KIRQL OldIrql; PROS_SHARED_CACHE_MAP SharedCacheMap; + LARGE_INTEGER OldSectionSize; CCTRACE(CC_API_DEBUG, "FileObject=%p FileSizes=%p\n", FileObject, FileSizes); @@ -294,7 +300,15 @@ CcSetFileSizes ( if (SharedCacheMap == NULL) return; - if (FileSizes->AllocationSize.QuadPart < SharedCacheMap->SectionSize.QuadPart) + /* Update the relevant fields */ + KeAcquireSpinLock(&SharedCacheMap->CacheMapLock, &OldIrql); + OldSectionSize = SharedCacheMap->SectionSize; + SharedCacheMap->SectionSize = FileSizes->AllocationSize; + SharedCacheMap->FileSize = FileSizes->FileSize; + SharedCacheMap->ValidDataLength = FileSizes->ValidDataLength; + KeReleaseSpinLock(&SharedCacheMap->CacheMapLock, OldIrql); + + if (FileSizes->AllocationSize.QuadPart < OldSectionSize.QuadPart) { CcPurgeCacheSection(FileObject->SectionObjectPointer, &FileSizes->AllocationSize, @@ -303,46 +317,9 @@ CcSetFileSizes ( } else { - PROS_VACB LastVacb; - - /* - * If file (allocation) size has increased, then we need to check whether - * it just grows in a single VACB (the last one). - * If so, we must mark the VACB as invalid to trigger a read to the - * FSD at the next VACB usage, and thus avoid returning garbage - */ - - /* Check for allocation size and the last VACB */ - if (SharedCacheMap->SectionSize.QuadPart < FileSizes->AllocationSize.QuadPart && - SharedCacheMap->SectionSize.QuadPart % VACB_MAPPING_GRANULARITY) - { - LastVacb = CcRosLookupVacb(SharedCacheMap, - SharedCacheMap->SectionSize.QuadPart); - if (LastVacb != NULL) - { - /* Mark it as invalid */ - CcRosReleaseVacb(SharedCacheMap, LastVacb, LastVacb->Dirty ? LastVacb->Valid : FALSE, FALSE, FALSE); - } - } - - /* Check for file size and the last VACB */ - if (SharedCacheMap->FileSize.QuadPart < FileSizes->FileSize.QuadPart && - SharedCacheMap->FileSize.QuadPart % VACB_MAPPING_GRANULARITY) - { - LastVacb = CcRosLookupVacb(SharedCacheMap, - SharedCacheMap->FileSize.QuadPart); - if (LastVacb != NULL) - { - /* Mark it as invalid */ - CcRosReleaseVacb(SharedCacheMap, LastVacb, LastVacb->Dirty ? LastVacb->Valid : FALSE, FALSE, FALSE); - } - } + /* Extend our section object */ + MmExtendSection(SharedCacheMap->Section, &SharedCacheMap->SectionSize); } - - KeAcquireSpinLock(&SharedCacheMap->CacheMapLock, &oldirql); - SharedCacheMap->SectionSize = FileSizes->AllocationSize; - SharedCacheMap->FileSize = FileSizes->FileSize; - KeReleaseSpinLock(&SharedCacheMap->CacheMapLock, oldirql); } /* diff --git a/ntoskrnl/cc/pin.c b/ntoskrnl/cc/pin.c index b839e71dafd..c48f1068042 100644 --- a/ntoskrnl/cc/pin.c +++ b/ntoskrnl/cc/pin.c @@ -67,91 +67,6 @@ CcpFindBcb( return (Found ? Bcb : NULL); } -static -BOOLEAN -NTAPI -CcpMapData( - IN PROS_SHARED_CACHE_MAP SharedCacheMap, - IN PLARGE_INTEGER FileOffset, - IN ULONG Length, - IN ULONG Flags, - OUT PROS_VACB *pVacb, - OUT PVOID *pBuffer) -{ - LONGLONG ReadOffset, BaseOffset; - BOOLEAN Valid; - PROS_VACB Vacb; - NTSTATUS Status; - LONGLONG ROffset; - - ReadOffset = FileOffset->QuadPart; - - DPRINT("SectionSize %I64x, FileSize %I64x\n", - SharedCacheMap->SectionSize.QuadPart, - SharedCacheMap->FileSize.QuadPart); - - if (ReadOffset % VACB_MAPPING_GRANULARITY + Length > VACB_MAPPING_GRANULARITY) - { - CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> FALSE\n", - SharedCacheMap->FileObject, FileOffset, Length, Flags); - return FALSE; - } - - if (!BooleanFlagOn(Flags, MAP_NO_READ)) - { - static int Warned = 0; - - SetFlag(Flags, MAP_NO_READ); - if (!Warned) - { - DPRINT1("Mapping/pinning with no read not implemented. Forcing read, might fail if wait not allowed\n"); - Warned++; - } - } - - /* Properly round offset and call internal helper for getting a VACB */ - ROffset = ROUND_DOWN(ReadOffset, VACB_MAPPING_GRANULARITY); - Status = CcRosGetVacb(SharedCacheMap, - ROffset, - &BaseOffset, - pBuffer, - &Valid, - &Vacb); - if (!NT_SUCCESS(Status)) - { - CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> FALSE\n", - SharedCacheMap->FileObject, FileOffset, Length, Flags); - ExRaiseStatus(Status); - return FALSE; - } - - if (!Valid && BooleanFlagOn(Flags, MAP_NO_READ)) - { - if (!BooleanFlagOn(Flags, MAP_WAIT)) - { - CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE, FALSE); - CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> FALSE\n", - SharedCacheMap->FileObject, FileOffset, Length, Flags); - return FALSE; - } - - Status = CcReadVirtualAddress(Vacb); - if (!NT_SUCCESS(Status)) - { - CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE, FALSE); - CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> FALSE\n", - SharedCacheMap->FileObject, FileOffset, Length, Flags); - ExRaiseStatus(Status); - return FALSE; - } - } - - *pBuffer = (PUCHAR)*pBuffer + ReadOffset % VACB_MAPPING_GRANULARITY; - *pVacb = Vacb; - - return TRUE; -} - static VOID CcpDereferenceBcb( @@ -176,7 +91,6 @@ CcpDereferenceBcb( */ CcRosReleaseVacb(SharedCacheMap, Bcb->Vacb, - TRUE, FALSE, FALSE); @@ -206,13 +120,13 @@ CcpGetAppropriateBcb( iBcb = ExAllocateFromNPagedLookasideList(&iBcbLookasideList); if (iBcb == NULL) { - CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); + CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE); return NULL; } RtlZeroMemory(iBcb, sizeof(*iBcb)); - iBcb->PFCB.NodeTypeCode = 0xDE45; /* Undocumented (CAPTIVE_PUBLIC_BCB_NODETYPECODE) */ - iBcb->PFCB.NodeByteSize = sizeof(PUBLIC_BCB); + iBcb->PFCB.NodeTypeCode = 0x2FD; /* As per KMTests */ + iBcb->PFCB.NodeByteSize = 0; iBcb->PFCB.MappedLength = Length; iBcb->PFCB.MappedFileOffset = *FileOffset; iBcb->Vacb = Vacb; @@ -257,7 +171,7 @@ CcpGetAppropriateBcb( if (DupBcb != NULL) { /* Delete the loser */ - CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); + CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE); ExDeleteResourceLite(&iBcb->Lock); ExFreeToNPagedLookasideList(&iBcbLookasideList, iBcb); } @@ -304,44 +218,48 @@ CcpPinData( OUT PVOID * Buffer) { PINTERNAL_BCB NewBcb; - BOOLEAN Result; - PROS_VACB Vacb; KIRQL OldIrql; - ULONG MapFlags; + ULONG VacbOffset; + NTSTATUS Status; + BOOLEAN Result; + + VacbOffset = (ULONG)(FileOffset->QuadPart % VACB_MAPPING_GRANULARITY); + + if ((VacbOffset + Length) > VACB_MAPPING_GRANULARITY) + { + /* Complain loudly, we shoud pin the whole range */ + DPRINT1("TRUNCATING DATA PIN FROM %lu to %lu!\n", Length, VACB_MAPPING_GRANULARITY - VacbOffset); + Length = VACB_MAPPING_GRANULARITY - VacbOffset; + } KeAcquireSpinLock(&SharedCacheMap->BcbSpinLock, &OldIrql); NewBcb = CcpFindBcb(SharedCacheMap, FileOffset, Length, TRUE); if (NewBcb != NULL) { + BOOLEAN Result; + ++NewBcb->RefCount; KeReleaseSpinLock(&SharedCacheMap->BcbSpinLock, OldIrql); if (BooleanFlagOn(Flags, PIN_EXCLUSIVE)) - { Result = ExAcquireResourceExclusiveLite(&NewBcb->Lock, BooleanFlagOn(Flags, PIN_WAIT)); - } else - { Result = ExAcquireSharedStarveExclusive(&NewBcb->Lock, BooleanFlagOn(Flags, PIN_WAIT)); - } if (!Result) { CcpDereferenceBcb(SharedCacheMap, NewBcb); - NewBcb = NULL; - } - else - { - NewBcb->PinCount++; - *Bcb = NewBcb; - *Buffer = (PUCHAR)NewBcb->Vacb->BaseAddress + FileOffset->QuadPart % VACB_MAPPING_GRANULARITY; + return FALSE; } - return Result; + NewBcb->PinCount++; } else { + LONGLONG ROffset; + PROS_VACB Vacb; + KeReleaseSpinLock(&SharedCacheMap->BcbSpinLock, OldIrql); if (BooleanFlagOn(Flags, PIN_IF_BCB)) @@ -349,29 +267,50 @@ CcpPinData( return FALSE; } - MapFlags = Flags & PIN_WAIT; - if (BooleanFlagOn(Flags, PIN_NO_READ)) + /* Properly round offset and call internal helper for getting a VACB */ + ROffset = ROUND_DOWN(FileOffset->QuadPart, VACB_MAPPING_GRANULARITY); + Status = CcRosGetVacb(SharedCacheMap, ROffset, &Vacb); + if (!NT_SUCCESS(Status)) { - SetFlag(MapFlags, MAP_NO_READ); + CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> FALSE\n", + SharedCacheMap->FileObject, FileOffset, Length, Flags); + ExRaiseStatus(Status); + return FALSE; } - Result = CcpMapData(SharedCacheMap, FileOffset, Length, MapFlags, &Vacb, Buffer); - if (Result) + NewBcb = CcpGetAppropriateBcb(SharedCacheMap, Vacb, FileOffset, Length, Flags, TRUE); + if (NewBcb == NULL) { - NewBcb = CcpGetAppropriateBcb(SharedCacheMap, Vacb, FileOffset, Length, Flags, TRUE); - if (NewBcb == NULL) - { - CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); - Result = FALSE; - } - else - { - *Bcb = NewBcb; - } + CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE); + return FALSE; } } - return Result; + Result = FALSE; + _SEH2_TRY + { + /* Ensure the pages are resident */ + Result = CcRosEnsureVacbResident(NewBcb->Vacb, + BooleanFlagOn(Flags, PIN_WAIT), + BooleanFlagOn(Flags, PIN_NO_READ), + VacbOffset, Length); + } + _SEH2_FINALLY + { + if (!Result) + { + CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> FALSE\n", + SharedCacheMap->FileObject, FileOffset, Length, Flags); + CcUnpinData(&NewBcb->PFCB); + return FALSE; + } + } + _SEH2_END; + + *Bcb = &NewBcb->PFCB; + *Buffer = (PVOID)((ULONG_PTR)NewBcb->Vacb->BaseAddress + VacbOffset); + + return TRUE; } /* @@ -387,13 +326,15 @@ CcMapData ( OUT PVOID *pBcb, OUT PVOID *pBuffer) { - BOOLEAN Ret; KIRQL OldIrql; PINTERNAL_BCB iBcb; PROS_VACB Vacb; PROS_SHARED_CACHE_MAP SharedCacheMap; + ULONG VacbOffset; + NTSTATUS Status; + BOOLEAN Result; - DPRINT("CcMapData(FileObject 0x%p, FileOffset %I64x, Length %lu, Flags 0x%lx," + CCTRACE(CC_API_DEBUG, "CcMapData(FileObject 0x%p, FileOffset 0x%I64x, Length %lu, Flags 0x%lx," " pBcb 0x%p, pBuffer 0x%p)\n", FileObject, FileOffset->QuadPart, Length, Flags, pBcb, pBuffer); @@ -413,6 +354,14 @@ CcMapData ( ++CcMapDataNoWait; } + VacbOffset = (ULONG)(FileOffset->QuadPart % VACB_MAPPING_GRANULARITY); + /* KMTests seem to show that it is allowed to call accross mapping granularity */ + if ((VacbOffset + Length) > VACB_MAPPING_GRANULARITY) + { + DPRINT1("TRUNCATING DATA MAP FROM %lu to %lu!\n", Length, VACB_MAPPING_GRANULARITY - VacbOffset); + Length = VACB_MAPPING_GRANULARITY - VacbOffset; + } + KeAcquireSpinLock(&SharedCacheMap->BcbSpinLock, &OldIrql); iBcb = CcpFindBcb(SharedCacheMap, FileOffset, Length, FALSE); @@ -420,34 +369,54 @@ CcMapData ( { KeReleaseSpinLock(&SharedCacheMap->BcbSpinLock, OldIrql); - Ret = CcpMapData(SharedCacheMap, FileOffset, Length, Flags, &Vacb, pBuffer); - if (Ret) + /* Call internal helper for getting a VACB */ + Status = CcRosGetVacb(SharedCacheMap, FileOffset->QuadPart, &Vacb); + if (!NT_SUCCESS(Status)) { - iBcb = CcpGetAppropriateBcb(SharedCacheMap, Vacb, FileOffset, Length, 0, FALSE); - if (iBcb == NULL) - { - CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); - Ret = FALSE; - } - else - { - *pBcb = iBcb; - } + CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> FALSE\n", + SharedCacheMap->FileObject, FileOffset, Length, Flags); + ExRaiseStatus(Status); + return FALSE; + } + + iBcb = CcpGetAppropriateBcb(SharedCacheMap, Vacb, FileOffset, Length, 0, FALSE); + if (iBcb == NULL) + { + CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE); + CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> FALSE\n", + SharedCacheMap->FileObject, FileOffset, Length, Flags); + return FALSE; } } else { ++iBcb->RefCount; KeReleaseSpinLock(&SharedCacheMap->BcbSpinLock, OldIrql); - - *pBcb = iBcb; - *pBuffer = (PUCHAR)iBcb->Vacb->BaseAddress + FileOffset->QuadPart % VACB_MAPPING_GRANULARITY; - Ret = TRUE; } - CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> %d Bcb=%p\n", - FileObject, FileOffset, Length, Flags, Ret, *pBcb); - return Ret; + _SEH2_TRY + { + Result = FALSE; + /* Ensure the pages are resident */ + Result = CcRosEnsureVacbResident(iBcb->Vacb, BooleanFlagOn(Flags, MAP_WAIT), + BooleanFlagOn(Flags, MAP_NO_READ), VacbOffset, Length); + } + _SEH2_FINALLY + { + if (!Result) + { + CcpDereferenceBcb(SharedCacheMap, iBcb); + return FALSE; + } + } + _SEH2_END; + + *pBcb = &iBcb->PFCB; + *pBuffer = (PVOID)((ULONG_PTR)iBcb->Vacb->BaseAddress + VacbOffset); + + CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> TRUE Bcb=%p, Buffer %p\n", + FileObject, FileOffset, Length, Flags, *pBcb, *pBuffer); + return Result; } /* @@ -482,14 +451,14 @@ CcPinMappedData ( return FALSE; } - iBcb = *Bcb; + iBcb = *Bcb ? CONTAINING_RECORD(*Bcb, INTERNAL_BCB, PFCB) : NULL; ++CcPinMappedDataCount; Result = CcpPinData(SharedCacheMap, FileOffset, Length, Flags, Bcb, &Buffer); if (Result) { - CcUnpinData(iBcb); + CcUnpinData(&iBcb->PFCB); } return Result; @@ -573,10 +542,14 @@ CcSetDirtyPinnedData ( IN PVOID Bcb, IN PLARGE_INTEGER Lsn) { - PINTERNAL_BCB iBcb = Bcb; + PINTERNAL_BCB iBcb = CONTAINING_RECORD(Bcb, INTERNAL_BCB, PFCB); - CCTRACE(CC_API_DEBUG, "Bcb=%p Lsn=%p\n", - Bcb, Lsn); + CCTRACE(CC_API_DEBUG, "Bcb=%p Lsn=%p\n", Bcb, Lsn); + + /* Tell Mm */ + MmMakePagesDirty(NULL, + Add2Ptr(iBcb->Vacb->BaseAddress, iBcb->PFCB.MappedFileOffset.QuadPart - iBcb->Vacb->FileOffset.QuadPart), + iBcb->PFCB.MappedLength); if (!iBcb->Vacb->Dirty) { @@ -606,8 +579,7 @@ CcUnpinDataForThread ( IN PVOID Bcb, IN ERESOURCE_THREAD ResourceThreadId) { - PINTERNAL_BCB iBcb = Bcb; - PROS_SHARED_CACHE_MAP SharedCacheMap; + PINTERNAL_BCB iBcb = CONTAINING_RECORD(Bcb, INTERNAL_BCB, PFCB); CCTRACE(CC_API_DEBUG, "Bcb=%p ResourceThreadId=%lu\n", Bcb, ResourceThreadId); @@ -617,8 +589,7 @@ CcUnpinDataForThread ( iBcb->PinCount--; } - SharedCacheMap = iBcb->Vacb->SharedCacheMap; - CcpDereferenceBcb(SharedCacheMap, iBcb); + CcpDereferenceBcb(iBcb->Vacb->SharedCacheMap, iBcb); } /* @@ -629,7 +600,7 @@ NTAPI CcRepinBcb ( IN PVOID Bcb) { - PINTERNAL_BCB iBcb = Bcb; + PINTERNAL_BCB iBcb = CONTAINING_RECORD(Bcb, INTERNAL_BCB, PFCB); CCTRACE(CC_API_DEBUG, "Bcb=%p\n", Bcb); @@ -646,7 +617,7 @@ CcUnpinRepinnedBcb ( IN BOOLEAN WriteThrough, IN PIO_STATUS_BLOCK IoStatus) { - PINTERNAL_BCB iBcb = Bcb; + PINTERNAL_BCB iBcb = CONTAINING_RECORD(Bcb, INTERNAL_BCB, PFCB); KIRQL OldIrql; PROS_SHARED_CACHE_MAP SharedCacheMap; @@ -655,29 +626,25 @@ CcUnpinRepinnedBcb ( SharedCacheMap = iBcb->Vacb->SharedCacheMap; IoStatus->Status = STATUS_SUCCESS; + if (WriteThrough) + { + CcFlushCache(iBcb->Vacb->SharedCacheMap->FileObject->SectionObjectPointer, + &iBcb->PFCB.MappedFileOffset, + iBcb->PFCB.MappedLength, + IoStatus); + } + else + { + IoStatus->Status = STATUS_SUCCESS; + IoStatus->Information = 0; + } + KeAcquireSpinLock(&SharedCacheMap->BcbSpinLock, &OldIrql); if (--iBcb->RefCount == 0) { RemoveEntryList(&iBcb->BcbEntry); KeReleaseSpinLock(&SharedCacheMap->BcbSpinLock, OldIrql); - IoStatus->Information = 0; - if (WriteThrough) - { - if (iBcb->Vacb->Dirty) - { - IoStatus->Status = CcRosFlushVacb(iBcb->Vacb); - } - else - { - IoStatus->Status = STATUS_SUCCESS; - } - } - else - { - IoStatus->Status = STATUS_SUCCESS; - } - if (iBcb->PinCount != 0) { ExReleaseResourceLite(&iBcb->Lock); @@ -692,7 +659,6 @@ CcUnpinRepinnedBcb ( */ CcRosReleaseVacb(iBcb->Vacb->SharedCacheMap, iBcb->Vacb, - TRUE, FALSE, FALSE); diff --git a/ntoskrnl/cc/view.c b/ntoskrnl/cc/view.c index adb28bb84ab..4c094f074d5 100644 --- a/ntoskrnl/cc/view.c +++ b/ntoskrnl/cc/view.c @@ -110,7 +110,6 @@ ULONG CcRosVacbGetRefCount_(PROS_VACB vacb, PCSTR file, INT line) /* FUNCTIONS *****************************************************************/ VOID -NTAPI CcRosTraceCacheMap ( PROS_SHARED_CACHE_MAP SharedCacheMap, BOOLEAN Trace ) @@ -138,8 +137,8 @@ CcRosTraceCacheMap ( current = CONTAINING_RECORD(current_entry, ROS_VACB, CacheMapVacbListEntry); current_entry = current_entry->Flink; - DPRINT1(" VACB 0x%p enabled, RefCount %lu, Dirty %u, PageOut %lu\n", - current, current->ReferenceCount, current->Dirty, current->PageOut ); + DPRINT1(" VACB 0x%p enabled, RefCount %lu, Dirty %u, PageOut %lu, BaseAddress %p, FileOffset %I64d\n", + current, current->ReferenceCount, current->Dirty, current->PageOut, current->BaseAddress, current->FileOffset.QuadPart); } KeReleaseSpinLockFromDpcLevel(&SharedCacheMap->CacheMapLock); @@ -157,25 +156,51 @@ CcRosTraceCacheMap ( } NTSTATUS -NTAPI CcRosFlushVacb ( - PROS_VACB Vacb) + _In_ PROS_VACB Vacb, + _In_ PIO_STATUS_BLOCK Iosb) { NTSTATUS Status; + BOOLEAN HaveLock = FALSE; + PROS_SHARED_CACHE_MAP SharedCacheMap = Vacb->SharedCacheMap; CcRosUnmarkDirtyVacb(Vacb, TRUE); - Status = CcWriteVirtualAddress(Vacb); - if (!NT_SUCCESS(Status)) + /* Lock for flush, if we are not already the top-level */ + if (IoGetTopLevelIrp() != (PIRP)FSRTL_CACHE_TOP_LEVEL_IRP) { + Status = FsRtlAcquireFileForCcFlushEx(Vacb->SharedCacheMap->FileObject); + if (!NT_SUCCESS(Status)) + goto quit; + HaveLock = TRUE; + } + + Status = MmFlushSegment(SharedCacheMap->FileObject->SectionObjectPointer, + &Vacb->FileOffset, + VACB_MAPPING_GRANULARITY, + Iosb); + + if (HaveLock) + { + FsRtlReleaseFileForCcFlush(Vacb->SharedCacheMap->FileObject); + } + +quit: + if (!NT_SUCCESS(Status)) CcRosMarkDirtyVacb(Vacb); + else + { + /* Update VDL */ + if (SharedCacheMap->ValidDataLength.QuadPart < (Vacb->FileOffset.QuadPart + VACB_MAPPING_GRANULARITY)) + { + SharedCacheMap->ValidDataLength.QuadPart = Vacb->FileOffset.QuadPart + VACB_MAPPING_GRANULARITY; + } } return Status; } NTSTATUS -NTAPI CcRosFlushDirtyPages ( ULONG Target, PULONG Count, @@ -183,10 +208,9 @@ CcRosFlushDirtyPages ( BOOLEAN CalledFromLazy) { PLIST_ENTRY current_entry; - PROS_VACB current; - BOOLEAN Locked; NTSTATUS Status; KIRQL OldIrql; + BOOLEAN FlushAll = (Target == MAXULONG); DPRINT("CcRosFlushDirtyPages(Target %lu)\n", Target); @@ -201,8 +225,20 @@ CcRosFlushDirtyPages ( DPRINT("No Dirty pages\n"); } - while ((current_entry != &DirtyVacbListHead) && (Target > 0)) + while (((current_entry != &DirtyVacbListHead) && (Target > 0)) || FlushAll) { + PROS_SHARED_CACHE_MAP SharedCacheMap; + PROS_VACB current; + BOOLEAN Locked; + + if (current_entry == &DirtyVacbListHead) + { + ASSERT(FlushAll); + if (IsListEmpty(&DirtyVacbListHead)) + break; + current_entry = DirtyVacbListHead.Flink; + } + current = CONTAINING_RECORD(current_entry, ROS_VACB, DirtyVacbListEntry); @@ -210,17 +246,17 @@ CcRosFlushDirtyPages ( CcRosVacbIncRefCount(current); + SharedCacheMap = current->SharedCacheMap; + /* When performing lazy write, don't handle temporary files */ - if (CalledFromLazy && - BooleanFlagOn(current->SharedCacheMap->FileObject->Flags, FO_TEMPORARY_FILE)) + if (CalledFromLazy && BooleanFlagOn(SharedCacheMap->FileObject->Flags, FO_TEMPORARY_FILE)) { CcRosVacbDecRefCount(current); continue; } /* Don't attempt to lazy write the files that asked not to */ - if (CalledFromLazy && - BooleanFlagOn(current->SharedCacheMap->Flags, WRITEBEHIND_DISABLED)) + if (CalledFromLazy && BooleanFlagOn(SharedCacheMap->Flags, WRITEBEHIND_DISABLED)) { CcRosVacbDecRefCount(current); continue; @@ -228,21 +264,32 @@ CcRosFlushDirtyPages ( ASSERT(current->Dirty); - KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); - - Locked = current->SharedCacheMap->Callbacks->AcquireForLazyWrite( - current->SharedCacheMap->LazyWriteContext, Wait); - if (!Locked) + /* Do not lazy-write the same file concurrently. Fastfat ASSERTS on that */ + if (SharedCacheMap->Flags & SHARED_CACHE_MAP_IN_LAZYWRITE) { - OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); CcRosVacbDecRefCount(current); continue; } - Status = CcRosFlushVacb(current); + SharedCacheMap->Flags |= SHARED_CACHE_MAP_IN_LAZYWRITE; - current->SharedCacheMap->Callbacks->ReleaseFromLazyWrite( - current->SharedCacheMap->LazyWriteContext); + KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); + + Locked = SharedCacheMap->Callbacks->AcquireForLazyWrite(SharedCacheMap->LazyWriteContext, Wait); + if (!Locked) + { + DPRINT("Not locked!"); + ASSERT(!Wait); + CcRosVacbDecRefCount(current); + OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); + SharedCacheMap->Flags &= ~SHARED_CACHE_MAP_IN_LAZYWRITE; + continue; + } + + IO_STATUS_BLOCK Iosb; + Status = CcRosFlushVacb(current, &Iosb); + + SharedCacheMap->Callbacks->ReleaseFromLazyWrite(SharedCacheMap->LazyWriteContext); /* We release the VACB before acquiring the lock again, because * CcRosVacbDecRefCount might free the VACB, as CcRosFlushVacb dropped a @@ -251,6 +298,8 @@ CcRosFlushDirtyPages ( CcRosVacbDecRefCount(current); OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); + SharedCacheMap->Flags &= ~SHARED_CACHE_MAP_IN_LAZYWRITE; + if (!NT_SUCCESS(Status) && (Status != STATUS_END_OF_FILE) && (Status != STATUS_MEDIA_WRITE_PROTECTED)) { @@ -261,18 +310,21 @@ CcRosFlushDirtyPages ( ULONG PagesFreed; /* How many pages did we free? */ - PagesFreed = VACB_MAPPING_GRANULARITY / PAGE_SIZE; + PagesFreed = Iosb.Information / PAGE_SIZE; (*Count) += PagesFreed; - /* Make sure we don't overflow target! */ - if (Target < PagesFreed) + if (!Wait) { - /* If we would have, jump to zero directly */ - Target = 0; - } - else - { - Target -= PagesFreed; + /* Make sure we don't overflow target! */ + if (Target < PagesFreed) + { + /* If we would have, jump to zero directly */ + Target = 0; + } + else + { + Target -= PagesFreed; + } } } @@ -287,151 +339,16 @@ CcRosFlushDirtyPages ( } NTSTATUS -CcRosTrimCache ( - ULONG Target, - ULONG Priority, - PULONG NrFreed) -/* - * FUNCTION: Try to free some memory from the file cache. - * ARGUMENTS: - * Target - The number of pages to be freed. - * Priority - The priority of free (currently unused). - * NrFreed - Points to a variable where the number of pages - * actually freed is returned. - */ -{ - PLIST_ENTRY current_entry; - PROS_VACB current; - ULONG PagesFreed; - KIRQL oldIrql; - LIST_ENTRY FreeList; - PFN_NUMBER Page; - ULONG i; - BOOLEAN FlushedPages = FALSE; - - DPRINT("CcRosTrimCache(Target %lu)\n", Target); - - InitializeListHead(&FreeList); - - *NrFreed = 0; - -retry: - oldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); - - current_entry = VacbLruListHead.Flink; - while (current_entry != &VacbLruListHead) - { - ULONG Refs; - - current = CONTAINING_RECORD(current_entry, - ROS_VACB, - VacbLruListEntry); - current_entry = current_entry->Flink; - - KeAcquireSpinLockAtDpcLevel(¤t->SharedCacheMap->CacheMapLock); - - /* Reference the VACB */ - CcRosVacbIncRefCount(current); - - /* Check if it's mapped and not dirty */ - if (InterlockedCompareExchange((PLONG)¤t->MappedCount, 0, 0) > 0 && !current->Dirty) - { - /* We have to break these locks because Cc sucks */ - KeReleaseSpinLockFromDpcLevel(¤t->SharedCacheMap->CacheMapLock); - KeReleaseQueuedSpinLock(LockQueueMasterLock, oldIrql); - - /* Page out the VACB */ - for (i = 0; i < VACB_MAPPING_GRANULARITY / PAGE_SIZE; i++) - { - Page = (PFN_NUMBER)(MmGetPhysicalAddress((PUCHAR)current->BaseAddress + (i * PAGE_SIZE)).QuadPart >> PAGE_SHIFT); - - MmPageOutPhysicalAddress(Page); - } - - /* Reacquire the locks */ - oldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); - KeAcquireSpinLockAtDpcLevel(¤t->SharedCacheMap->CacheMapLock); - } - - /* Dereference the VACB */ - Refs = CcRosVacbDecRefCount(current); - - /* Check if we can free this entry now */ - if (Refs < 2) - { - ASSERT(!current->Dirty); - ASSERT(!current->MappedCount); - ASSERT(Refs == 1); - - RemoveEntryList(¤t->CacheMapVacbListEntry); - RemoveEntryList(¤t->VacbLruListEntry); - InitializeListHead(¤t->VacbLruListEntry); - InsertHeadList(&FreeList, ¤t->CacheMapVacbListEntry); - - /* Calculate how many pages we freed for Mm */ - PagesFreed = min(VACB_MAPPING_GRANULARITY / PAGE_SIZE, Target); - Target -= PagesFreed; - (*NrFreed) += PagesFreed; - } - - KeReleaseSpinLockFromDpcLevel(¤t->SharedCacheMap->CacheMapLock); - } - - KeReleaseQueuedSpinLock(LockQueueMasterLock, oldIrql); - - /* Try flushing pages if we haven't met our target */ - if ((Target > 0) && !FlushedPages) - { - /* Flush dirty pages to disk */ - CcRosFlushDirtyPages(Target, &PagesFreed, FALSE, FALSE); - FlushedPages = TRUE; - - /* We can only swap as many pages as we flushed */ - if (PagesFreed < Target) Target = PagesFreed; - - /* Check if we flushed anything */ - if (PagesFreed != 0) - { - /* Try again after flushing dirty pages */ - DPRINT("Flushed %lu dirty cache pages to disk\n", PagesFreed); - goto retry; - } - } - - while (!IsListEmpty(&FreeList)) - { - ULONG Refs; - - current_entry = RemoveHeadList(&FreeList); - current = CONTAINING_RECORD(current_entry, - ROS_VACB, - CacheMapVacbListEntry); - InitializeListHead(¤t->CacheMapVacbListEntry); - Refs = CcRosVacbDecRefCount(current); - ASSERT(Refs == 0); - } - - DPRINT("Evicted %lu cache pages\n", (*NrFreed)); - - return STATUS_SUCCESS; -} - -NTSTATUS -NTAPI CcRosReleaseVacb ( PROS_SHARED_CACHE_MAP SharedCacheMap, PROS_VACB Vacb, - BOOLEAN Valid, BOOLEAN Dirty, BOOLEAN Mapped) { ULONG Refs; ASSERT(SharedCacheMap); - DPRINT("CcRosReleaseVacb(SharedCacheMap 0x%p, Vacb 0x%p, Valid %u)\n", - SharedCacheMap, Vacb, Valid); - - Vacb->Valid = Valid; + DPRINT("CcRosReleaseVacb(SharedCacheMap 0x%p, Vacb 0x%p)\n", SharedCacheMap, Vacb); if (Dirty && !Vacb->Dirty) { @@ -454,7 +371,6 @@ CcRosReleaseVacb ( /* Returns with VACB Lock Held! */ PROS_VACB -NTAPI CcRosLookupVacb ( PROS_SHARED_CACHE_MAP SharedCacheMap, LONGLONG FileOffset) @@ -498,7 +414,6 @@ CcRosLookupVacb ( } VOID -NTAPI CcRosMarkDirtyVacb ( PROS_VACB Vacb) { @@ -513,6 +428,7 @@ CcRosMarkDirtyVacb ( ASSERT(!Vacb->Dirty); InsertTailList(&DirtyVacbListHead, &Vacb->DirtyVacbListEntry); + /* FIXME: There is no reason to account for the whole VACB. */ CcTotalDirtyPages += VACB_MAPPING_GRANULARITY / PAGE_SIZE; Vacb->SharedCacheMap->DirtyPages += VACB_MAPPING_GRANULARITY / PAGE_SIZE; CcRosVacbIncRefCount(Vacb); @@ -534,7 +450,6 @@ CcRosMarkDirtyVacb ( } VOID -NTAPI CcRosUnmarkDirtyVacb ( PROS_VACB Vacb, BOOLEAN LockViews) @@ -556,8 +471,10 @@ CcRosUnmarkDirtyVacb ( RemoveEntryList(&Vacb->DirtyVacbListEntry); InitializeListHead(&Vacb->DirtyVacbListEntry); + CcTotalDirtyPages -= VACB_MAPPING_GRANULARITY / PAGE_SIZE; Vacb->SharedCacheMap->DirtyPages -= VACB_MAPPING_GRANULARITY / PAGE_SIZE; + CcRosVacbDecRefCount(Vacb); if (LockViews) @@ -567,132 +484,6 @@ CcRosUnmarkDirtyVacb ( } } -NTSTATUS -NTAPI -CcRosMarkDirtyFile ( - PROS_SHARED_CACHE_MAP SharedCacheMap, - LONGLONG FileOffset) -{ - PROS_VACB Vacb; - - ASSERT(SharedCacheMap); - - DPRINT("CcRosMarkDirtyVacb(SharedCacheMap 0x%p, FileOffset %I64u)\n", - SharedCacheMap, FileOffset); - - Vacb = CcRosLookupVacb(SharedCacheMap, FileOffset); - if (Vacb == NULL) - { - KeBugCheck(CACHE_MANAGER); - } - - CcRosReleaseVacb(SharedCacheMap, Vacb, Vacb->Valid, TRUE, FALSE); - - return STATUS_SUCCESS; -} - -/* - * Note: this is not the contrary function of - * CcRosMapVacbInKernelSpace() - */ -NTSTATUS -NTAPI -CcRosUnmapVacb ( - PROS_SHARED_CACHE_MAP SharedCacheMap, - LONGLONG FileOffset, - BOOLEAN NowDirty) -{ - PROS_VACB Vacb; - - ASSERT(SharedCacheMap); - - DPRINT("CcRosUnmapVacb(SharedCacheMap 0x%p, FileOffset %I64u, NowDirty %u)\n", - SharedCacheMap, FileOffset, NowDirty); - - Vacb = CcRosLookupVacb(SharedCacheMap, FileOffset); - if (Vacb == NULL) - { - return STATUS_UNSUCCESSFUL; - } - - ASSERT(Vacb->MappedCount != 0); - if (InterlockedDecrement((PLONG)&Vacb->MappedCount) == 0) - { - CcRosVacbDecRefCount(Vacb); - } - - CcRosReleaseVacb(SharedCacheMap, Vacb, Vacb->Valid, NowDirty, FALSE); - - return STATUS_SUCCESS; -} - -static -NTSTATUS -CcRosMapVacbInKernelSpace( - PROS_VACB Vacb) -{ - ULONG i; - NTSTATUS Status; - ULONG_PTR NumberOfPages; - PVOID BaseAddress = NULL; - - /* Create a memory area. */ - MmLockAddressSpace(MmGetKernelAddressSpace()); - Status = MmCreateMemoryArea(MmGetKernelAddressSpace(), - 0, // nothing checks for VACB mareas, so set to 0 - &BaseAddress, - VACB_MAPPING_GRANULARITY, - PAGE_READWRITE, - (PMEMORY_AREA*)&Vacb->MemoryArea, - 0, - PAGE_SIZE); - ASSERT(Vacb->BaseAddress == NULL); - Vacb->BaseAddress = BaseAddress; - MmUnlockAddressSpace(MmGetKernelAddressSpace()); - if (!NT_SUCCESS(Status)) - { - DPRINT1("MmCreateMemoryArea failed with %lx for VACB %p\n", Status, Vacb); - return Status; - } - - ASSERT(((ULONG_PTR)Vacb->BaseAddress % PAGE_SIZE) == 0); - ASSERT((ULONG_PTR)Vacb->BaseAddress > (ULONG_PTR)MmSystemRangeStart); - ASSERT((ULONG_PTR)Vacb->BaseAddress + VACB_MAPPING_GRANULARITY - 1 > (ULONG_PTR)MmSystemRangeStart); - - /* Create a virtual mapping for this memory area */ - NumberOfPages = BYTES_TO_PAGES(VACB_MAPPING_GRANULARITY); - for (i = 0; i < NumberOfPages; i++) - { - PFN_NUMBER PageFrameNumber; - - MI_SET_USAGE(MI_USAGE_CACHE); - Status = MmRequestPageMemoryConsumer(MC_CACHE, TRUE, &PageFrameNumber); - if (PageFrameNumber == 0) - { - DPRINT1("Unable to allocate page\n"); - KeBugCheck(MEMORY_MANAGEMENT); - } - - ASSERT(BaseAddress == Vacb->BaseAddress); - ASSERT(i * PAGE_SIZE < VACB_MAPPING_GRANULARITY); - ASSERT((ULONG_PTR)Vacb->BaseAddress + (i * PAGE_SIZE) >= (ULONG_PTR)BaseAddress); - ASSERT((ULONG_PTR)Vacb->BaseAddress + (i * PAGE_SIZE) > (ULONG_PTR)MmSystemRangeStart); - - Status = MmCreateVirtualMapping(NULL, - (PVOID)((ULONG_PTR)Vacb->BaseAddress + (i * PAGE_SIZE)), - PAGE_READWRITE, - &PageFrameNumber, - 1); - if (!NT_SUCCESS(Status)) - { - DPRINT1("Unable to create virtual mapping\n"); - KeBugCheck(MEMORY_MANAGEMENT); - } - } - - return STATUS_SUCCESS; -} - static BOOLEAN CcRosFreeUnusedVacb ( @@ -789,30 +580,18 @@ CcRosCreateVacb ( KIRQL oldIrql; ULONG Refs; BOOLEAN Retried; + SIZE_T ViewSize = VACB_MAPPING_GRANULARITY; ASSERT(SharedCacheMap); DPRINT("CcRosCreateVacb()\n"); - if (FileOffset >= SharedCacheMap->SectionSize.QuadPart) - { - *Vacb = NULL; - return STATUS_INVALID_PARAMETER; - } - current = ExAllocateFromNPagedLookasideList(&VacbLookasideList); current->BaseAddress = NULL; - current->Valid = FALSE; current->Dirty = FALSE; current->PageOut = FALSE; current->FileOffset.QuadPart = ROUND_DOWN(FileOffset, VACB_MAPPING_GRANULARITY); current->SharedCacheMap = SharedCacheMap; -#if DBG - if (SharedCacheMap->Trace) - { - DPRINT1("CacheMap 0x%p: new VACB: 0x%p\n", SharedCacheMap, current); - } -#endif current->MappedCount = 0; current->ReferenceCount = 0; InitializeListHead(¤t->CacheMapVacbListEntry); @@ -823,8 +602,9 @@ CcRosCreateVacb ( Retried = FALSE; Retry: - /* Map VACB in kernel space */ - Status = CcRosMapVacbInKernelSpace(current); + /* Map VACB in system space */ + Status = MmMapViewInSystemSpaceEx(SharedCacheMap->Section, ¤t->BaseAddress, &ViewSize, ¤t->FileOffset, 0); + if (!NT_SUCCESS(Status)) { ULONG Freed; @@ -844,6 +624,14 @@ Retry: return Status; } +#if DBG + if (SharedCacheMap->Trace) + { + DPRINT1("CacheMap 0x%p: new VACB: 0x%p, file offset %I64d, BaseAddress %p\n", + SharedCacheMap, current, current->FileOffset.QuadPart, current->BaseAddress); + } +#endif + oldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); *Vacb = current; @@ -932,14 +720,57 @@ Retry: return Status; } +BOOLEAN +CcRosEnsureVacbResident( + _In_ PROS_VACB Vacb, + _In_ BOOLEAN Wait, + _In_ BOOLEAN NoRead, + _In_ ULONG Offset, + _In_ ULONG Length +) +{ + PVOID BaseAddress; + + ASSERT((Offset + Length) <= VACB_MAPPING_GRANULARITY); + +#if 0 + if ((Vacb->FileOffset.QuadPart + Offset) > Vacb->SharedCacheMap->SectionSize.QuadPart) + { + DPRINT1("Vacb read beyond the file size!\n"); + return FALSE; + } +#endif + + BaseAddress = (PVOID)((ULONG_PTR)Vacb->BaseAddress + Offset); + + /* Check if the pages are resident */ + if (!MmArePagesResident(NULL, BaseAddress, Length)) + { + if (!Wait) + { + return FALSE; + } + + if (!NoRead) + { + PROS_SHARED_CACHE_MAP SharedCacheMap = Vacb->SharedCacheMap; + NTSTATUS Status = MmMakeDataSectionResident(SharedCacheMap->FileObject->SectionObjectPointer, + Vacb->FileOffset.QuadPart + Offset, + Length, + &SharedCacheMap->ValidDataLength); + if (!NT_SUCCESS(Status)) + ExRaiseStatus(Status); + } + } + + return TRUE; +} + + NTSTATUS -NTAPI CcRosGetVacb ( PROS_SHARED_CACHE_MAP SharedCacheMap, LONGLONG FileOffset, - PLONGLONG BaseOffset, - PVOID* BaseAddress, - PBOOLEAN UptoDate, PROS_VACB *Vacb) { PROS_VACB current; @@ -978,13 +809,9 @@ CcRosGetVacb ( KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); /* - * Return information about the VACB to the caller. + * Return the VACB to the caller. */ - *UptoDate = current->Valid; - *BaseAddress = current->BaseAddress; - DPRINT("*BaseAddress %p\n", *BaseAddress); *Vacb = current; - *BaseOffset = current->FileOffset.QuadPart; ASSERT(Refs > 1); @@ -992,18 +819,14 @@ CcRosGetVacb ( } NTSTATUS -NTAPI CcRosRequestVacb ( PROS_SHARED_CACHE_MAP SharedCacheMap, LONGLONG FileOffset, - PVOID* BaseAddress, - PBOOLEAN UptoDate, PROS_VACB *Vacb) /* * FUNCTION: Request a page mapping for a shared cache map */ { - LONGLONG BaseOffset; ASSERT(SharedCacheMap); @@ -1016,30 +839,9 @@ CcRosRequestVacb ( return CcRosGetVacb(SharedCacheMap, FileOffset, - &BaseOffset, - BaseAddress, - UptoDate, Vacb); } -static -VOID -CcFreeCachePage ( - PVOID Context, - MEMORY_AREA* MemoryArea, - PVOID Address, - PFN_NUMBER Page, - SWAPENTRY SwapEntry, - BOOLEAN Dirty) -{ - ASSERT(SwapEntry == 0); - if (Page != 0) - { - ASSERT(MmGetReferenceCountPage(Page) == 1); - MmReleasePageMemoryConsumer(MC_CACHE, Page); - } -} - NTSTATUS CcRosInternalFreeVacb ( PROS_VACB Vacb) @@ -1047,6 +849,8 @@ CcRosInternalFreeVacb ( * FUNCTION: Releases a VACB associated with a shared cache map */ { + NTSTATUS Status; + DPRINT("Freeing VACB 0x%p\n", Vacb); #if DBG if (Vacb->SharedCacheMap->Trace) @@ -1055,13 +859,6 @@ CcRosInternalFreeVacb ( } #endif - MmLockAddressSpace(MmGetKernelAddressSpace()); - MmFreeMemoryArea(MmGetKernelAddressSpace(), - Vacb->MemoryArea, - CcFreeCachePage, - NULL); - MmUnlockAddressSpace(MmGetKernelAddressSpace()); - if (Vacb->ReferenceCount != 0) { DPRINT1("Invalid free: %ld\n", Vacb->ReferenceCount); @@ -1075,6 +872,16 @@ CcRosInternalFreeVacb ( ASSERT(IsListEmpty(&Vacb->CacheMapVacbListEntry)); ASSERT(IsListEmpty(&Vacb->DirtyVacbListEntry)); ASSERT(IsListEmpty(&Vacb->VacbLruListEntry)); + + /* Delete the mapping */ + Status = MmUnmapViewInSystemSpace(Vacb->BaseAddress); + if (!NT_SUCCESS(Status)) + { + DPRINT1("Failed to unmap VACB from System address space! Status 0x%08X\n", Status); + ASSERT(FALSE); + /* Proceed with the deĺetion anyway */ + } + RtlFillMemory(Vacb, sizeof(*Vacb), 0xfd); ExFreeToNPagedLookasideList(&VacbLookasideList, Vacb); return STATUS_SUCCESS; @@ -1092,70 +899,122 @@ CcFlushCache ( OUT PIO_STATUS_BLOCK IoStatus) { PROS_SHARED_CACHE_MAP SharedCacheMap; - LARGE_INTEGER Offset; - LONGLONG RemainingLength; - PROS_VACB current; + LONGLONG FlushStart, FlushEnd; NTSTATUS Status; - CCTRACE(CC_API_DEBUG, "SectionObjectPointers=%p FileOffset=%p Length=%lu\n", - SectionObjectPointers, FileOffset, Length); + CCTRACE(CC_API_DEBUG, "SectionObjectPointers=%p FileOffset=0x%I64X Length=%lu\n", + SectionObjectPointers, FileOffset ? FileOffset->QuadPart : 0LL, Length); - DPRINT("CcFlushCache(SectionObjectPointers 0x%p, FileOffset 0x%p, Length %lu, IoStatus 0x%p)\n", - SectionObjectPointers, FileOffset, Length, IoStatus); - - if (SectionObjectPointers && SectionObjectPointers->SharedCacheMap) + if (!SectionObjectPointers) { - SharedCacheMap = SectionObjectPointers->SharedCacheMap; - ASSERT(SharedCacheMap); - if (FileOffset) - { - Offset = *FileOffset; - RemainingLength = Length; - } - else - { - Offset.QuadPart = 0; - RemainingLength = SharedCacheMap->FileSize.QuadPart; - } + Status = STATUS_INVALID_PARAMETER; + goto quit; + } - if (IoStatus) - { - IoStatus->Status = STATUS_SUCCESS; - IoStatus->Information = 0; - } + if (!SectionObjectPointers->SharedCacheMap) + { + /* Forward this to Mm */ + MmFlushSegment(SectionObjectPointers, FileOffset, Length, IoStatus); + return; + } - while (RemainingLength > 0) - { - current = CcRosLookupVacb(SharedCacheMap, Offset.QuadPart); - if (current != NULL) - { - if (current->Dirty) - { - Status = CcRosFlushVacb(current); - if (!NT_SUCCESS(Status) && IoStatus != NULL) - { - IoStatus->Status = Status; - } - } - - CcRosReleaseVacb(SharedCacheMap, current, current->Valid, FALSE, FALSE); - } - - Offset.QuadPart += VACB_MAPPING_GRANULARITY; - RemainingLength -= min(RemainingLength, VACB_MAPPING_GRANULARITY); - } + SharedCacheMap = SectionObjectPointers->SharedCacheMap; + ASSERT(SharedCacheMap); + if (FileOffset) + { + FlushStart = FileOffset->QuadPart; + Status = RtlLongLongAdd(FlushStart, Length, &FlushEnd); + if (!NT_SUCCESS(Status)) + goto quit; } else { - if (IoStatus) + FlushStart = 0; + FlushEnd = SharedCacheMap->FileSize.QuadPart; + } + + Status = STATUS_SUCCESS; + if (IoStatus) + { + IoStatus->Information = 0; + } + + /* + * We flush the VACBs that we find here. + * If there is no (dirty) VACB, it doesn't mean that there is no data to flush, so we call Mm to be sure. + * This is suboptimal, but this is due to the lack of granularity of how we track dirty cache data + */ + while (FlushStart < FlushEnd) + { + BOOLEAN DirtyVacb = FALSE; + PROS_VACB vacb = CcRosLookupVacb(SharedCacheMap, FlushStart); + + if (vacb != NULL) { - IoStatus->Status = STATUS_INVALID_PARAMETER; + IO_STATUS_BLOCK VacbIosb; + if (vacb->Dirty) + { + Status = CcRosFlushVacb(vacb, &VacbIosb); + if (!NT_SUCCESS(Status)) + { + goto quit; + } + DirtyVacb = TRUE; + } + + CcRosReleaseVacb(SharedCacheMap, vacb, FALSE, FALSE); + + if (IoStatus) + IoStatus->Information += VacbIosb.Information; } + + if (!DirtyVacb) + { + IO_STATUS_BLOCK MmIosb; + LARGE_INTEGER MmOffset; + + MmOffset.QuadPart = FlushStart; + + if (FlushEnd - (FlushEnd % VACB_MAPPING_GRANULARITY) <= FlushStart) + { + /* The whole range fits within a VACB chunk. */ + Status = MmFlushSegment(SectionObjectPointers, &MmOffset, FlushEnd - FlushStart, &MmIosb); + } + else + { + ULONG MmLength = VACB_MAPPING_GRANULARITY - (FlushStart % VACB_MAPPING_GRANULARITY); + Status = MmFlushSegment(SectionObjectPointers, &MmOffset, MmLength, &MmIosb); + } + + if (!NT_SUCCESS(Status)) + goto quit; + + if (IoStatus) + IoStatus->Information += MmIosb.Information; + + /* Update VDL */ + if (SharedCacheMap->ValidDataLength.QuadPart < FlushEnd) + SharedCacheMap->ValidDataLength.QuadPart = FlushEnd; + } + + if (!NT_SUCCESS(RtlLongLongAdd(FlushStart, VACB_MAPPING_GRANULARITY, &FlushStart))) + { + /* We're at the end of file ! */ + break; + } + + /* Round down to next VACB start now */ + FlushStart -= FlushStart % VACB_MAPPING_GRANULARITY; + } + +quit: + if (IoStatus) + { + IoStatus->Status = Status; } } NTSTATUS -NTAPI CcRosDeleteFileCache ( PFILE_OBJECT FileObject, PROS_SHARED_CACHE_MAP SharedCacheMap, @@ -1165,108 +1024,97 @@ CcRosDeleteFileCache ( */ { PLIST_ENTRY current_entry; - PROS_VACB current; - LIST_ENTRY FreeList; ASSERT(SharedCacheMap); + ASSERT(SharedCacheMap == FileObject->SectionObjectPointer->SharedCacheMap); + ASSERT(SharedCacheMap->OpenCount == 0); - SharedCacheMap->OpenCount++; + /* Remove all VACBs from the global lists */ + KeAcquireSpinLockAtDpcLevel(&SharedCacheMap->CacheMapLock); + current_entry = SharedCacheMap->CacheMapVacbListHead.Flink; + while (current_entry != &SharedCacheMap->CacheMapVacbListHead) + { + PROS_VACB Vacb = CONTAINING_RECORD(current_entry, ROS_VACB, CacheMapVacbListEntry); + + RemoveEntryList(&Vacb->VacbLruListEntry); + InitializeListHead(&Vacb->VacbLruListEntry); + + if (Vacb->Dirty) + { + CcRosUnmarkDirtyVacb(Vacb, FALSE); + /* Mark it as dirty again so we know that we have to flush before freeing it */ + Vacb->Dirty = TRUE; + } + + current_entry = current_entry->Flink; + } + + /* Make sure there is no trace anymore of this map */ + FileObject->SectionObjectPointer->SharedCacheMap = NULL; + RemoveEntryList(&SharedCacheMap->SharedCacheMapLinks); + + KeReleaseSpinLockFromDpcLevel(&SharedCacheMap->CacheMapLock); KeReleaseQueuedSpinLock(LockQueueMasterLock, *OldIrql); - CcFlushCache(FileObject->SectionObjectPointer, NULL, 0, NULL); - - *OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); - SharedCacheMap->OpenCount--; - if (SharedCacheMap->OpenCount == 0) + /* Now that we're out of the locks, free everything for real */ + while (!IsListEmpty(&SharedCacheMap->CacheMapVacbListHead)) { - FileObject->SectionObjectPointer->SharedCacheMap = NULL; + PROS_VACB Vacb = CONTAINING_RECORD(RemoveHeadList(&SharedCacheMap->CacheMapVacbListHead), ROS_VACB, CacheMapVacbListEntry); + ULONG RefCount; - /* - * Release all VACBs - */ - InitializeListHead(&FreeList); - KeAcquireSpinLockAtDpcLevel(&SharedCacheMap->CacheMapLock); - while (!IsListEmpty(&SharedCacheMap->CacheMapVacbListHead)) + InitializeListHead(&Vacb->CacheMapVacbListEntry); + + /* Flush to disk, if needed */ + if (Vacb->Dirty) { - current_entry = RemoveTailList(&SharedCacheMap->CacheMapVacbListHead); - KeReleaseSpinLockFromDpcLevel(&SharedCacheMap->CacheMapLock); + IO_STATUS_BLOCK Iosb; + NTSTATUS Status; - current = CONTAINING_RECORD(current_entry, ROS_VACB, CacheMapVacbListEntry); - RemoveEntryList(¤t->VacbLruListEntry); - InitializeListHead(¤t->VacbLruListEntry); - if (current->Dirty) + Status = MmFlushSegment(FileObject->SectionObjectPointer, &Vacb->FileOffset, VACB_MAPPING_GRANULARITY, &Iosb); + if (!NT_SUCCESS(Status)) { - KeAcquireSpinLockAtDpcLevel(&SharedCacheMap->CacheMapLock); - CcRosUnmarkDirtyVacb(current, FALSE); - KeReleaseSpinLockFromDpcLevel(&SharedCacheMap->CacheMapLock); - DPRINT1("Freeing dirty VACB\n"); + /* Complain. There's not much we can do */ + DPRINT1("Failed to flush VACB to disk while deleting the cache entry. Status: 0x%08x\n", Status); } - if (current->MappedCount != 0) - { - current->MappedCount = 0; - NT_VERIFY(CcRosVacbDecRefCount(current) > 0); - DPRINT1("Freeing mapped VACB\n"); - } - InsertHeadList(&FreeList, ¤t->CacheMapVacbListEntry); - - KeAcquireSpinLockAtDpcLevel(&SharedCacheMap->CacheMapLock); + Vacb->Dirty = FALSE; } -#if DBG - SharedCacheMap->Trace = FALSE; -#endif - KeReleaseSpinLockFromDpcLevel(&SharedCacheMap->CacheMapLock); - KeReleaseQueuedSpinLock(LockQueueMasterLock, *OldIrql); - ObDereferenceObject(SharedCacheMap->FileObject); - - while (!IsListEmpty(&FreeList)) - { - ULONG Refs; - - current_entry = RemoveTailList(&FreeList); - current = CONTAINING_RECORD(current_entry, ROS_VACB, CacheMapVacbListEntry); - InitializeListHead(¤t->CacheMapVacbListEntry); - Refs = CcRosVacbDecRefCount(current); + RefCount = CcRosVacbDecRefCount(Vacb); #if DBG // CORE-14578 - if (Refs != 0) + if (RefCount != 0) + { + DPRINT1("Leaking VACB %p attached to %p (%I64d)\n", Vacb, FileObject, Vacb->FileOffset.QuadPart); + DPRINT1("There are: %d references left\n", RefCount); + DPRINT1("Map: %d\n", Vacb->MappedCount); + DPRINT1("Dirty: %d\n", Vacb->Dirty); + if (FileObject->FileName.Length != 0) { - DPRINT1("Leaking VACB %p attached to %p (%I64d)\n", current, FileObject, current->FileOffset.QuadPart); - DPRINT1("There are: %d references left\n", Refs); - DPRINT1("Map: %d\n", current->MappedCount); - DPRINT1("Dirty: %d\n", current->Dirty); - if (FileObject->FileName.Length != 0) - { - DPRINT1("File was: %wZ\n", &FileObject->FileName); - } - else if (FileObject->FsContext != NULL && - ((PFSRTL_COMMON_FCB_HEADER)(FileObject->FsContext))->NodeTypeCode == 0x0502 && - ((PFSRTL_COMMON_FCB_HEADER)(FileObject->FsContext))->NodeByteSize == 0x1F8 && - ((PUNICODE_STRING)(((PUCHAR)FileObject->FsContext) + 0x100))->Length != 0) - { - DPRINT1("File was: %wZ (FastFAT)\n", (PUNICODE_STRING)(((PUCHAR)FileObject->FsContext) + 0x100)); - } - else - { - DPRINT1("No name for the file\n"); - } + DPRINT1("File was: %wZ\n", &FileObject->FileName); + } + else + { + DPRINT1("No name for the file\n"); } -#else - ASSERT(Refs == 0); -#endif } - - *OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); - RemoveEntryList(&SharedCacheMap->SharedCacheMapLinks); - KeReleaseQueuedSpinLock(LockQueueMasterLock, *OldIrql); - - ExFreeToNPagedLookasideList(&SharedCacheMapLookasideList, SharedCacheMap); - *OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); +#else + (void)RefCount; +#endif } + + /* Release the references we own */ + if(SharedCacheMap->Section) + ObDereferenceObject(SharedCacheMap->Section); + ObDereferenceObject(SharedCacheMap->FileObject); + + ExFreeToNPagedLookasideList(&SharedCacheMapLookasideList, SharedCacheMap); + + /* Acquire the lock again for our caller */ + *OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); + return STATUS_SUCCESS; } VOID -NTAPI CcRosReferenceCache ( PFILE_OBJECT FileObject) { @@ -1281,56 +1129,7 @@ CcRosReferenceCache ( KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); } -VOID -NTAPI -CcRosRemoveIfClosed ( - PSECTION_OBJECT_POINTERS SectionObjectPointer) -{ - PROS_SHARED_CACHE_MAP SharedCacheMap; - KIRQL OldIrql; - - DPRINT("CcRosRemoveIfClosed()\n"); - OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); - SharedCacheMap = SectionObjectPointer->SharedCacheMap; - if (SharedCacheMap && SharedCacheMap->OpenCount == 0) - { - CcRosDeleteFileCache(SharedCacheMap->FileObject, SharedCacheMap, &OldIrql); - } - KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); -} - - -VOID -NTAPI -CcRosDereferenceCache ( - PFILE_OBJECT FileObject) -{ - PROS_SHARED_CACHE_MAP SharedCacheMap; - KIRQL OldIrql; - - OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); - SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; - ASSERT(SharedCacheMap); - if (SharedCacheMap->OpenCount > 0) - { - SharedCacheMap->OpenCount--; - if (SharedCacheMap->OpenCount == 0) - { - KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); - MmFreeSectionSegments(SharedCacheMap->FileObject); - - OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); - CcRosDeleteFileCache(FileObject, SharedCacheMap, &OldIrql); - KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); - - return; - } - } - KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); -} - NTSTATUS -NTAPI CcRosReleaseFileCache ( PFILE_OBJECT FileObject) /* @@ -1373,20 +1172,12 @@ CcRosReleaseFileCache ( PrivateMap->NodeTypeCode = 0; } - if (SharedCacheMap->OpenCount > 0) + ASSERT(SharedCacheMap->OpenCount > 0); + + SharedCacheMap->OpenCount--; + if (SharedCacheMap->OpenCount == 0) { - SharedCacheMap->OpenCount--; - if (SharedCacheMap->OpenCount == 0) - { - KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); - MmFreeSectionSegments(SharedCacheMap->FileObject); - - OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); - CcRosDeleteFileCache(FileObject, SharedCacheMap, &OldIrql); - KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); - - return STATUS_SUCCESS; - } + CcRosDeleteFileCache(FileObject, SharedCacheMap, &OldIrql); } } } @@ -1395,7 +1186,6 @@ CcRosReleaseFileCache ( } NTSTATUS -NTAPI CcRosInitializeFileCache ( PFILE_OBJECT FileObject, PCC_FILE_SIZES FileSizes, @@ -1412,6 +1202,8 @@ CcRosInitializeFileCache ( DPRINT("CcRosInitializeFileCache(FileObject 0x%p)\n", FileObject); + OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); + Allocated = FALSE; SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; if (SharedCacheMap == NULL) @@ -1430,6 +1222,7 @@ CcRosInitializeFileCache ( SharedCacheMap->LazyWriteContext = LazyWriterContext; SharedCacheMap->SectionSize = FileSizes->AllocationSize; SharedCacheMap->FileSize = FileSizes->FileSize; + SharedCacheMap->ValidDataLength = FileSizes->ValidDataLength; SharedCacheMap->PinAccess = PinAccess; SharedCacheMap->DirtyPageThreshold = 0; SharedCacheMap->DirtyPages = 0; @@ -1437,27 +1230,37 @@ CcRosInitializeFileCache ( KeInitializeSpinLock(&SharedCacheMap->CacheMapLock); InitializeListHead(&SharedCacheMap->CacheMapVacbListHead); InitializeListHead(&SharedCacheMap->BcbList); - } - OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); - if (Allocated) + SharedCacheMap->Flags = SHARED_CACHE_MAP_IN_CREATION; + + ObReferenceObjectByPointer(FileObject, + FILE_ALL_ACCESS, + NULL, + KernelMode); + + FileObject->SectionObjectPointer->SharedCacheMap = SharedCacheMap; + + //CcRosTraceCacheMap(SharedCacheMap, TRUE); + } + else if (SharedCacheMap->Flags & SHARED_CACHE_MAP_IN_CREATION) { - if (FileObject->SectionObjectPointer->SharedCacheMap == NULL) - { - ObReferenceObjectByPointer(FileObject, - FILE_ALL_ACCESS, - NULL, - KernelMode); - FileObject->SectionObjectPointer->SharedCacheMap = SharedCacheMap; + /* The shared cache map is being created somewhere else. Wait for that to happen */ + KEVENT Waiter; + PKEVENT PreviousWaiter = SharedCacheMap->CreateEvent; - InsertTailList(&CcCleanSharedCacheMapList, &SharedCacheMap->SharedCacheMapLinks); - } - else - { - ExFreeToNPagedLookasideList(&SharedCacheMapLookasideList, SharedCacheMap); - SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; - } + KeInitializeEvent(&Waiter, NotificationEvent, FALSE); + SharedCacheMap->CreateEvent = &Waiter; + + KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); + + KeWaitForSingleObject(&Waiter, Executive, KernelMode, FALSE, NULL); + + if (PreviousWaiter) + KeSetEvent(PreviousWaiter, IO_NO_INCREMENT, FALSE); + + OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); } + if (FileObject->PrivateCacheMap == NULL) { PPRIVATE_CACHE_MAP PrivateMap; @@ -1503,8 +1306,48 @@ CcRosInitializeFileCache ( FileObject->PrivateCacheMap = PrivateMap; SharedCacheMap->OpenCount++; } + KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); + /* Create the section */ + if (Allocated) + { + NTSTATUS Status; + + ASSERT(SharedCacheMap->Section == NULL); + + Status = MmCreateSection( + &SharedCacheMap->Section, + SECTION_ALL_ACCESS, + NULL, + &SharedCacheMap->SectionSize, + PAGE_READWRITE, + SEC_RESERVE, + NULL, + FileObject); + + ASSERT(NT_SUCCESS(Status)); + + if (!NT_SUCCESS(Status)) + { + CcRosReleaseFileCache(FileObject); + return Status; + } + + OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); + + InsertTailList(&CcCleanSharedCacheMapList, &SharedCacheMap->SharedCacheMapLinks); + SharedCacheMap->Flags &= ~SHARED_CACHE_MAP_IN_CREATION; + + if (SharedCacheMap->CreateEvent) + { + KeSetEvent(SharedCacheMap->CreateEvent, IO_NO_INCREMENT, FALSE); + SharedCacheMap->CreateEvent = NULL; + } + + KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); + } + return STATUS_SUCCESS; } @@ -1564,8 +1407,6 @@ CcInitView ( TAG_VACB, 20); - MmInitializeMemoryConsumer(MC_CACHE, CcRosTrimCache); - CcInitCacheZeroPage(); } @@ -1577,14 +1418,14 @@ ExpKdbgExtFileCache(ULONG Argc, PCHAR Argv[]) UNICODE_STRING NoName = RTL_CONSTANT_STRING(L"No name for File"); KdbpPrint(" Usage Summary (in kb)\n"); - KdbpPrint("Shared\t\tValid\tDirty\tName\n"); + KdbpPrint("Shared\t\tMapped\tDirty\tName\n"); /* No need to lock the spin lock here, we're in DBG */ for (ListEntry = CcCleanSharedCacheMapList.Flink; ListEntry != &CcCleanSharedCacheMapList; ListEntry = ListEntry->Flink) { PLIST_ENTRY Vacbs; - ULONG Valid = 0, Dirty = 0; + ULONG Mapped = 0, Dirty = 0; PROS_SHARED_CACHE_MAP SharedCacheMap; PUNICODE_STRING FileName; PWSTR Extra = L""; @@ -1599,13 +1440,7 @@ ExpKdbgExtFileCache(ULONG Argc, PCHAR Argv[]) Vacbs != &SharedCacheMap->CacheMapVacbListHead; Vacbs = Vacbs->Flink) { - PROS_VACB Vacb; - - Vacb = CONTAINING_RECORD(Vacbs, ROS_VACB, CacheMapVacbListEntry); - if (Vacb->Valid) - { - Valid += VACB_MAPPING_GRANULARITY / 1024; - } + Mapped += VACB_MAPPING_GRANULARITY / 1024; } /* Setup name */ @@ -1629,7 +1464,7 @@ ExpKdbgExtFileCache(ULONG Argc, PCHAR Argv[]) } /* And print */ - KdbpPrint("%p\t%d\t%d\t%wZ%S\n", SharedCacheMap, Valid, Dirty, FileName, Extra); + KdbpPrint("%p\t%d\t%d\t%wZ%S\n", SharedCacheMap, Mapped, Dirty, FileName, Extra); } return TRUE; diff --git a/ntoskrnl/ex/init.c b/ntoskrnl/ex/init.c index 8eb867b09cd..fc98cccd430 100644 --- a/ntoskrnl/ex/init.c +++ b/ntoskrnl/ex/init.c @@ -1970,7 +1970,6 @@ Phase1InitializationDiscard(IN PVOID Context) InbvEnableDisplayString(TRUE); /* Launch initial process */ - DPRINT("Free non-cache pages: %lx\n", MmAvailablePages + MiMemoryConsumers[MC_CACHE].PagesUsed); ProcessInfo = &InitBuffer->ProcessInfo; ExpLoadInitialProcess(InitBuffer, &ProcessParameters, &Environment); @@ -2009,7 +2008,6 @@ Phase1InitializationDiscard(IN PVOID Context) /* Free the boot buffer */ ExFreePoolWithTag(InitBuffer, TAG_INIT); - DPRINT("Free non-cache pages: %lx\n", MmAvailablePages + MiMemoryConsumers[MC_CACHE].PagesUsed); } VOID diff --git a/ntoskrnl/ex/sysinfo.c b/ntoskrnl/ex/sysinfo.c index 64cbffc5ba1..ffb8ddc906d 100644 --- a/ntoskrnl/ex/sysinfo.c +++ b/ntoskrnl/ex/sysinfo.c @@ -278,7 +278,7 @@ ExpGetRawSMBiosTable( DPRINT1("IoWMIOpenBlock failed: 0x%08lx\n", Status); return Status; } - + AllData = ExAllocatePoolWithTag(PagedPool, WMIBufSize, 'itfS'); if (AllData == NULL) { @@ -719,7 +719,6 @@ QSI_DEF(SystemPerformanceInformation) * Not sure this is right. 8^\ */ Spi->CommittedPages = MiMemoryConsumers[MC_SYSTEM].PagesUsed + - MiMemoryConsumers[MC_CACHE].PagesUsed + MiMemoryConsumers[MC_USER].PagesUsed + MiUsedSwapPages; /* @@ -767,7 +766,7 @@ QSI_DEF(SystemPerformanceInformation) Spi->TotalSystemDriverPages = 0; /* FIXME */ Spi->Spare3Count = 0; /* FIXME */ - Spi->ResidentSystemCachePage = MiMemoryConsumers[MC_CACHE].PagesUsed; + Spi->ResidentSystemCachePage = MiMemoryConsumers[MC_USER].PagesUsed; /* FIXME */ Spi->ResidentPagedPoolPage = 0; /* FIXME */ Spi->ResidentSystemDriverPage = 0; /* FIXME */ @@ -1477,13 +1476,10 @@ QSI_DEF(SystemFileCacheInformation) RtlZeroMemory(Sci, sizeof(SYSTEM_FILECACHE_INFORMATION)); /* Return the Byte size not the page size. */ - Sci->CurrentSize = - MiMemoryConsumers[MC_CACHE].PagesUsed * PAGE_SIZE; - Sci->PeakSize = - MiMemoryConsumers[MC_CACHE].PagesUsed * PAGE_SIZE; /* FIXME */ + Sci->CurrentSize = MiMemoryConsumers[MC_USER].PagesUsed; /* FIXME */ + Sci->PeakSize = MiMemoryConsumers[MC_USER].PagesUsed; /* FIXME */ /* Taskmgr multiplies this one by page size right away */ - Sci->CurrentSizeIncludingTransitionInPages = - MiMemoryConsumers[MC_CACHE].PagesUsed; /* FIXME: Should be */ + Sci->CurrentSizeIncludingTransitionInPages = MiMemoryConsumers[MC_USER].PagesUsed; /* FIXME: Should be */ /* system working set and standby pages. */ Sci->PageFaultCount = 0; /* FIXME */ Sci->MinimumWorkingSet = 0; /* FIXME */ diff --git a/ntoskrnl/fsrtl/fastio.c b/ntoskrnl/fsrtl/fastio.c index a056b67e109..ce629f77a02 100644 --- a/ntoskrnl/fsrtl/fastio.c +++ b/ntoskrnl/fsrtl/fastio.c @@ -1714,9 +1714,13 @@ FsRtlAcquireFileForCcFlushEx(IN PFILE_OBJECT FileObject) /* Return either success or inability to wait. In case of other failure - fall through */ - if (Status == STATUS_SUCCESS || - Status == STATUS_CANT_WAIT) + if (NT_SUCCESS(Status)) + return Status; + + if (Status == STATUS_CANT_WAIT) { + DPRINT1("STATUS_CANT_WAIT\n"); + FsRtlExitFileSystem(); return Status; } } diff --git a/ntoskrnl/include/internal/cc.h b/ntoskrnl/include/internal/cc.h index 18c0b9effa2..52a7b19f0a6 100644 --- a/ntoskrnl/include/internal/cc.h +++ b/ntoskrnl/include/internal/cc.h @@ -175,10 +175,13 @@ typedef struct _ROS_SHARED_CACHE_MAP LARGE_INTEGER FileSize; LIST_ENTRY BcbList; LARGE_INTEGER SectionSize; + LARGE_INTEGER ValidDataLength; PFILE_OBJECT FileObject; ULONG DirtyPages; LIST_ENTRY SharedCacheMapLinks; ULONG Flags; + PVOID Section; + PKEVENT CreateEvent; PCACHE_MANAGER_CALLBACKS Callbacks; PVOID LazyWriteContext; LIST_ENTRY PrivateList; @@ -197,15 +200,13 @@ typedef struct _ROS_SHARED_CACHE_MAP #define READAHEAD_DISABLED 0x1 #define WRITEBEHIND_DISABLED 0x2 +#define SHARED_CACHE_MAP_IN_CREATION 0x4 +#define SHARED_CACHE_MAP_IN_LAZYWRITE 0x8 typedef struct _ROS_VACB { /* Base address of the region where the view's data is mapped. */ PVOID BaseAddress; - /* Memory area representing the region where the view's data is mapped. */ - struct _MEMORY_AREA* MemoryArea; - /* Are the contents of the view valid. */ - BOOLEAN Valid; /* Are the contents of the view newer than those on disk. */ BOOLEAN Dirty; /* Page out in progress */ @@ -308,20 +309,24 @@ CcMdlWriteComplete2( ); NTSTATUS -NTAPI -CcRosFlushVacb(PROS_VACB Vacb); +CcRosFlushVacb(PROS_VACB Vacb, PIO_STATUS_BLOCK Iosb); NTSTATUS -NTAPI CcRosGetVacb( PROS_SHARED_CACHE_MAP SharedCacheMap, LONGLONG FileOffset, - PLONGLONG BaseOffset, - PVOID *BaseAddress, - PBOOLEAN UptoDate, PROS_VACB *Vacb ); +BOOLEAN +CcRosEnsureVacbResident( + _In_ PROS_VACB Vacb, + _In_ BOOLEAN Wait, + _In_ BOOLEAN NoRead, + _In_ ULONG Offset, + _In_ ULONG Length +); + VOID NTAPI CcInitView(VOID); @@ -330,28 +335,10 @@ VOID NTAPI CcShutdownLazyWriter(VOID); -NTSTATUS -NTAPI -CcReadVirtualAddress(PROS_VACB Vacb); - -NTSTATUS -NTAPI -CcWriteVirtualAddress(PROS_VACB Vacb); - BOOLEAN -NTAPI CcInitializeCacheManager(VOID); -NTSTATUS -NTAPI -CcRosUnmapVacb( - PROS_SHARED_CACHE_MAP SharedCacheMap, - LONGLONG FileOffset, - BOOLEAN NowDirty -); - PROS_VACB -NTAPI CcRosLookupVacb( PROS_SHARED_CACHE_MAP SharedCacheMap, LONGLONG FileOffset @@ -361,26 +348,16 @@ VOID NTAPI CcInitCacheZeroPage(VOID); -NTSTATUS -NTAPI -CcRosMarkDirtyFile( - PROS_SHARED_CACHE_MAP SharedCacheMap, - LONGLONG FileOffset -); - VOID -NTAPI CcRosMarkDirtyVacb( PROS_VACB Vacb); VOID -NTAPI CcRosUnmarkDirtyVacb( PROS_VACB Vacb, BOOLEAN LockViews); NTSTATUS -NTAPI CcRosFlushDirtyPages( ULONG Target, PULONG Count, @@ -389,39 +366,27 @@ CcRosFlushDirtyPages( ); VOID -NTAPI CcRosDereferenceCache(PFILE_OBJECT FileObject); VOID -NTAPI CcRosReferenceCache(PFILE_OBJECT FileObject); -VOID -NTAPI -CcRosRemoveIfClosed(PSECTION_OBJECT_POINTERS SectionObjectPointer); - NTSTATUS -NTAPI CcRosReleaseVacb( PROS_SHARED_CACHE_MAP SharedCacheMap, PROS_VACB Vacb, - BOOLEAN Valid, BOOLEAN Dirty, BOOLEAN Mapped ); NTSTATUS -NTAPI CcRosRequestVacb( PROS_SHARED_CACHE_MAP SharedCacheMap, LONGLONG FileOffset, - PVOID* BaseAddress, - PBOOLEAN UptoDate, PROS_VACB *Vacb ); NTSTATUS -NTAPI CcRosInitializeFileCache( PFILE_OBJECT FileObject, PCC_FILE_SIZES FileSizes, @@ -431,7 +396,6 @@ CcRosInitializeFileCache( ); NTSTATUS -NTAPI CcRosReleaseFileCache( PFILE_OBJECT FileObject ); diff --git a/ntoskrnl/include/internal/fsrtl.h b/ntoskrnl/include/internal/fsrtl.h index 79cc3b6cfbc..228bb7a1cf8 100644 --- a/ntoskrnl/include/internal/fsrtl.h +++ b/ntoskrnl/include/internal/fsrtl.h @@ -148,3 +148,11 @@ NTSTATUS NTAPI FsRtlAcquireToCreateMappedSection(_In_ PFILE_OBJECT FileObject, _In_ ULONG SectionPageProtection); + +VOID +NTAPI +FsRtlReleaseFileForCcFlush(IN PFILE_OBJECT FileObject); + +NTSTATUS +NTAPI +FsRtlAcquireFileForCcFlushEx(IN PFILE_OBJECT FileObject); diff --git a/ntoskrnl/include/internal/mm.h b/ntoskrnl/include/internal/mm.h index d1754ec4f5c..a661aa469f3 100644 --- a/ntoskrnl/include/internal/mm.h +++ b/ntoskrnl/include/internal/mm.h @@ -69,7 +69,9 @@ typedef ULONG_PTR SWAPENTRY; #endif #define MEMORY_AREA_SECTION_VIEW (1) +#ifdef NEWCC #define MEMORY_AREA_CACHE (2) +#endif #define MEMORY_AREA_OWNED_BY_ARM3 (15) #define MEMORY_AREA_STATIC (0x80000000) @@ -87,13 +89,9 @@ typedef ULONG_PTR SWAPENTRY; #define SEC_PHYSICALMEMORY (0x80000000) -#define MM_PAGEFILE_SEGMENT (0x1) -#define MM_DATAFILE_SEGMENT (0x2) - -#define MC_CACHE (0) -#define MC_USER (1) -#define MC_SYSTEM (2) -#define MC_MAXIMUM (3) +#define MC_USER (0) +#define MC_SYSTEM (1) +#define MC_MAXIMUM (2) #define PAGED_POOL_MASK 1 #define MUST_SUCCEED_POOL_MASK 2 @@ -165,14 +163,15 @@ typedef ULONG_PTR SWAPENTRY; typedef struct _MM_SECTION_SEGMENT { + PFILE_OBJECT FileObject; + FAST_MUTEX Lock; /* lock which protects the page directory */ - PFILE_OBJECT FileObject; LARGE_INTEGER RawLength; /* length of the segment which is part of the mapped file */ LARGE_INTEGER Length; /* absolute length of the segment */ - ULONG ReferenceCount; - ULONG CacheCount; + PLONG64 ReferenceCount; + ULONG SectionCount; ULONG Protection; - ULONG Flags; + PULONG Flags; BOOLEAN WriteCopy; BOOLEAN Locked; @@ -183,32 +182,32 @@ typedef struct _MM_SECTION_SEGMENT ULONG Characteristics; } Image; - LIST_ENTRY ListOfSegments; + LONG64 RefCount; + ULONG SegFlags; + + ULONGLONG LastPage; + RTL_GENERIC_TABLE PageTable; } MM_SECTION_SEGMENT, *PMM_SECTION_SEGMENT; typedef struct _MM_IMAGE_SECTION_OBJECT { + PFILE_OBJECT FileObject; + + LONG64 RefCount; + ULONG SegFlags; + SECTION_IMAGE_INFORMATION ImageInformation; PVOID BasedAddress; ULONG NrSegments; PMM_SECTION_SEGMENT Segments; } MM_IMAGE_SECTION_OBJECT, *PMM_IMAGE_SECTION_OBJECT; -typedef struct _ROS_SECTION_OBJECT -{ - CSHORT Type; - CSHORT Size; - LARGE_INTEGER MaximumSize; - ULONG SectionPageProtection; - ULONG AllocationAttributes; - PFILE_OBJECT FileObject; - union - { - PMM_IMAGE_SECTION_OBJECT ImageSection; - PMM_SECTION_SEGMENT Segment; - }; -} ROS_SECTION_OBJECT, *PROS_SECTION_OBJECT; +#define MM_PHYSICALMEMORY_SEGMENT (0x1) +#define MM_DATAFILE_SEGMENT (0x2) +#define MM_SEGMENT_INDELETE (0x4) +#define MM_SEGMENT_INCREATE (0x8) + #define MA_GetStartingAddress(_MemoryArea) ((_MemoryArea)->VadNode.StartingVpn << PAGE_SHIFT) #define MA_GetEndingAddress(_MemoryArea) (((_MemoryArea)->VadNode.EndingVpn + 1) << PAGE_SHIFT) @@ -218,25 +217,17 @@ typedef struct _MEMORY_AREA MMVAD VadNode; ULONG Type; - ULONG Protect; ULONG Flags; BOOLEAN DeleteInProgress; ULONG Magic; PVOID Vad; - union + + struct { - struct - { - ROS_SECTION_OBJECT* Section; - LARGE_INTEGER ViewOffset; - PMM_SECTION_SEGMENT Segment; - LIST_ENTRY RegionListHead; - } SectionData; - struct - { - LIST_ENTRY RegionListHead; - } VirtualMemoryData; - } Data; + LONGLONG ViewOffset; + PMM_SECTION_SEGMENT Segment; + LIST_ENTRY RegionListHead; + } SectionData; } MEMORY_AREA, *PMEMORY_AREA; typedef struct _MM_RMAP_ENTRY @@ -368,6 +359,8 @@ typedef struct _MMPFN // HACK until WS lists are supported MMWSLE Wsle; + struct _MMPFN* NextLRU; + struct _MMPFN* PreviousLRU; } MMPFN, *PMMPFN; extern PMMPFN MmPfnDatabase; @@ -777,15 +770,6 @@ MmAccessFault( IN PVOID TrapInformation ); -/* kmap.c ********************************************************************/ - -NTSTATUS -NTAPI -MiCopyFromUserPage( - PFN_NUMBER DestPage, - const VOID *SrcAddress -); - /* process.c *****************************************************************/ PVOID @@ -837,6 +821,8 @@ NTAPI MmRebalanceMemoryConsumers(VOID); /* rmap.c **************************************************************/ +#define RMAP_SEGMENT_MASK ~((ULONG_PTR)0xff) +#define RMAP_IS_SEGMENT(x) (((ULONG_PTR)(x) & RMAP_SEGMENT_MASK) == RMAP_SEGMENT_MASK) VOID NTAPI @@ -880,11 +866,6 @@ MmInitializeRmapList(VOID); VOID NTAPI MmSetCleanAllRmaps(PFN_NUMBER Page); - -VOID -NTAPI -MmSetDirtyAllRmaps(PFN_NUMBER Page); - BOOLEAN NTAPI MmIsDirtyPageRmap(PFN_NUMBER Page); @@ -893,6 +874,11 @@ NTSTATUS NTAPI MmPageOutPhysicalAddress(PFN_NUMBER Page); +PMM_SECTION_SEGMENT +NTAPI +MmGetSectionAssociation(PFN_NUMBER Page, + PLARGE_INTEGER Offset); + /* freelist.c **********************************************************/ FORCEINLINE @@ -966,20 +952,12 @@ MiGetPfnEntryIndex(IN PMMPFN Pfn1) PFN_NUMBER NTAPI -MmGetLRUNextUserPage(PFN_NUMBER PreviousPage); +MmGetLRUNextUserPage(PFN_NUMBER PreviousPage, BOOLEAN MoveToLast); PFN_NUMBER NTAPI MmGetLRUFirstUserPage(VOID); -VOID -NTAPI -MmInsertLRULastUserPage(PFN_NUMBER Page); - -VOID -NTAPI -MmRemoveLRUUserPage(PFN_NUMBER Page); - VOID NTAPI MmDumpArmPfnDatabase( @@ -1204,6 +1182,14 @@ MmIsDirtyPage( PVOID Address ); +VOID +NTAPI +MmClearPageAccessedBit(PEPROCESS Process, PVOID Address); + +BOOLEAN +NTAPI +MmIsPageAccessed(PEPROCESS Process, PVOID Address); + /* wset.c ********************************************************************/ NTSTATUS @@ -1248,6 +1234,41 @@ MmFindRegion( /* section.c *****************************************************************/ +#define PFN_FROM_SSE(E) ((PFN_NUMBER)((E) >> PAGE_SHIFT)) +#define IS_SWAP_FROM_SSE(E) ((E) & 0x00000001) +#define MM_IS_WAIT_PTE(E) \ + (IS_SWAP_FROM_SSE(E) && SWAPENTRY_FROM_SSE(E) == MM_WAIT_ENTRY) +#define MAKE_PFN_SSE(P) ((ULONG_PTR)((P) << PAGE_SHIFT)) +#define SWAPENTRY_FROM_SSE(E) ((E) >> 1) +#define MAKE_SWAP_SSE(S) (((ULONG_PTR)(S) << 1) | 0x1) +#define DIRTY_SSE(E) ((E) | 2) +#define CLEAN_SSE(E) ((E) & ~2) +#define IS_DIRTY_SSE(E) ((E) & 2) +#define WRITE_SSE(E) ((E) | 4) +#define IS_WRITE_SSE(E) ((E) & 4) +#define PAGE_FROM_SSE(E) ((E) & 0xFFFFF000) +#define SHARE_COUNT_FROM_SSE(E) (((E) & 0x00000FFC) >> 3) +#define MAX_SHARE_COUNT 0x1FF +#define MAKE_SSE(P, C) ((ULONG_PTR)((P) | ((C) << 3))) +#define BUMPREF_SSE(E) (PAGE_FROM_SSE(E) | ((SHARE_COUNT_FROM_SSE(E) + 1) << 3) | ((E) & 0x7)) +#define DECREF_SSE(E) (PAGE_FROM_SSE(E) | ((SHARE_COUNT_FROM_SSE(E) - 1) << 3) | ((E) & 0x7)) + +VOID +NTAPI +_MmLockSectionSegment(PMM_SECTION_SEGMENT Segment, + const char *file, + int line); + +#define MmLockSectionSegment(x) _MmLockSectionSegment(x,__FILE__,__LINE__) + +VOID +NTAPI +_MmUnlockSectionSegment(PMM_SECTION_SEGMENT Segment, + const char *file, + int line); + +#define MmUnlockSectionSegment(x) _MmUnlockSectionSegment(x,__FILE__,__LINE__) + VOID NTAPI MmGetImageInformation( @@ -1306,15 +1327,6 @@ MmNotPresentFaultSectionView( BOOLEAN Locked ); -NTSTATUS -NTAPI -MmPageOutSectionView( - PMMSUPPORT AddressSpace, - PMEMORY_AREA MemoryArea, - PVOID Address, - ULONG_PTR Entry -); - NTSTATUS NTAPI MmCreatePhysicalMemorySection(VOID); @@ -1331,6 +1343,110 @@ VOID NTAPI MmFreeSectionSegments(PFILE_OBJECT FileObject); +/* Exported from NT 6.2 Onward. We keep it internal. */ +NTSTATUS +NTAPI +MmMapViewInSystemSpaceEx ( + _In_ PVOID Section, + _Outptr_result_bytebuffer_ (*ViewSize) PVOID *MappedBase, + _Inout_ PSIZE_T ViewSize, + _Inout_ PLARGE_INTEGER SectionOffset, + _In_ ULONG_PTR Flags + ); + +BOOLEAN +NTAPI +MmArePagesResident( + _In_ PEPROCESS Process, + _In_ PVOID BaseAddress, + _In_ ULONG Length); + +NTSTATUS +NTAPI +MmMakePagesDirty( + _In_ PEPROCESS Process, + _In_ PVOID Address, + _In_ ULONG Length); + +NTSTATUS +NTAPI +MmRosFlushVirtualMemory( + _In_ PEPROCESS Process, + _Inout_ PVOID* Address, + _Inout_ PSIZE_T Length, + _Out_ PIO_STATUS_BLOCK Iosb); + +NTSTATUS +NTAPI +MmFlushSegment( + _In_ PSECTION_OBJECT_POINTERS SectionObjectPointer, + _In_opt_ PLARGE_INTEGER Offset, + _In_ ULONG Length, + _In_opt_ PIO_STATUS_BLOCK Iosb); + +NTSTATUS +NTAPI +MmMakeDataSectionResident( + _In_ PSECTION_OBJECT_POINTERS SectionObjectPointer, + _In_ LONGLONG Offset, + _In_ ULONG Length, + _In_ PLARGE_INTEGER ValidDataLength); + +BOOLEAN +NTAPI +MmPurgeSegment( + _In_ PSECTION_OBJECT_POINTERS SectionObjectPointer, + _In_opt_ PLARGE_INTEGER Offset, + _In_ ULONG Length); + +BOOLEAN +NTAPI +MmCheckDirtySegment( + PMM_SECTION_SEGMENT Segment, + PLARGE_INTEGER Offset, + BOOLEAN ForceDirty, + BOOLEAN PageOut); + +BOOLEAN +NTAPI +MmUnsharePageEntrySectionSegment(PMEMORY_AREA MemoryArea, + PMM_SECTION_SEGMENT Segment, + PLARGE_INTEGER Offset, + BOOLEAN Dirty, + BOOLEAN PageOut, + ULONG_PTR *InEntry); + +VOID +NTAPI +MmDereferenceSegment(PMM_SECTION_SEGMENT Segment); + +NTSTATUS +NTAPI +MmExtendSection( + _In_ PVOID Section, + _Inout_ PLARGE_INTEGER NewSize); + +/* sptab.c *******************************************************************/ + +NTSTATUS +NTAPI +_MmSetPageEntrySectionSegment(PMM_SECTION_SEGMENT Segment, + PLARGE_INTEGER Offset, + ULONG_PTR Entry, + const char *file, + int line); + +ULONG_PTR +NTAPI +_MmGetPageEntrySectionSegment(PMM_SECTION_SEGMENT Segment, + PLARGE_INTEGER Offset, + const char *file, + int line); + +#define MmSetPageEntrySectionSegment(S,O,E) _MmSetPageEntrySectionSegment(S,O,E,__FILE__,__LINE__) + +#define MmGetPageEntrySectionSegment(S,O) _MmGetPageEntrySectionSegment(S,O,__FILE__,__LINE__) + /* sysldr.c ******************************************************************/ VOID diff --git a/ntoskrnl/include/ntoskrnl.h b/ntoskrnl/include/ntoskrnl.h index 7685c36bcf1..353efeb44c3 100644 --- a/ntoskrnl/include/ntoskrnl.h +++ b/ntoskrnl/include/ntoskrnl.h @@ -63,7 +63,9 @@ #include #include #include +#define ENABLE_INTSAFE_SIGNED_FUNCTIONS #include +#undef ENABLE_INTSAFE_SIGNED_FUNCTIONS /* C Headers */ #include diff --git a/ntoskrnl/io/iomgr/iofunc.c b/ntoskrnl/io/iomgr/iofunc.c index 80814a70ba3..7d0a08c5395 100644 --- a/ntoskrnl/io/iomgr/iofunc.c +++ b/ntoskrnl/io/iomgr/iofunc.c @@ -3014,10 +3014,8 @@ NtReadFile(IN HANDLE FileHandle, /* Now set the deferred read flags */ Irp->Flags |= (IRP_READ_OPERATION | IRP_DEFER_IO_COMPLETION); -#if 0 - /* FIXME: VFAT SUCKS */ + if (FileObject->Flags & FO_NO_INTERMEDIATE_BUFFERING) Irp->Flags |= IRP_NOCACHE; -#endif /* Perform the call */ return IopPerformSynchronousRequest(DeviceObject, @@ -4082,10 +4080,8 @@ NtWriteFile(IN HANDLE FileHandle, /* Now set the deferred read flags */ Irp->Flags |= (IRP_WRITE_OPERATION | IRP_DEFER_IO_COMPLETION); -#if 0 - /* FIXME: VFAT SUCKS */ + if (FileObject->Flags & FO_NO_INTERMEDIATE_BUFFERING) Irp->Flags |= IRP_NOCACHE; -#endif /* Perform the call */ return IopPerformSynchronousRequest(DeviceObject, diff --git a/ntoskrnl/mm/ARM3/miarm.h b/ntoskrnl/mm/ARM3/miarm.h index b374e94ad75..025cd6f6561 100644 --- a/ntoskrnl/mm/ARM3/miarm.h +++ b/ntoskrnl/mm/ARM3/miarm.h @@ -1071,9 +1071,8 @@ FORCEINLINE BOOLEAN MiIsRosSectionObject(IN PVOID Section) { - PROS_SECTION_OBJECT RosSection = Section; - if ((RosSection->Type == 'SC') && (RosSection->Size == 'TN')) return TRUE; - return FALSE; + PSECTION RosSection = Section; + return RosSection->u.Flags.filler; } #define MI_IS_ROS_PFN(x) ((x)->u4.AweAllocation == TRUE) diff --git a/ntoskrnl/mm/ARM3/procsup.c b/ntoskrnl/mm/ARM3/procsup.c index a183f8a099f..eb8e68180d4 100644 --- a/ntoskrnl/mm/ARM3/procsup.c +++ b/ntoskrnl/mm/ARM3/procsup.c @@ -941,7 +941,6 @@ MmInitializeProcessAddressSpace(IN PEPROCESS Process, NTSTATUS Status = STATUS_SUCCESS; SIZE_T ViewSize = 0; PVOID ImageBase = 0; - PROS_SECTION_OBJECT SectionObject = Section; PMMPTE PointerPte; KIRQL OldIrql; PMMPDE PointerPde; @@ -1048,10 +1047,11 @@ MmInitializeProcessAddressSpace(IN PEPROCESS Process, #endif /* Check if there's a Section Object */ - if (SectionObject) + if (Section) { /* Determine the image file name and save it to EPROCESS */ - FileName = SectionObject->FileObject->FileName; + PFILE_OBJECT FileObject = MmGetFileObjectForSection(Section); + FileName = FileObject->FileName; Source = (PWCHAR)((PCHAR)FileName.Buffer + FileName.Length); if (FileName.Buffer) { @@ -1083,9 +1083,7 @@ MmInitializeProcessAddressSpace(IN PEPROCESS Process, if (AuditName) { /* Setup the audit name */ - Status = SeInitializeProcessAuditName(SectionObject->FileObject, - FALSE, - AuditName); + Status = SeInitializeProcessAuditName(FileObject, FALSE, AuditName); if (!NT_SUCCESS(Status)) { /* Fail */ diff --git a/ntoskrnl/mm/ARM3/section.c b/ntoskrnl/mm/ARM3/section.c index 02ead9a70a5..3e6a25b44db 100644 --- a/ntoskrnl/mm/ARM3/section.c +++ b/ntoskrnl/mm/ARM3/section.c @@ -412,16 +412,20 @@ MiInsertInSystemSpace(IN PMMSESSION Session, return Base; } +static NTSTATUS -NTAPI MiAddMappedPtes(IN PMMPTE FirstPte, IN PFN_NUMBER PteCount, - IN PCONTROL_AREA ControlArea) + IN PCONTROL_AREA ControlArea, + IN LONGLONG SectionOffset) { MMPTE TempPte; PMMPTE PointerPte, ProtoPte, LastProtoPte, LastPte; PSUBSECTION Subsection; + /* Mapping at offset not supported yet */ + ASSERT(SectionOffset == 0); + /* ARM3 doesn't support this yet */ ASSERT(ControlArea->u.Flags.GlobalOnlyPerSession == 0); ASSERT(ControlArea->u.Flags.Rom == 0); @@ -829,12 +833,17 @@ MiUnmapViewOfSection(IN PEPROCESS Process, PEPROCESS CurrentProcess = PsGetCurrentProcess(); PAGED_CODE(); + /* Check if we need to lock the address space */ + if (!Flags) MmLockAddressSpace(&Process->Vm); + /* Check for Mm Region */ MemoryArea = MmLocateMemoryAreaByAddress(&Process->Vm, BaseAddress); if ((MemoryArea) && (MemoryArea->Type != MEMORY_AREA_OWNED_BY_ARM3)) { /* Call Mm API */ - return MiRosUnmapViewOfSection(Process, BaseAddress, Process->ProcessExiting); + NTSTATUS Status = MiRosUnmapViewOfSection(Process, BaseAddress, Process->ProcessExiting); + if (!Flags) MmUnlockAddressSpace(&Process->Vm); + return Status; } /* Check if we should attach to the process */ @@ -845,10 +854,7 @@ MiUnmapViewOfSection(IN PEPROCESS Process, Attached = TRUE; } - /* Check if we need to lock the address space */ - if (!Flags) MmLockAddressSpace(&Process->Vm); - - /* Check if the process is already daed */ + /* Check if the process is already dead */ if (Process->VmDeleted) { /* Fail the call */ @@ -1048,15 +1054,17 @@ _WARN("MiSessionCommitPageTables halfplemented for amd64") } NTSTATUS -NTAPI -MiMapViewInSystemSpace(IN PVOID Section, - IN PMMSESSION Session, - OUT PVOID *MappedBase, - IN OUT PSIZE_T ViewSize) +MiMapViewInSystemSpace( + _In_ PVOID Section, + _In_ PMMSESSION Session, + _Outptr_result_bytebuffer_ (*ViewSize) PVOID *MappedBase, + _Inout_ PSIZE_T ViewSize, + _Inout_ PLARGE_INTEGER SectionOffset) { PVOID Base; PCONTROL_AREA ControlArea; - ULONG Buckets, SectionSize; + ULONG Buckets; + LONGLONG SectionSize; NTSTATUS Status; PAGED_CODE(); @@ -1073,13 +1081,31 @@ MiMapViewInSystemSpace(IN PVOID Section, ASSERT(NT_SUCCESS(Status)); /* Get the section size at creation time */ - SectionSize = ((PSECTION)Section)->SizeOfSection.LowPart; + SectionSize = ((PSECTION)Section)->SizeOfSection.QuadPart; - /* If the caller didn't specify a view size, assume the whole section */ - if (!(*ViewSize)) *ViewSize = SectionSize; + /* If the caller didn't specify a view size, assume until the end of the section */ + if (!(*ViewSize)) + { + /* Check for overflow first */ + if ((SectionSize - SectionOffset->QuadPart) > SIZE_T_MAX) + { + DPRINT1("Section end is too far away from the specified offset.\n"); + MiDereferenceControlArea(ControlArea); + return STATUS_INVALID_VIEW_SIZE; + } + *ViewSize = SectionSize - SectionOffset->QuadPart; + } + + /* Check overflow */ + if ((SectionOffset->QuadPart + *ViewSize) < SectionOffset->QuadPart) + { + DPRINT1("Integer overflow between size & offset!\n"); + MiDereferenceControlArea(ControlArea); + return STATUS_INVALID_VIEW_SIZE; + } /* Check if the caller wanted a larger section than the view */ - if (*ViewSize > SectionSize) + if (SectionOffset->QuadPart + *ViewSize > SectionSize) { /* Fail */ DPRINT1("View is too large\n"); @@ -1129,7 +1155,8 @@ MiMapViewInSystemSpace(IN PVOID Section, /* Create the actual prototype PTEs for this mapping */ Status = MiAddMappedPtes(MiAddressToPte(Base), BYTES_TO_PAGES(*ViewSize), - ControlArea); + ControlArea, + SectionOffset->QuadPart); ASSERT(NT_SUCCESS(Status)); /* Return the base adress of the mapping and success */ @@ -1216,6 +1243,8 @@ MiLoadUserSymbols(IN PCONTROL_AREA ControlArea, _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER) { ExFreePoolWithTag(LdrEntry, 'bDmM'); + ExReleaseResourceLite(&PsLoadedModuleResource); + KeLeaveCriticalRegion(); _SEH2_YIELD(return); } _SEH2_END; @@ -1655,20 +1684,23 @@ MiGetFileObjectForSectionAddress( if (Vad->u.VadFlags.Spare != 0) { PMEMORY_AREA MemoryArea = (PMEMORY_AREA)Vad; - PROS_SECTION_OBJECT Section; /* Check if it's a section view (RosMm section) */ if (MemoryArea->Type == MEMORY_AREA_SECTION_VIEW) { /* Get the section pointer to the SECTION_OBJECT */ - Section = MemoryArea->Data.SectionData.Section; - *FileObject = Section->FileObject; + *FileObject = MemoryArea->SectionData.Segment->FileObject; } else { +#ifdef NEWCC ASSERT(MemoryArea->Type == MEMORY_AREA_CACHE); DPRINT1("Address is a cache section!\n"); return STATUS_SECTION_NOT_IMAGE; +#else + ASSERT(FALSE); + return STATUS_SECTION_NOT_IMAGE; +#endif } } else @@ -1700,7 +1732,7 @@ PFILE_OBJECT NTAPI MmGetFileObjectForSection(IN PVOID SectionObject) { - PSECTION_OBJECT Section; + PSECTION Section = SectionObject; ASSERT(KeGetCurrentIrql() == PASSIVE_LEVEL); ASSERT(SectionObject != NULL); @@ -1708,12 +1740,11 @@ MmGetFileObjectForSection(IN PVOID SectionObject) if (MiIsRosSectionObject(SectionObject) == FALSE) { /* Return the file pointer stored in the control area */ - Section = SectionObject; return Section->Segment->ControlArea->FilePointer; } /* Return the file object */ - return ((PROS_SECTION_OBJECT)SectionObject)->FileObject; + return ((PMM_SECTION_SEGMENT)Section->Segment)->FileObject; } static @@ -1728,19 +1759,21 @@ MiGetFileObjectForVad( if (Vad->u.VadFlags.Spare != 0) { PMEMORY_AREA MemoryArea = (PMEMORY_AREA)Vad; - PROS_SECTION_OBJECT Section; /* Check if it's a section view (RosMm section) */ if (MemoryArea->Type == MEMORY_AREA_SECTION_VIEW) { /* Get the section pointer to the SECTION_OBJECT */ - Section = MemoryArea->Data.SectionData.Section; - FileObject = Section->FileObject; + FileObject = MemoryArea->SectionData.Segment->FileObject; } else { +#ifdef NEWCC ASSERT(MemoryArea->Type == MEMORY_AREA_CACHE); DPRINT1("VAD is a cache section!\n"); +#else + ASSERT(FALSE); +#endif return NULL; } } @@ -1773,15 +1806,21 @@ VOID NTAPI MmGetImageInformation (OUT PSECTION_IMAGE_INFORMATION ImageInformation) { - PSECTION_OBJECT SectionObject; + PSECTION SectionObject; /* Get the section object of this process*/ SectionObject = PsGetCurrentProcess()->SectionObject; ASSERT(SectionObject != NULL); ASSERT(MiIsRosSectionObject(SectionObject) == TRUE); + if (SectionObject->u.Flags.Image == 0) + { + RtlZeroMemory(ImageInformation, sizeof(*ImageInformation)); + return; + } + /* Return the image information */ - *ImageInformation = ((PROS_SECTION_OBJECT)SectionObject)->ImageSection->ImageInformation; + *ImageInformation = ((PMM_IMAGE_SECTION_OBJECT)SectionObject->Segment)->ImageInformation; } NTSTATUS @@ -1822,19 +1861,10 @@ MmGetFileNameForSection(IN PVOID Section, OUT POBJECT_NAME_INFORMATION *ModuleName) { PFILE_OBJECT FileObject; + PSECTION SectionObject = Section; /* Make sure it's an image section */ - if (MiIsRosSectionObject(Section) == FALSE) - { - /* Check ARM3 Section flag */ - if (((PSECTION)Section)->u.Flags.Image == 0) - { - /* It's not, fail */ - DPRINT1("Not an image section\n"); - return STATUS_SECTION_NOT_IMAGE; - } - } - else if (!(((PROS_SECTION_OBJECT)Section)->AllocationAttributes & SEC_IMAGE)) + if (SectionObject->u.Flags.Image == 0) { /* It's not, fail */ DPRINT1("Not an image section\n"); @@ -2916,7 +2946,7 @@ MmMapViewOfArm3Section(IN PVOID SectionObject, if (!(*ViewSize)) { /* Compute it for the caller */ - CalculatedViewSize = Section->SizeOfSection.QuadPart - + CalculatedViewSize = Section->SizeOfSection.QuadPart - SectionOffset->QuadPart; /* Check if it's larger than 4GB or overflows into kernel-mode */ @@ -3015,6 +3045,7 @@ MmMapViewInSessionSpace(IN PVOID Section, IN OUT PSIZE_T ViewSize) { PAGED_CODE(); + LARGE_INTEGER SectionOffset; // HACK if (MiIsRosSectionObject(Section)) @@ -3031,10 +3062,12 @@ MmMapViewInSessionSpace(IN PVOID Section, /* Use the system space API, but with the session view instead */ ASSERT(MmIsAddressValid(MmSessionSpace) == TRUE); + SectionOffset.QuadPart = 0; return MiMapViewInSystemSpace(Section, &MmSessionSpace->Session, MappedBase, - ViewSize); + ViewSize, + &SectionOffset); } /* @@ -3087,11 +3120,15 @@ MmUnmapViewInSystemSpace(IN PVOID MappedBase) PAGED_CODE(); /* Was this mapped by RosMm? */ + MmLockAddressSpace(MmGetKernelAddressSpace()); MemoryArea = MmLocateMemoryAreaByAddress(MmGetKernelAddressSpace(), MappedBase); if ((MemoryArea) && (MemoryArea->Type != MEMORY_AREA_OWNED_BY_ARM3)) { - return MiRosUnmapViewInSystemSpace(MappedBase); + NTSTATUS Status = MiRosUnmapViewInSystemSpace(MappedBase); + MmUnlockAddressSpace(MmGetKernelAddressSpace()); + return Status; } + MmUnlockAddressSpace(MmGetKernelAddressSpace()); /* It was not, call the ARM3 routine */ return MiUnmapViewInSystemSpace(&MmSession, MappedBase); @@ -3585,7 +3622,7 @@ NtMapViewOfSection(IN HANDLE SectionHandle, PVOID SafeBaseAddress; LARGE_INTEGER SafeSectionOffset; SIZE_T SafeViewSize; - PROS_SECTION_OBJECT Section; + PSECTION Section; PEPROCESS Process; NTSTATUS Status; ACCESS_MASK DesiredAccess; @@ -3715,8 +3752,7 @@ NtMapViewOfSection(IN HANDLE SectionHandle, return Status; } - if (MiIsRosSectionObject(Section) && - (Section->AllocationAttributes & SEC_PHYSICALMEMORY)) + if (Section->u.Flags.PhysicalMemory) { if (PreviousMode == UserMode && SafeSectionOffset.QuadPart + SafeViewSize > MmHighestPhysicalPage << PAGE_SHIFT) @@ -3764,8 +3800,7 @@ NtMapViewOfSection(IN HANDLE SectionHandle, if (NT_SUCCESS(Status)) { /* Check if this is an image for the current process */ - if (MiIsRosSectionObject(Section) && - (Section->AllocationAttributes & SEC_IMAGE) && + if ((Section->u.Flags.Image) && (Process == PsGetCurrentProcess()) && (Status != STATUS_IMAGE_NOT_AT_BASE)) { @@ -3836,7 +3871,7 @@ NtExtendSection(IN HANDLE SectionHandle, IN OUT PLARGE_INTEGER NewMaximumSize) { LARGE_INTEGER SafeNewMaximumSize; - PROS_SECTION_OBJECT Section; + PSECTION Section; NTSTATUS Status; KPROCESSOR_MODE PreviousMode = ExGetPreviousMode(); @@ -3872,30 +3907,24 @@ NtExtendSection(IN HANDLE SectionHandle, NULL); if (!NT_SUCCESS(Status)) return Status; - /* Really this should go in MmExtendSection */ - if (!(Section->AllocationAttributes & SEC_FILE)) - { - DPRINT1("Not extending a file\n"); - ObDereferenceObject(Section); - return STATUS_SECTION_NOT_EXTENDED; - } - - /* FIXME: Do the work */ + Status = MmExtendSection(Section, &SafeNewMaximumSize); /* Dereference the section */ ObDereferenceObject(Section); - /* Enter SEH */ - _SEH2_TRY + if (NT_SUCCESS(Status)) { - /* Write back the new size */ - *NewMaximumSize = SafeNewMaximumSize; + _SEH2_TRY + { + /* Write back the new size */ + *NewMaximumSize = SafeNewMaximumSize; + } + _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER) + { + Status = _SEH2_GetExceptionCode(); + } + _SEH2_END; } - _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER) - { - /* Nothing to do */ - } - _SEH2_END; /* Return the status */ return STATUS_NOT_IMPLEMENTED; diff --git a/ntoskrnl/mm/ARM3/sysldr.c b/ntoskrnl/mm/ARM3/sysldr.c index 811defcde8b..0ebcae75e29 100644 --- a/ntoskrnl/mm/ARM3/sysldr.c +++ b/ntoskrnl/mm/ARM3/sysldr.c @@ -81,13 +81,13 @@ MiCacheImageSymbols(IN PVOID BaseAddress) NTSTATUS NTAPI -MiLoadImageSection(IN OUT PVOID *SectionPtr, - OUT PVOID *ImageBase, - IN PUNICODE_STRING FileName, - IN BOOLEAN SessionLoad, - IN PLDR_DATA_TABLE_ENTRY LdrEntry) +MiLoadImageSection(_Inout_ PSECTION *SectionPtr, + _Out_ PVOID *ImageBase, + _In_ PUNICODE_STRING FileName, + _In_ BOOLEAN SessionLoad, + _In_ PLDR_DATA_TABLE_ENTRY LdrEntry) { - PROS_SECTION_OBJECT Section = *SectionPtr; + PSECTION Section = *SectionPtr; NTSTATUS Status; PEPROCESS Process; PVOID Base = NULL; @@ -158,7 +158,7 @@ MiLoadImageSection(IN OUT PVOID *SectionPtr, } /* Reserve system PTEs needed */ - PteCount = ROUND_TO_PAGES(Section->ImageSection->ImageInformation.ImageFileSize) >> PAGE_SHIFT; + PteCount = ROUND_TO_PAGES(((PMM_IMAGE_SECTION_OBJECT)Section->Segment)->ImageInformation.ImageFileSize) >> PAGE_SHIFT; PointerPte = MiReserveSystemPtes(PteCount, SystemPteSpace); if (!PointerPte) { @@ -2837,7 +2837,7 @@ MmLoadSystemImage(IN PUNICODE_STRING FileName, PWCHAR MissingDriverName; HANDLE SectionHandle; ACCESS_MASK DesiredAccess; - PVOID Section = NULL; + PSECTION Section = NULL; BOOLEAN LockOwned = FALSE; PLIST_ENTRY NextEntry; IMAGE_INFO ImageInfo; @@ -3054,7 +3054,7 @@ LoaderScan: SECTION_MAP_EXECUTE, MmSectionObjectType, KernelMode, - &Section, + (PVOID*)&Section, NULL); ZwClose(SectionHandle); if (!NT_SUCCESS(Status)) goto Quickie; @@ -3085,7 +3085,7 @@ LoaderScan: ASSERT(Status != STATUS_ALREADY_COMMITTED); /* Get the size of the driver */ - DriverSize = ((PROS_SECTION_OBJECT)Section)->ImageSection->ImageInformation.ImageFileSize; + DriverSize = ((PMM_IMAGE_SECTION_OBJECT)Section->Segment)->ImageInformation.ImageFileSize; /* Make sure we're not being loaded into session space */ if (!Flags) diff --git a/ntoskrnl/mm/ARM3/virtual.c b/ntoskrnl/mm/ARM3/virtual.c index 19eb8a69b01..00271e03e7b 100644 --- a/ntoskrnl/mm/ARM3/virtual.c +++ b/ntoskrnl/mm/ARM3/virtual.c @@ -1356,12 +1356,8 @@ MmFlushVirtualMemory(IN PEPROCESS Process, OUT PIO_STATUS_BLOCK IoStatusBlock) { PAGED_CODE(); - UNIMPLEMENTED; - - // - // Fake success - // - return STATUS_SUCCESS; + /* For now we call the old Mm */ + return MmRosFlushVirtualMemory(Process, BaseAddress, RegionSize, IoStatusBlock); } ULONG diff --git a/ntoskrnl/mm/ARM3/zeropage.c b/ntoskrnl/mm/ARM3/zeropage.c index 5fe22294199..80725e8a5a7 100644 --- a/ntoskrnl/mm/ARM3/zeropage.c +++ b/ntoskrnl/mm/ARM3/zeropage.c @@ -46,7 +46,7 @@ MmZeroPageThread(VOID) /* Get the discardable sections to free them */ MiFindInitializationCode(&StartAddress, &EndAddress); if (StartAddress) MiFreeInitializationCode(StartAddress, EndAddress); - DPRINT("Free non-cache pages: %lx\n", MmAvailablePages + MiMemoryConsumers[MC_CACHE].PagesUsed); + DPRINT("Free pages: %lx\n", MmAvailablePages); /* Set our priority to 0 */ Thread->BasePriority = 0; diff --git a/ntoskrnl/mm/balance.c b/ntoskrnl/mm/balance.c index 5cd5e267a23..1cab99be0ec 100644 --- a/ntoskrnl/mm/balance.c +++ b/ntoskrnl/mm/balance.c @@ -28,7 +28,6 @@ MM_ALLOCATION_REQUEST, *PMM_ALLOCATION_REQUEST; MM_MEMORY_CONSUMER MiMemoryConsumers[MC_MAXIMUM]; static ULONG MiMinimumAvailablePages; -static ULONG MiNrTotalPages; static LIST_ENTRY AllocationListHead; static KSPIN_LOCK AllocationListLock; static ULONG MiMinimumPagesPerRun; @@ -38,6 +37,8 @@ static HANDLE MiBalancerThreadHandle = NULL; static KEVENT MiBalancerEvent; static KTIMER MiBalancerTimer; +static LONG PageOutThreadActive; + /* FUNCTIONS ****************************************************************/ CODE_SEG("INIT") @@ -49,24 +50,10 @@ MmInitializeBalancer(ULONG NrAvailablePages, ULONG NrSystemPages) InitializeListHead(&AllocationListHead); KeInitializeSpinLock(&AllocationListLock); - MiNrTotalPages = NrAvailablePages; - /* Set up targets. */ MiMinimumAvailablePages = 256; MiMinimumPagesPerRun = 256; - if ((NrAvailablePages + NrSystemPages) >= 8192) - { - MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 4 * 3; - } - else if ((NrAvailablePages + NrSystemPages) >= 4096) - { - MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 3 * 2; - } - else - { - MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 8; - } - MiMemoryConsumers[MC_USER].PagesTarget = NrAvailablePages - MiMinimumAvailablePages; + MiMemoryConsumers[MC_USER].PagesTarget = NrAvailablePages / 2; } CODE_SEG("INIT") @@ -95,11 +82,7 @@ MmReleasePageMemoryConsumer(ULONG Consumer, PFN_NUMBER Page) KeBugCheck(MEMORY_MANAGEMENT); } - if (MmGetReferenceCountPage(Page) == 1) - { - if(Consumer == MC_USER) MmRemoveLRUUserPage(Page); - (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed); - } + (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed); MmDereferencePage(Page); @@ -121,28 +104,21 @@ MiTrimMemoryConsumer(ULONG Consumer, ULONG InitialTarget) return InitialTarget; } - if (MiMemoryConsumers[Consumer].PagesUsed > MiMemoryConsumers[Consumer].PagesTarget) - { - /* Consumer page limit exceeded */ - Target = max(Target, MiMemoryConsumers[Consumer].PagesUsed - MiMemoryConsumers[Consumer].PagesTarget); - } if (MmAvailablePages < MiMinimumAvailablePages) { /* Global page limit exceeded */ Target = (ULONG)max(Target, MiMinimumAvailablePages - MmAvailablePages); } + else if (MiMemoryConsumers[Consumer].PagesUsed > MiMemoryConsumers[Consumer].PagesTarget) + { + /* Consumer page limit exceeded */ + Target = max(Target, MiMemoryConsumers[Consumer].PagesUsed - MiMemoryConsumers[Consumer].PagesTarget); + } if (Target) { - if (!InitialTarget) - { - /* If there was no initial target, - * swap at least MiMinimumPagesPerRun */ - Target = max(Target, MiMinimumPagesPerRun); - } - /* Now swap the pages out */ - Status = MiMemoryConsumers[Consumer].Trim(Target, 0, &NrFreedPages); + Status = MiMemoryConsumers[Consumer].Trim(Target, MmAvailablePages < MiMinimumAvailablePages, &NrFreedPages); DPRINT("Trimming consumer %lu: Freed %lu pages with a target of %lu pages\n", Consumer, NrFreedPages, Target); @@ -150,50 +126,128 @@ MiTrimMemoryConsumer(ULONG Consumer, ULONG InitialTarget) { KeBugCheck(MEMORY_MANAGEMENT); } - - /* Update the target */ - if (NrFreedPages < Target) - Target -= NrFreedPages; - else - Target = 0; - - /* Return the remaining pages needed to meet the target */ - return Target; - } - else - { - /* Initial target is zero and we don't have anything else to add */ - return 0; } + + /* Return the page count needed to be freed to meet the initial target */ + return (InitialTarget > NrFreedPages) ? (InitialTarget - NrFreedPages) : 0; } NTSTATUS MmTrimUserMemory(ULONG Target, ULONG Priority, PULONG NrFreedPages) { PFN_NUMBER CurrentPage; - PFN_NUMBER NextPage; NTSTATUS Status; (*NrFreedPages) = 0; + DPRINT1("MM BALANCER: %s\n", Priority ? "Paging out!" : "Removing access bit!"); + CurrentPage = MmGetLRUFirstUserPage(); while (CurrentPage != 0 && Target > 0) { - Status = MmPageOutPhysicalAddress(CurrentPage); - if (NT_SUCCESS(Status)) + if (Priority) { - DPRINT("Succeeded\n"); + Status = MmPageOutPhysicalAddress(CurrentPage); + if (NT_SUCCESS(Status)) + { + DPRINT("Succeeded\n"); + Target--; + (*NrFreedPages)++; + } + } + else + { + /* When not paging-out agressively, just reset the accessed bit */ + PEPROCESS Process = NULL; + PVOID Address = NULL; + BOOLEAN Accessed = FALSE; + + /* + * We have a lock-ordering problem here. We cant lock the PFN DB before the Process address space. + * So we must use circonvoluted loops. + * Well... + */ + while (TRUE) + { + KAPC_STATE ApcState; + KIRQL OldIrql = MiAcquirePfnLock(); + PMM_RMAP_ENTRY Entry = MmGetRmapListHeadPage(CurrentPage); + while (Entry) + { + if (RMAP_IS_SEGMENT(Entry->Address)) + { + Entry = Entry->Next; + continue; + } + + /* Check that we didn't treat this entry before */ + if (Entry->Address < Address) + { + Entry = Entry->Next; + continue; + } + + if ((Entry->Address == Address) && (Entry->Process <= Process)) + { + Entry = Entry->Next; + continue; + } + + break; + } + + if (!Entry) + { + MiReleasePfnLock(OldIrql); + break; + } + + Process = Entry->Process; + Address = Entry->Address; + + MiReleasePfnLock(OldIrql); + + KeStackAttachProcess(&Process->Pcb, &ApcState); + + MmLockAddressSpace(&Process->Vm); + + /* Be sure this is still valid. */ + PMMPTE Pte = MiAddressToPte(Address); + if (Pte->u.Hard.Valid) + { + Accessed = Accessed || Pte->u.Hard.Accessed; + Pte->u.Hard.Accessed = 0; + + /* There is no need to invalidate, the balancer thread is never on a user process */ + //KeInvalidateTlbEntry(Address); + } + + MmUnlockAddressSpace(&Process->Vm); + + KeUnstackDetachProcess(&ApcState); + } + + if (!Accessed) + { + /* Nobody accessed this page since the last time we check. Time to clean up */ + + Status = MmPageOutPhysicalAddress(CurrentPage); + // DPRINT1("Paged-out one page: %s\n", NT_SUCCESS(Status) ? "Yes" : "No"); + (void)Status; + } + + /* Done for this page. */ Target--; - (*NrFreedPages)++; } - NextPage = MmGetLRUNextUserPage(CurrentPage); - if (NextPage <= CurrentPage) - { - /* We wrapped around, so we're done */ - break; - } - CurrentPage = NextPage; + CurrentPage = MmGetLRUNextUserPage(CurrentPage, TRUE); + } + + if (CurrentPage) + { + KIRQL OldIrql = MiAcquirePfnLock(); + MmDereferencePage(CurrentPage); + MiReleasePfnLock(OldIrql); } return STATUS_SUCCESS; @@ -210,8 +264,7 @@ VOID NTAPI MmRebalanceMemoryConsumers(VOID) { - if (MiBalancerThreadHandle != NULL && - !MiIsBalancerThread()) + if (InterlockedCompareExchange(&PageOutThreadActive, 0, 1) == 0) { KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE); } @@ -222,80 +275,10 @@ NTAPI MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait, PPFN_NUMBER AllocatedPage) { - ULONG PagesUsed; PFN_NUMBER Page; - /* - * Make sure we don't exceed our individual target. - */ - PagesUsed = InterlockedIncrementUL(&MiMemoryConsumers[Consumer].PagesUsed); - if (PagesUsed > MiMemoryConsumers[Consumer].PagesTarget && - !MiIsBalancerThread()) - { - MmRebalanceMemoryConsumers(); - } - - /* - * Allocate always memory for the non paged pool and for the pager thread. - */ - if ((Consumer == MC_SYSTEM) /* || MiIsBalancerThread() */) - { - Page = MmAllocPage(Consumer); - if (Page == 0) - { - KeBugCheck(NO_PAGES_AVAILABLE); - } - if (Consumer == MC_USER) MmInsertLRULastUserPage(Page); - *AllocatedPage = Page; - if (MmAvailablePages < MiMinimumAvailablePages) - MmRebalanceMemoryConsumers(); - return(STATUS_SUCCESS); - } - - /* - * Make sure we don't exceed global targets. - */ - if (((MmAvailablePages < MiMinimumAvailablePages) && !MiIsBalancerThread()) - || (MmAvailablePages < (MiMinimumAvailablePages / 2))) - { - MM_ALLOCATION_REQUEST Request; - - if (!CanWait) - { - (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed); - MmRebalanceMemoryConsumers(); - return(STATUS_NO_MEMORY); - } - - /* Insert an allocation request. */ - Request.Page = 0; - KeInitializeEvent(&Request.Event, NotificationEvent, FALSE); - - ExInterlockedInsertTailList(&AllocationListHead, &Request.ListEntry, &AllocationListLock); - MmRebalanceMemoryConsumers(); - - KeWaitForSingleObject(&Request.Event, - 0, - KernelMode, - FALSE, - NULL); - - Page = Request.Page; - if (Page == 0) - { - KeBugCheck(NO_PAGES_AVAILABLE); - } - - if(Consumer == MC_USER) MmInsertLRULastUserPage(Page); - *AllocatedPage = Page; - - if (MmAvailablePages < MiMinimumAvailablePages) - { - MmRebalanceMemoryConsumers(); - } - - return(STATUS_SUCCESS); - } + /* Update the target */ + InterlockedIncrementUL(&MiMemoryConsumers[Consumer].PagesUsed); /* * Actually allocate the page. @@ -305,14 +288,8 @@ MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait, { KeBugCheck(NO_PAGES_AVAILABLE); } - if(Consumer == MC_USER) MmInsertLRULastUserPage(Page); *AllocatedPage = Page; - if (MmAvailablePages < MiMinimumAvailablePages) - { - MmRebalanceMemoryConsumers(); - } - return(STATUS_SUCCESS); } @@ -387,6 +364,9 @@ MiBalancerThread(PVOID Unused) } } while (InitialTarget != 0); + + if (Status == STATUS_WAIT_0) + InterlockedDecrement(&PageOutThreadActive); } else { @@ -443,22 +423,14 @@ MiInitBalancerThread(VOID) { KPRIORITY Priority; NTSTATUS Status; -#if !defined(__GNUC__) - - LARGE_INTEGER dummyJunkNeeded; - dummyJunkNeeded.QuadPart = -20000000; /* 2 sec */ - ; -#endif - + LARGE_INTEGER Timeout; KeInitializeEvent(&MiBalancerEvent, SynchronizationEvent, FALSE); KeInitializeTimerEx(&MiBalancerTimer, SynchronizationTimer); + + Timeout.QuadPart = -20000000; /* 2 sec */ KeSetTimerEx(&MiBalancerTimer, -#if defined(__GNUC__) - (LARGE_INTEGER)(LONGLONG)-20000000LL, /* 2 sec */ -#else - dummyJunkNeeded, -#endif + Timeout, 2000, /* 2 sec */ NULL); diff --git a/ntoskrnl/mm/freelist.c b/ntoskrnl/mm/freelist.c index 93d9bbd9f59..4aa15a12582 100644 --- a/ntoskrnl/mm/freelist.c +++ b/ntoskrnl/mm/freelist.c @@ -35,95 +35,127 @@ SIZE_T MmPagedPoolCommit; SIZE_T MmPeakCommitment; SIZE_T MmtotalCommitLimitMaximum; -static RTL_BITMAP MiUserPfnBitMap; +PMMPFN FirstUserLRUPfn; +PMMPFN LastUserLRUPfn; /* FUNCTIONS *************************************************************/ -VOID -NTAPI -MiInitializeUserPfnBitmap(VOID) -{ - PVOID Bitmap; - - /* Allocate enough buffer for the PFN bitmap and align it on 32-bits */ - Bitmap = ExAllocatePoolWithTag(NonPagedPool, - (((MmHighestPhysicalPage + 1) + 31) / 32) * 4, - TAG_MM); - ASSERT(Bitmap); - - /* Initialize it and clear all the bits to begin with */ - RtlInitializeBitMap(&MiUserPfnBitMap, - Bitmap, - (ULONG)MmHighestPhysicalPage + 1); - RtlClearAllBits(&MiUserPfnBitMap); -} - PFN_NUMBER NTAPI MmGetLRUFirstUserPage(VOID) { - ULONG Position; + PFN_NUMBER Page; KIRQL OldIrql; /* Find the first user page */ OldIrql = MiAcquirePfnLock(); - Position = RtlFindSetBits(&MiUserPfnBitMap, 1, 0); - MiReleasePfnLock(OldIrql); - if (Position == 0xFFFFFFFF) return 0; - /* Return it */ - ASSERT(Position != 0); - ASSERT_IS_ROS_PFN(MiGetPfnEntry(Position)); - return Position; + if (FirstUserLRUPfn == NULL) + { + MiReleasePfnLock(OldIrql); + return 0; + } + + Page = MiGetPfnEntryIndex(FirstUserLRUPfn); + MmReferencePage(Page); + + MiReleasePfnLock(OldIrql); + + return Page; } +static VOID -NTAPI -MmInsertLRULastUserPage(PFN_NUMBER Pfn) +MmInsertLRULastUserPage(PFN_NUMBER Page) { - KIRQL OldIrql; + MI_ASSERT_PFN_LOCK_HELD(); - /* Set the page as a user page */ - ASSERT(Pfn != 0); - ASSERT_IS_ROS_PFN(MiGetPfnEntry(Pfn)); - ASSERT(!RtlCheckBit(&MiUserPfnBitMap, (ULONG)Pfn)); - OldIrql = MiAcquirePfnLock(); - RtlSetBit(&MiUserPfnBitMap, (ULONG)Pfn); - MiReleasePfnLock(OldIrql); + PMMPFN Pfn = MiGetPfnEntry(Page); + + if (FirstUserLRUPfn == NULL) + FirstUserLRUPfn = Pfn; + + Pfn->PreviousLRU = LastUserLRUPfn; + + if (LastUserLRUPfn != NULL) + LastUserLRUPfn->NextLRU = Pfn; + LastUserLRUPfn = Pfn; +} + +static +VOID +MmRemoveLRUUserPage(PFN_NUMBER Page) +{ + MI_ASSERT_PFN_LOCK_HELD(); + + /* Unset the page as a user page */ + ASSERT(Page != 0); + + PMMPFN Pfn = MiGetPfnEntry(Page); + + ASSERT_IS_ROS_PFN(Pfn); + + if (Pfn->PreviousLRU) + { + ASSERT(Pfn->PreviousLRU->NextLRU == Pfn); + Pfn->PreviousLRU->NextLRU = Pfn->NextLRU; + } + else + { + ASSERT(FirstUserLRUPfn == Pfn); + FirstUserLRUPfn = Pfn->NextLRU; + } + + if (Pfn->NextLRU) + { + ASSERT(Pfn->NextLRU->PreviousLRU == Pfn); + Pfn->NextLRU->PreviousLRU = Pfn->PreviousLRU; + } + else + { + ASSERT(Pfn == LastUserLRUPfn); + LastUserLRUPfn = Pfn->PreviousLRU; + } + + Pfn->PreviousLRU = Pfn->NextLRU = NULL; } PFN_NUMBER NTAPI -MmGetLRUNextUserPage(PFN_NUMBER PreviousPfn) +MmGetLRUNextUserPage(PFN_NUMBER PreviousPage, BOOLEAN MoveToLast) { - ULONG Position; + PFN_NUMBER Page = 0; KIRQL OldIrql; /* Find the next user page */ OldIrql = MiAcquirePfnLock(); - Position = RtlFindSetBits(&MiUserPfnBitMap, 1, (ULONG)PreviousPfn + 1); + + PMMPFN PreviousPfn = MiGetPfnEntry(PreviousPage); + PMMPFN NextPfn = PreviousPfn->NextLRU; + + /* + * Move this one at the end of the list. + * It may be freed by MmDereferencePage below. + * If it's not, then it means it is still hanging in some process address space. + * This avoids paging-out e.g. ntdll early just because it's mapped first time. + */ + if ((MoveToLast) && (MmGetReferenceCountPage(PreviousPage) > 1)) + { + MmRemoveLRUUserPage(PreviousPage); + MmInsertLRULastUserPage(PreviousPage); + } + + if (NextPfn) + { + Page = MiGetPfnEntryIndex(NextPfn); + MmReferencePage(Page); + } + + MmDereferencePage(PreviousPage); + MiReleasePfnLock(OldIrql); - if (Position == 0xFFFFFFFF) return 0; - /* Return it */ - ASSERT(Position != 0); - ASSERT_IS_ROS_PFN(MiGetPfnEntry(Position)); - return Position; -} - -VOID -NTAPI -MmRemoveLRUUserPage(PFN_NUMBER Page) -{ - KIRQL OldIrql; - - /* Unset the page as a user page */ - ASSERT(Page != 0); - ASSERT_IS_ROS_PFN(MiGetPfnEntry(Page)); - ASSERT(RtlCheckBit(&MiUserPfnBitMap, (ULONG)Page)); - OldIrql = MiAcquirePfnLock(); - RtlClearBit(&MiUserPfnBitMap, (ULONG)Page); - MiReleasePfnLock(OldIrql); + return Page; } BOOLEAN @@ -392,10 +424,11 @@ VOID NTAPI MmSetRmapListHeadPage(PFN_NUMBER Pfn, PMM_RMAP_ENTRY ListHead) { - KIRQL oldIrql; PMMPFN Pfn1; - oldIrql = MiAcquirePfnLock(); + /* PFN database must be locked */ + MI_ASSERT_PFN_LOCK_HELD(); + Pfn1 = MiGetPfnEntry(Pfn); ASSERT(Pfn1); ASSERT_IS_ROS_PFN(Pfn1); @@ -418,35 +451,31 @@ MmSetRmapListHeadPage(PFN_NUMBER Pfn, PMM_RMAP_ENTRY ListHead) /* ReactOS semantics will now release the page, which will make it free and enter a colored list */ } - - MiReleasePfnLock(oldIrql); } PMM_RMAP_ENTRY NTAPI MmGetRmapListHeadPage(PFN_NUMBER Pfn) { - KIRQL oldIrql; - PMM_RMAP_ENTRY ListHead; PMMPFN Pfn1; - /* Lock PFN database */ - oldIrql = MiAcquirePfnLock(); + /* PFN database must be locked */ + MI_ASSERT_PFN_LOCK_HELD(); /* Get the entry */ Pfn1 = MiGetPfnEntry(Pfn); ASSERT(Pfn1); - ASSERT_IS_ROS_PFN(Pfn1); - /* Get the list head */ - ListHead = Pfn1->RmapListHead; + if (!MI_IS_ROS_PFN(Pfn1)) + { + return NULL; + } /* Should not have an RMAP for a non-active page */ ASSERT(MiIsPfnInUse(Pfn1) == TRUE); - /* Release PFN database and return rmap list head */ - MiReleasePfnLock(oldIrql); - return ListHead; + /* Get the list head */ + return Pfn1->RmapListHead; } VOID @@ -550,6 +579,13 @@ MmDereferencePage(PFN_NUMBER Pfn) Pfn1->u3.e2.ReferenceCount--; if (Pfn1->u3.e2.ReferenceCount == 0) { + /* Apply LRU hack */ + if (Pfn1->u4.MustBeCached) + { + MmRemoveLRUUserPage(Pfn); + Pfn1->u4.MustBeCached = 0; + } + /* Mark the page temporarily as valid, we're going to make it free soon */ Pfn1->u3.e1.PageLocation = ActiveAndValid; @@ -592,6 +628,15 @@ MmAllocPage(ULONG Type) Pfn1->u1.SwapEntry = 0; Pfn1->RmapListHead = NULL; + Pfn1->NextLRU = NULL; + Pfn1->PreviousLRU = NULL; + + if (Type == MC_USER) + { + Pfn1->u4.MustBeCached = 1; /* HACK again */ + MmInsertLRULastUserPage(PfnOffset); + } + MiReleasePfnLock(OldIrql); return PfnOffset; } diff --git a/ntoskrnl/mm/i386/page.c b/ntoskrnl/mm/i386/page.c index bcc2be911b1..b541b5d4752 100644 --- a/ntoskrnl/mm/i386/page.c +++ b/ntoskrnl/mm/i386/page.c @@ -585,6 +585,46 @@ MmSetDirtyPage(PEPROCESS Process, PVOID Address) } } +VOID +NTAPI +MmClearPageAccessedBit(PEPROCESS Process, PVOID Address) +{ + PULONG Pt; + LONG Pte; + KIRQL OldIrql; + + if (Address < MmSystemRangeStart && Process == NULL) + { + DPRINT1("MmClearPageAccessedBit is called for user space without a process.\n"); + KeBugCheck(MEMORY_MANAGEMENT); + } + + Pt = MmGetPageTableForProcess(Process, Address, FALSE, &OldIrql); + if (Pt == NULL) + { + KeBugCheck(MEMORY_MANAGEMENT); + } + + do + { + Pte = *Pt; + } while (Pte != InterlockedCompareExchangePte(Pt, Pte & ~PA_ACCESSED, Pte)); + + if (!(Pte & PA_PRESENT)) + { + KeBugCheck(MEMORY_MANAGEMENT); + } + + MiFlushTlb(Pt, Address, OldIrql); +} + +BOOLEAN +NTAPI +MmIsPageAccessed(PEPROCESS Process, PVOID Address) +{ + return BooleanFlagOn(MmGetPageEntryForProcess(Process, Address), PA_ACCESSED); +} + BOOLEAN NTAPI MmIsPagePresent(PEPROCESS Process, PVOID Address) diff --git a/ntoskrnl/mm/marea.c b/ntoskrnl/mm/marea.c index d953f422f96..6e4226b6c35 100644 --- a/ntoskrnl/mm/marea.c +++ b/ntoskrnl/mm/marea.c @@ -165,12 +165,13 @@ MiMakeProtectionMask( static VOID MmInsertMemoryArea( PMMSUPPORT AddressSpace, - PMEMORY_AREA marea) + PMEMORY_AREA marea, + ULONG Protect) { PEPROCESS Process = MmGetAddressSpaceOwner(AddressSpace); marea->VadNode.u.VadFlags.Spare = 1; - marea->VadNode.u.VadFlags.Protection = MiMakeProtectionMask(marea->Protect); + marea->VadNode.u.VadFlags.Protection = MiMakeProtectionMask(Protect); /* Build a lame VAD if this is a user-space allocation */ if (marea->VadNode.EndingVpn + 1 < (ULONG_PTR)MmSystemRangeStart >> PAGE_SHIFT) @@ -178,7 +179,11 @@ MmInsertMemoryArea( ASSERT(Process != NULL); if (marea->Type != MEMORY_AREA_OWNED_BY_ARM3) { +#ifdef NEWCC ASSERT(marea->Type == MEMORY_AREA_SECTION_VIEW || marea->Type == MEMORY_AREA_CACHE); +#else + ASSERT(marea->Type == MEMORY_AREA_SECTION_VIEW); +#endif /* Insert the VAD */ MiLockProcessWorkingSetUnsafe(PsGetCurrentProcess(), PsGetCurrentThread()); @@ -353,7 +358,11 @@ MmFreeMemoryArea( if (MemoryArea->Vad) { ASSERT(MemoryArea->VadNode.EndingVpn + 1 < (ULONG_PTR)MmSystemRangeStart >> PAGE_SHIFT); +#ifdef NEWCC ASSERT(MemoryArea->Type == MEMORY_AREA_SECTION_VIEW || MemoryArea->Type == MEMORY_AREA_CACHE); +#else + ASSERT(MemoryArea->Type == MEMORY_AREA_SECTION_VIEW); +#endif /* MmCleanProcessAddressSpace might have removed it (and this would be MmDeleteProcessAdressSpace) */ ASSERT(MemoryArea->VadNode.u.VadFlags.Spare != 0); @@ -449,7 +458,6 @@ MmCreateMemoryArea(PMMSUPPORT AddressSpace, RtlZeroMemory(MemoryArea, sizeof(MEMORY_AREA)); MemoryArea->Type = Type & ~MEMORY_AREA_STATIC; - MemoryArea->Protect = Protect; MemoryArea->Flags = AllocationFlags; MemoryArea->Magic = 'erAM'; MemoryArea->DeleteInProgress = FALSE; @@ -470,7 +478,7 @@ MmCreateMemoryArea(PMMSUPPORT AddressSpace, MemoryArea->VadNode.StartingVpn = (ULONG_PTR)*BaseAddress >> PAGE_SHIFT; MemoryArea->VadNode.EndingVpn = ((ULONG_PTR)*BaseAddress + tmpLength - 1) >> PAGE_SHIFT; - MmInsertMemoryArea(AddressSpace, MemoryArea); + MmInsertMemoryArea(AddressSpace, MemoryArea, Protect); } else { @@ -508,7 +516,7 @@ MmCreateMemoryArea(PMMSUPPORT AddressSpace, MemoryArea->VadNode.StartingVpn = (ULONG_PTR)*BaseAddress >> PAGE_SHIFT; MemoryArea->VadNode.EndingVpn = ((ULONG_PTR)*BaseAddress + tmpLength - 1) >> PAGE_SHIFT; - MmInsertMemoryArea(AddressSpace, MemoryArea); + MmInsertMemoryArea(AddressSpace, MemoryArea, Protect); } *Result = MemoryArea; @@ -535,9 +543,6 @@ MiRosCleanupMemoryArea( (Process->ActiveThreads == 1)) || (Process->ActiveThreads == 0)); - /* We are in cleanup, we don't need to synchronize */ - MmUnlockAddressSpace(&Process->Vm); - MemoryArea = (PMEMORY_AREA)Vad; BaseAddress = (PVOID)MA_GetStartingAddress(MemoryArea); @@ -545,10 +550,12 @@ MiRosCleanupMemoryArea( { Status = MiRosUnmapViewOfSection(Process, BaseAddress, Process->ProcessExiting); } +#ifdef NEWCC else if (MemoryArea->Type == MEMORY_AREA_CACHE) { Status = MmUnmapViewOfCacheSegment(&Process->Vm, BaseAddress); } +#endif else { /* There shouldn't be anything else! */ @@ -557,9 +564,6 @@ MiRosCleanupMemoryArea( /* Make sure this worked! */ ASSERT(NT_SUCCESS(Status)); - - /* Lock the address space again */ - MmLockAddressSpace(&Process->Vm); } VOID diff --git a/ntoskrnl/mm/mmfault.c b/ntoskrnl/mm/mmfault.c index 68910e7a2d9..75f7b584a22 100644 --- a/ntoskrnl/mm/mmfault.c +++ b/ntoskrnl/mm/mmfault.c @@ -79,7 +79,7 @@ MmpAccessFault(KPROCESSOR_MODE Mode, MemoryArea, (PVOID)Address); break; - +#ifdef NEWCC case MEMORY_AREA_CACHE: // This code locks for itself to keep from having to break a lock // passed in. @@ -89,7 +89,7 @@ MmpAccessFault(KPROCESSOR_MODE Mode, if (!FromMdl) MmLockAddressSpace(AddressSpace); break; - +#endif default: Status = STATUS_ACCESS_VIOLATION; break; @@ -171,7 +171,7 @@ MmNotPresentFault(KPROCESSOR_MODE Mode, (PVOID)Address, FromMdl); break; - +#ifdef NEWCC case MEMORY_AREA_CACHE: // This code locks for itself to keep from having to break a lock // passed in. @@ -181,7 +181,7 @@ MmNotPresentFault(KPROCESSOR_MODE Mode, if (!FromMdl) MmLockAddressSpace(AddressSpace); break; - +#endif default: Status = STATUS_ACCESS_VIOLATION; break; diff --git a/ntoskrnl/mm/mminit.c b/ntoskrnl/mm/mminit.c index 422ba568bb9..3e58ad9bbc8 100644 --- a/ntoskrnl/mm/mminit.c +++ b/ntoskrnl/mm/mminit.c @@ -17,8 +17,6 @@ /* GLOBALS *******************************************************************/ -VOID NTAPI MiInitializeUserPfnBitmap(VOID); - BOOLEAN Mm64BitPhysicalAddress = FALSE; ULONG MmReadClusterSize; // @@ -214,12 +212,17 @@ MmInitSystem(IN ULONG Phase, /* Initialize the kernel address space */ ASSERT(Phase == 1); +#ifdef NEWCC InitializeListHead(&MiSegmentList); ExInitializeFastMutex(&MiGlobalPageOperation); KeInitializeEvent(&MmWaitPageEvent, SynchronizationEvent, FALSE); // Until we're fully demand paged, we can do things the old way through // the balance manager + // CcInitView will override this... MmInitializeMemoryConsumer(MC_CACHE, MiRosTrimCache); +#else + KeInitializeEvent(&MmWaitPageEvent, SynchronizationEvent, FALSE); +#endif MmKernelAddressSpace = &PsIdleProcess->Vm; @@ -230,7 +233,6 @@ MmInitSystem(IN ULONG Phase, MiDbgDumpAddressSpace(); MmInitGlobalKernelPageDirectory(); - MiInitializeUserPfnBitmap(); MmInitializeMemoryConsumer(MC_USER, MmTrimUserMemory); MmInitializeRmapList(); MmInitSectionImplementation(); diff --git a/ntoskrnl/mm/pagefile.c b/ntoskrnl/mm/pagefile.c index 37870520998..e0f9e087150 100644 --- a/ntoskrnl/mm/pagefile.c +++ b/ntoskrnl/mm/pagefile.c @@ -94,9 +94,6 @@ NTAPI MmBuildMdlFromPages(PMDL Mdl, PPFN_NUMBER Pages) { memcpy(Mdl + 1, Pages, sizeof(PFN_NUMBER) * (PAGE_ROUND_UP(Mdl->ByteOffset+Mdl->ByteCount)/PAGE_SIZE)); - - /* FIXME: this flag should be set by the caller perhaps? */ - Mdl->MdlFlags |= MDL_IO_PAGE_READ; } @@ -230,7 +227,7 @@ MiReadPageFile( MmInitializeMdl(Mdl, NULL, PAGE_SIZE); MmBuildMdlFromPages(Mdl, &Page); - Mdl->MdlFlags |= MDL_PAGES_LOCKED; + Mdl->MdlFlags |= MDL_PAGES_LOCKED | MDL_IO_PAGE_READ; file_offset.QuadPart = PageFileOffset * PAGE_SIZE; diff --git a/ntoskrnl/mm/rmap.c b/ntoskrnl/mm/rmap.c index aa6a7b0d6f2..cb3f956ef35 100644 --- a/ntoskrnl/mm/rmap.c +++ b/ntoskrnl/mm/rmap.c @@ -19,7 +19,6 @@ /* GLOBALS ******************************************************************/ static NPAGED_LOOKASIDE_LIST RmapLookasideList; -FAST_MUTEX RmapListLock; /* FUNCTIONS ****************************************************************/ @@ -38,7 +37,6 @@ VOID NTAPI MmInitializeRmapList(VOID) { - ExInitializeFastMutex(&RmapListLock); ExInitializeNPagedLookasideList (&RmapLookasideList, NULL, RmapListFree, @@ -55,37 +53,28 @@ MmPageOutPhysicalAddress(PFN_NUMBER Page) PMM_RMAP_ENTRY entry; PMEMORY_AREA MemoryArea; PMMSUPPORT AddressSpace; - ULONG Type; - PVOID Address; - PEPROCESS Process; - ULONGLONG Offset; + PVOID Address = NULL; + PEPROCESS Process = NULL; NTSTATUS Status = STATUS_SUCCESS; + PMM_SECTION_SEGMENT Segment; + LARGE_INTEGER SegmentOffset; + KIRQL OldIrql; + +GetEntry: + OldIrql = MiAcquirePfnLock(); - ExAcquireFastMutex(&RmapListLock); entry = MmGetRmapListHeadPage(Page); -#ifdef NEWCC - // Special case for NEWCC: we can have a page that's only in a segment - // page table - if (entry && RMAP_IS_SEGMENT(entry->Address) && entry->Next == NULL) - { - /* NEWCC does locking itself */ - ExReleaseFastMutex(&RmapListLock); - return MmpPageOutPhysicalAddress(Page); - } -#endif - while (entry && RMAP_IS_SEGMENT(entry->Address)) entry = entry->Next; if (entry == NULL) { - ExReleaseFastMutex(&RmapListLock); - return(STATUS_UNSUCCESSFUL); + MiReleasePfnLock(OldIrql); + goto WriteSegment; } Process = entry->Process; - Address = entry->Address; if ((((ULONG_PTR)Address) & 0xFFF) != 0) @@ -93,58 +82,64 @@ MmPageOutPhysicalAddress(PFN_NUMBER Page) KeBugCheck(MEMORY_MANAGEMENT); } - if (Address < MmSystemRangeStart) - { - if (!ExAcquireRundownProtection(&Process->RundownProtect)) - { - ExReleaseFastMutex(&RmapListLock); - return STATUS_PROCESS_IS_TERMINATING; - } + /* This is for user-mode address only */ + ASSERT(Address < MmSystemRangeStart); - Status = ObReferenceObjectByPointer(Process, PROCESS_ALL_ACCESS, NULL, KernelMode); - ExReleaseFastMutex(&RmapListLock); - if (!NT_SUCCESS(Status)) - { - ExReleaseRundownProtection(&Process->RundownProtect); - return Status; - } - AddressSpace = &Process->Vm; - } - else + if (!ExAcquireRundownProtection(&Process->RundownProtect)) { - ExReleaseFastMutex(&RmapListLock); - AddressSpace = MmGetKernelAddressSpace(); + MiReleasePfnLock(OldIrql); + return STATUS_PROCESS_IS_TERMINATING; } + Status = ObReferenceObjectByPointer(Process, PROCESS_ALL_ACCESS, NULL, KernelMode); + MiReleasePfnLock(OldIrql); + if (!NT_SUCCESS(Status)) + { + ExReleaseRundownProtection(&Process->RundownProtect); + return Status; + } + AddressSpace = &Process->Vm; + MmLockAddressSpace(AddressSpace); + + if (MmGetPfnForProcess(Process, Address) != Page) + { + /* This changed in the short window where we didn't have any locks */ + MmUnlockAddressSpace(AddressSpace); + ExReleaseRundownProtection(&Process->RundownProtect); + ObDereferenceObject(Process); + goto GetEntry; + } + MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace, Address); if (MemoryArea == NULL || MemoryArea->DeleteInProgress) { MmUnlockAddressSpace(AddressSpace); - if (Address < MmSystemRangeStart) - { - ExReleaseRundownProtection(&Process->RundownProtect); - ObDereferenceObject(Process); - } - return(STATUS_UNSUCCESSFUL); + ExReleaseRundownProtection(&Process->RundownProtect); + ObDereferenceObject(Process); + goto GetEntry; } - Type = MemoryArea->Type; - if (Type == MEMORY_AREA_SECTION_VIEW) + + if (MemoryArea->Type == MEMORY_AREA_SECTION_VIEW) { ULONG_PTR Entry; - Offset = MemoryArea->Data.SectionData.ViewOffset.QuadPart + + BOOLEAN Dirty; + PFN_NUMBER MapPage; + LARGE_INTEGER Offset; + BOOLEAN Released; + + Offset.QuadPart = MemoryArea->SectionData.ViewOffset + ((ULONG_PTR)Address - MA_GetStartingAddress(MemoryArea)); - MmLockSectionSegment(MemoryArea->Data.SectionData.Segment); + Segment = MemoryArea->SectionData.Segment; - /* - * Get or create a pageop - */ - Entry = MmGetPageEntrySectionSegment(MemoryArea->Data.SectionData.Segment, - (PLARGE_INTEGER)&Offset); + MmLockSectionSegment(Segment); + + Entry = MmGetPageEntrySectionSegment(Segment, &Offset); if (Entry && MM_IS_WAIT_PTE(Entry)) { - MmUnlockSectionSegment(MemoryArea->Data.SectionData.Segment); + /* The segment is being read or something. Give up */ + MmUnlockSectionSegment(Segment); MmUnlockAddressSpace(AddressSpace); if (Address < MmSystemRangeStart) { @@ -154,36 +149,147 @@ MmPageOutPhysicalAddress(PFN_NUMBER Page) return(STATUS_UNSUCCESSFUL); } - MmSetPageEntrySectionSegment(MemoryArea->Data.SectionData.Segment, (PLARGE_INTEGER)&Offset, MAKE_SWAP_SSE(MM_WAIT_ENTRY)); + /* Delete this virtual mapping in the process */ + MmDeleteVirtualMapping(Process, Address, &Dirty, &MapPage); - /* - * Release locks now we have a page op. - */ - MmUnlockSectionSegment(MemoryArea->Data.SectionData.Segment); + /* We checked this earlier */ + ASSERT(MapPage == Page); + + if (Page != PFN_FROM_SSE(Entry)) + { + SWAPENTRY SwapEntry; + + /* This page is private to the process */ + MmUnlockSectionSegment(Segment); + + /* Check if we should write it back to the page file */ + SwapEntry = MmGetSavedSwapEntryPage(Page); + + if ((SwapEntry == 0) && Dirty) + { + /* We don't have a Swap entry, yet the page is dirty. Get one */ + SwapEntry = MmAllocSwapPage(); + if (!SwapEntry) + { + PMM_REGION Region = MmFindRegion((PVOID)MA_GetStartingAddress(MemoryArea), + &MemoryArea->SectionData.RegionListHead, + Address, NULL); + + /* We can't, so let this page in the Process VM */ + MmCreateVirtualMapping(Process, Address, Region->Protect, &Page, 1); + MmSetDirtyPage(Process, Address); + + MmUnlockAddressSpace(AddressSpace); + if (Address < MmSystemRangeStart) + { + ExReleaseRundownProtection(&Process->RundownProtect); + ObDereferenceObject(Process); + } + + return STATUS_UNSUCCESSFUL; + } + } + + if (Dirty) + { + Status = MmWriteToSwapPage(SwapEntry, Page); + if (!NT_SUCCESS(Status)) + { + /* We failed at saving the content of this page. Keep it in */ + PMM_REGION Region = MmFindRegion((PVOID)MA_GetStartingAddress(MemoryArea), + &MemoryArea->SectionData.RegionListHead, + Address, NULL); + + /* This Swap Entry is useless to us */ + MmSetSavedSwapEntryPage(Page, 0); + MmFreeSwapPage(SwapEntry); + + /* We can't, so let this page in the Process VM */ + MmCreateVirtualMapping(Process, Address, Region->Protect, &Page, 1); + MmSetDirtyPage(Process, Address); + + MmUnlockAddressSpace(AddressSpace); + ExReleaseRundownProtection(&Process->RundownProtect); + ObDereferenceObject(Process); + + return STATUS_UNSUCCESSFUL; + } + } + + if (SwapEntry) + { + /* Keep this in the process VM */ + MmCreatePageFileMapping(Process, Address, SwapEntry); + MmSetSavedSwapEntryPage(Page, 0); + } + + MmUnlockAddressSpace(AddressSpace); + + /* We can finally let this page go */ + MmDeleteRmap(Page, Process, Address); +#if DBG + OldIrql = MiAcquirePfnLock(); + ASSERT(MmGetRmapListHeadPage(Page) == NULL); + MiReleasePfnLock(OldIrql); +#endif + MmReleasePageMemoryConsumer(MC_USER, Page); + + ExReleaseRundownProtection(&Process->RundownProtect); + ObDereferenceObject(Process); + + return STATUS_SUCCESS; + } + + /* Delete this RMAP */ + MmDeleteRmap(Page, Process, Address); + + /* One less mapping referencing this segment */ + Released = MmUnsharePageEntrySectionSegment(MemoryArea, Segment, &Offset, Dirty, TRUE, NULL); + + MmUnlockSectionSegment(Segment); MmUnlockAddressSpace(AddressSpace); - /* - * Do the actual page out work. - */ - Status = MmPageOutSectionView(AddressSpace, MemoryArea, Address, Entry); + ExReleaseRundownProtection(&Process->RundownProtect); + ObDereferenceObject(Process); + + if (Released) return STATUS_SUCCESS; } +#ifdef NEWCC else if (Type == MEMORY_AREA_CACHE) { /* NEWCC does locking itself */ MmUnlockAddressSpace(AddressSpace); Status = MmpPageOutPhysicalAddress(Page); } +#endif else { KeBugCheck(MEMORY_MANAGEMENT); } - if (Address < MmSystemRangeStart) +WriteSegment: + /* Now write this page to file, if needed */ + Segment = MmGetSectionAssociation(Page, &SegmentOffset); + if (Segment) { - ExReleaseRundownProtection(&Process->RundownProtect); - ObDereferenceObject(Process); + BOOLEAN Released; + + MmLockSectionSegment(Segment); + + Released = MmCheckDirtySegment(Segment, &SegmentOffset, FALSE, TRUE); + + MmUnlockSectionSegment(Segment); + + MmDereferenceSegment(Segment); + + if (Released) + { + return STATUS_SUCCESS; + } } - return(Status); + + /* If we are here, then we didn't release the page */ + return STATUS_UNSUCCESSFUL; } VOID @@ -191,12 +297,13 @@ NTAPI MmSetCleanAllRmaps(PFN_NUMBER Page) { PMM_RMAP_ENTRY current_entry; + KIRQL OldIrql; - ExAcquireFastMutex(&RmapListLock); + OldIrql = MiAcquirePfnLock(); current_entry = MmGetRmapListHeadPage(Page); if (current_entry == NULL) { - DPRINT1("MmIsDirtyRmap: No rmaps.\n"); + DPRINT1("MmSetCleanAllRmaps: No rmaps.\n"); KeBugCheck(MEMORY_MANAGEMENT); } while (current_entry != NULL) @@ -205,29 +312,7 @@ MmSetCleanAllRmaps(PFN_NUMBER Page) MmSetCleanPage(current_entry->Process, current_entry->Address); current_entry = current_entry->Next; } - ExReleaseFastMutex(&RmapListLock); -} - -VOID -NTAPI -MmSetDirtyAllRmaps(PFN_NUMBER Page) -{ - PMM_RMAP_ENTRY current_entry; - - ExAcquireFastMutex(&RmapListLock); - current_entry = MmGetRmapListHeadPage(Page); - if (current_entry == NULL) - { - DPRINT1("MmIsDirtyRmap: No rmaps.\n"); - KeBugCheck(MEMORY_MANAGEMENT); - } - while (current_entry != NULL) - { - if (!RMAP_IS_SEGMENT(current_entry->Address)) - MmSetDirtyPage(current_entry->Process, current_entry->Address); - current_entry = current_entry->Next; - } - ExReleaseFastMutex(&RmapListLock); + MiReleasePfnLock(OldIrql); } BOOLEAN @@ -235,27 +320,31 @@ NTAPI MmIsDirtyPageRmap(PFN_NUMBER Page) { PMM_RMAP_ENTRY current_entry; + KIRQL OldIrql; + BOOLEAN Dirty = FALSE; - ExAcquireFastMutex(&RmapListLock); + OldIrql = MiAcquirePfnLock(); current_entry = MmGetRmapListHeadPage(Page); if (current_entry == NULL) { - ExReleaseFastMutex(&RmapListLock); - return(FALSE); + DPRINT1("MmIsDirtyPageRmap: No rmaps.\n"); + KeBugCheck(MEMORY_MANAGEMENT); } while (current_entry != NULL) { - if ( - !RMAP_IS_SEGMENT(current_entry->Address) && - MmIsDirtyPage(current_entry->Process, current_entry->Address)) + if (!RMAP_IS_SEGMENT(current_entry->Address)) { - ExReleaseFastMutex(&RmapListLock); - return(TRUE); + if (MmIsDirtyPage(current_entry->Process, current_entry->Address)) + { + Dirty = TRUE; + break; + } } current_entry = current_entry->Next; } - ExReleaseFastMutex(&RmapListLock); - return(FALSE); + MiReleasePfnLock(OldIrql); + + return Dirty; } VOID @@ -266,6 +355,8 @@ MmInsertRmap(PFN_NUMBER Page, PEPROCESS Process, PMM_RMAP_ENTRY current_entry; PMM_RMAP_ENTRY new_entry; ULONG PrevSize; + KIRQL OldIrql; + if (!RMAP_IS_SEGMENT(Address)) Address = (PVOID)PAGE_ROUND_DOWN(Address); @@ -277,11 +368,7 @@ MmInsertRmap(PFN_NUMBER Page, PEPROCESS Process, new_entry->Address = Address; new_entry->Process = (PEPROCESS)Process; #if DBG -#ifdef __GNUC__ - new_entry->Caller = __builtin_return_address(0); -#else new_entry->Caller = _ReturnAddress(); -#endif #endif if ( @@ -296,89 +383,50 @@ MmInsertRmap(PFN_NUMBER Page, PEPROCESS Process, KeBugCheck(MEMORY_MANAGEMENT); } - ExAcquireFastMutex(&RmapListLock); + OldIrql = MiAcquirePfnLock(); current_entry = MmGetRmapListHeadPage(Page); - new_entry->Next = current_entry; -#if DBG - while (current_entry) - { - if (current_entry->Address == new_entry->Address && current_entry->Process == new_entry->Process) - { - DbgPrint("MmInsertRmap tries to add a second rmap entry for address %p\n current caller ", - current_entry->Address); - DbgPrint("%p", new_entry->Caller); - DbgPrint("\n previous caller "); - DbgPrint("%p", current_entry->Caller); - DbgPrint("\n"); - KeBugCheck(MEMORY_MANAGEMENT); - } - current_entry = current_entry->Next; - } -#endif - MmSetRmapListHeadPage(Page, new_entry); - ExReleaseFastMutex(&RmapListLock); - if (!RMAP_IS_SEGMENT(Address)) - { - if (Process == NULL) - { - Process = PsInitialSystemProcess; - } - if (Process) - { - PrevSize = InterlockedExchangeAddUL(&Process->Vm.WorkingSetSize, PAGE_SIZE); - if (PrevSize >= Process->Vm.PeakWorkingSetSize) - { - Process->Vm.PeakWorkingSetSize = PrevSize + PAGE_SIZE; - } - } - } -} -VOID -NTAPI -MmDeleteAllRmaps(PFN_NUMBER Page, PVOID Context, - VOID (*DeleteMapping)(PVOID Context, PEPROCESS Process, - PVOID Address)) -{ - PMM_RMAP_ENTRY current_entry; - PMM_RMAP_ENTRY previous_entry; - PEPROCESS Process; - - ExAcquireFastMutex(&RmapListLock); - current_entry = MmGetRmapListHeadPage(Page); - if (current_entry == NULL) - { - DPRINT1("MmDeleteAllRmaps: No rmaps.\n"); - KeBugCheck(MEMORY_MANAGEMENT); - } - MmSetRmapListHeadPage(Page, NULL); - ExReleaseFastMutex(&RmapListLock); - - while (current_entry != NULL) + PMM_RMAP_ENTRY previous_entry = NULL; + /* Keep the list sorted */ + while (current_entry && (current_entry->Address < Address)) { previous_entry = current_entry; current_entry = current_entry->Next; - if (!RMAP_IS_SEGMENT(previous_entry->Address)) + } + + /* In case of clash in the address, sort by process */ + if (current_entry && (current_entry->Address == Address)) + { + while (current_entry && (current_entry->Process < Process)) { - if (DeleteMapping) - { - DeleteMapping(Context, previous_entry->Process, - previous_entry->Address); - } - Process = previous_entry->Process; - ExFreeToNPagedLookasideList(&RmapLookasideList, previous_entry); - if (Process == NULL) - { - Process = PsInitialSystemProcess; - } - if (Process) - { - (void)InterlockedExchangeAddUL(&Process->Vm.WorkingSetSize, -PAGE_SIZE); - } + previous_entry = current_entry; + current_entry = current_entry->Next; } - else + } + + if (current_entry && (current_entry->Address == Address) && (current_entry->Process == Process)) + { + DbgPrint("MmInsertRmap tries to add a second rmap entry for address %p\n", current_entry->Address); + DbgPrint(" current caller %p\n", new_entry->Caller); + DbgPrint(" previous caller %p\n", current_entry->Caller); + KeBugCheck(MEMORY_MANAGEMENT); + } + + new_entry->Next = current_entry; + if (previous_entry) + previous_entry->Next = new_entry; + else + MmSetRmapListHeadPage(Page, new_entry); + + MiReleasePfnLock(OldIrql); + + if (!RMAP_IS_SEGMENT(Address)) + { + ASSERT(Process != NULL); + PrevSize = InterlockedExchangeAddUL(&Process->Vm.WorkingSetSize, PAGE_SIZE); + if (PrevSize >= Process->Vm.PeakWorkingSetSize) { - ExFreeToNPagedLookasideList(&RmapLookasideList, previous_entry); + Process->Vm.PeakWorkingSetSize = PrevSize + PAGE_SIZE; } } } @@ -389,8 +437,9 @@ MmDeleteRmap(PFN_NUMBER Page, PEPROCESS Process, PVOID Address) { PMM_RMAP_ENTRY current_entry, previous_entry; + KIRQL OldIrql; - ExAcquireFastMutex(&RmapListLock); + OldIrql = MiAcquirePfnLock(); previous_entry = NULL; current_entry = MmGetRmapListHeadPage(Page); @@ -407,18 +456,13 @@ MmDeleteRmap(PFN_NUMBER Page, PEPROCESS Process, { previous_entry->Next = current_entry->Next; } - ExReleaseFastMutex(&RmapListLock); + MiReleasePfnLock(OldIrql); + ExFreeToNPagedLookasideList(&RmapLookasideList, current_entry); if (!RMAP_IS_SEGMENT(Address)) { - if (Process == NULL) - { - Process = PsInitialSystemProcess; - } - if (Process) - { - (void)InterlockedExchangeAddUL(&Process->Vm.WorkingSetSize, -PAGE_SIZE); - } + ASSERT(Process != NULL); + (void)InterlockedExchangeAddUL(&Process->Vm.WorkingSetSize, -PAGE_SIZE); } return; } @@ -448,8 +492,8 @@ MmGetSegmentRmap(PFN_NUMBER Page, PULONG RawOffset) { PCACHE_SECTION_PAGE_TABLE Result = NULL; PMM_RMAP_ENTRY current_entry;//, previous_entry; + KIRQL OldIrql = MiAcquirePfnLock(); - ExAcquireFastMutex(&RmapListLock); //previous_entry = NULL; current_entry = MmGetRmapListHeadPage(Page); while (current_entry != NULL) @@ -458,14 +502,20 @@ MmGetSegmentRmap(PFN_NUMBER Page, PULONG RawOffset) { Result = (PCACHE_SECTION_PAGE_TABLE)current_entry->Process; *RawOffset = (ULONG_PTR)current_entry->Address & ~RMAP_SEGMENT_MASK; - InterlockedIncrementUL(&Result->Segment->ReferenceCount); - ExReleaseFastMutex(&RmapListLock); + if (*Result->Segment->Flags & MM_SEGMENT_INDELETE) + { + MiReleasePfnLock(OldIrql); + return NULL; + } + + InterlockedIncrement64(Result->Segment->ReferenceCount); + MiReleasePfnLock(OldIrql); return Result; } //previous_entry = current_entry; current_entry = current_entry->Next; } - ExReleaseFastMutex(&RmapListLock); + MiReleasePfnLock(OldIrql); return NULL; } @@ -480,8 +530,8 @@ NTAPI MmDeleteSectionAssociation(PFN_NUMBER Page) { PMM_RMAP_ENTRY current_entry, previous_entry; + KIRQL OldIrql = MiAcquirePfnLock(); - ExAcquireFastMutex(&RmapListLock); previous_entry = NULL; current_entry = MmGetRmapListHeadPage(Page); while (current_entry != NULL) @@ -496,12 +546,12 @@ MmDeleteSectionAssociation(PFN_NUMBER Page) { previous_entry->Next = current_entry->Next; } - ExReleaseFastMutex(&RmapListLock); + MiReleasePfnLock(OldIrql); ExFreeToNPagedLookasideList(&RmapLookasideList, current_entry); return; } previous_entry = current_entry; current_entry = current_entry->Next; } - ExReleaseFastMutex(&RmapListLock); + MiReleasePfnLock(OldIrql); } diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index f586ec37754..3f364b49f4d 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -60,12 +60,76 @@ extern MMSESSION MmSession; -NTSTATUS +static LARGE_INTEGER TinyTime = {{-1L, -1L}}; + +#ifndef NEWCC +KEVENT MmWaitPageEvent; + +VOID NTAPI +_MmLockSectionSegment(PMM_SECTION_SEGMENT Segment, const char *file, int line) +{ + //DPRINT("MmLockSectionSegment(%p,%s:%d)\n", Segment, file, line); + ExAcquireFastMutex(&Segment->Lock); + Segment->Locked = TRUE; +} + +VOID +NTAPI +_MmUnlockSectionSegment(PMM_SECTION_SEGMENT Segment, const char *file, int line) +{ + ASSERT(Segment->Locked); + Segment->Locked = FALSE; + ExReleaseFastMutex(&Segment->Lock); + //DPRINT("MmUnlockSectionSegment(%p,%s:%d)\n", Segment, file, line); +} +#endif + +static +PMM_SECTION_SEGMENT +MiGrabDataSection(PSECTION_OBJECT_POINTERS SectionObjectPointer) +{ + KIRQL OldIrql = MiAcquirePfnLock(); + PMM_SECTION_SEGMENT Segment = NULL; + + while (TRUE) + { + Segment = SectionObjectPointer->DataSectionObject; + if (!Segment) + break; + + if (Segment->SegFlags & (MM_SEGMENT_INCREATE | MM_SEGMENT_INDELETE)) + { + MiReleasePfnLock(OldIrql); + KeDelayExecutionThread(KernelMode, FALSE, &TinyTime); + OldIrql = MiAcquirePfnLock(); + continue; + } + + ASSERT(Segment->SegFlags & MM_DATAFILE_SEGMENT); + InterlockedIncrement64(&Segment->RefCount); + break; + } + + MiReleasePfnLock(OldIrql); + + return Segment; +} + +/* Somewhat grotesque, but eh... */ +PMM_IMAGE_SECTION_OBJECT ImageSectionObjectFromSegment(PMM_SECTION_SEGMENT Segment) +{ + ASSERT((Segment->SegFlags & MM_DATAFILE_SEGMENT) == 0); + + return CONTAINING_RECORD(Segment->ReferenceCount, MM_IMAGE_SECTION_OBJECT, RefCount); +} + +NTSTATUS MiMapViewInSystemSpace(IN PVOID Section, IN PVOID Session, OUT PVOID *MappedBase, - IN OUT PSIZE_T ViewSize); + IN OUT PSIZE_T ViewSize, + IN PLARGE_INTEGER SectionOffset); NTSTATUS NTAPI @@ -115,7 +179,7 @@ C_ASSERT(PEFMT_FIELDS_EQUAL(IMAGE_OPTIONAL_HEADER32, IMAGE_OPTIONAL_HEADER64, Si typedef struct { - PROS_SECTION_OBJECT Section; + PMEMORY_AREA MemoryArea; PMM_SECTION_SEGMENT Segment; LARGE_INTEGER Offset; BOOLEAN WasDirty; @@ -169,6 +233,51 @@ static GENERIC_MAPPING MmpSectionMapping = /* FUNCTIONS *****************************************************************/ + +NTSTATUS +NTAPI +MiWritePage(PMM_SECTION_SEGMENT Segment, + LONGLONG SegOffset, + PFN_NUMBER Page) +/* + * FUNCTION: write a page for a section backed memory area. + * PARAMETERS: + * MemoryArea - Memory area to write the page for. + * Offset - Offset of the page to write. + * Page - Page which contains the data to write. + */ +{ + NTSTATUS Status; + IO_STATUS_BLOCK IoStatus; + KEVENT Event; + UCHAR MdlBase[sizeof(MDL) + sizeof(PFN_NUMBER)]; + PMDL Mdl = (PMDL)MdlBase; + PFILE_OBJECT FileObject = Segment->FileObject; + LARGE_INTEGER FileOffset; + + FileOffset.QuadPart = Segment->Image.FileOffset + SegOffset; + + RtlZeroMemory(MdlBase, sizeof(MdlBase)); + MmInitializeMdl(Mdl, NULL, PAGE_SIZE); + MmBuildMdlFromPages(Mdl, &Page); + Mdl->MdlFlags |= MDL_PAGES_LOCKED; + + KeInitializeEvent(&Event, NotificationEvent, FALSE); + Status = IoSynchronousPageWrite(FileObject, Mdl, &FileOffset, &Event, &IoStatus); + if (Status == STATUS_PENDING) + { + KeWaitForSingleObject(&Event, Executive, KernelMode, FALSE, NULL); + Status = IoStatus.Status; + } + if (Mdl->MdlFlags & MDL_MAPPED_TO_SYSTEM_VA) + { + MmUnmapLockedPages (Mdl->MappedSystemVa, Mdl); + } + + return Status; +} + + /* References: [1] Microsoft Corporation, "Microsoft Portable Executable and Common Object @@ -689,6 +798,8 @@ l_ReadHeaderFromFile: nStatus = STATUS_INVALID_IMAGE_FORMAT; + ASSERT(ImageSectionObject->RefCount > 0); + /* convert the executable sections into segments. See also [1], section 4 */ for(i = 0; i < ImageSectionObject->NrSegments - 1; ++ i) { @@ -800,49 +911,146 @@ MmspWaitForFileLock(PFILE_OBJECT File) //return KeWaitForSingleObject(&File->Lock, 0, KernelMode, FALSE, NULL); } + + VOID NTAPI -MmFreeSectionSegments(PFILE_OBJECT FileObject) +MmpFreePageFileSegment(PMM_SECTION_SEGMENT Segment) { - if (FileObject->SectionObjectPointer->ImageSectionObject != NULL) + ULONG Length; + LARGE_INTEGER Offset; + ULONG_PTR Entry; + SWAPENTRY SavedSwapEntry; + PFN_NUMBER Page; + + Page = 0; + + MmLockSectionSegment(Segment); + + Length = PAGE_ROUND_UP(Segment->Length.QuadPart); + for (Offset.QuadPart = 0; Offset.QuadPart < Length; Offset.QuadPart += PAGE_SIZE) { - PMM_IMAGE_SECTION_OBJECT ImageSectionObject; + Entry = MmGetPageEntrySectionSegment(Segment, &Offset); + if (Entry) + { + MmSetPageEntrySectionSegment(Segment, &Offset, 0); + if (IS_SWAP_FROM_SSE(Entry)) + { + MmFreeSwapPage(SWAPENTRY_FROM_SSE(Entry)); + } + else + { + Page = PFN_FROM_SSE(Entry); + SavedSwapEntry = MmGetSavedSwapEntryPage(Page); + if (SavedSwapEntry != 0) + { + MmSetSavedSwapEntryPage(Page, 0); + MmFreeSwapPage(SavedSwapEntry); + } + MmReleasePageMemoryConsumer(MC_USER, Page); + } + } + } + + MmUnlockSectionSegment(Segment); +} + +static +VOID +NTAPI +FreeSegmentPage(PMM_SECTION_SEGMENT Segment, PLARGE_INTEGER Offset) +{ + ULONG_PTR Entry; + PFN_NUMBER Page; + + MmLockSectionSegment(Segment); + + Entry = MmGetPageEntrySectionSegment(Segment, Offset); + + MmUnlockSectionSegment(Segment); + + /* This must be either a valid entry or nothing */ + ASSERT(!IS_SWAP_FROM_SSE(Entry)); + + /* There should be no reference anymore */ + ASSERT(SHARE_COUNT_FROM_SSE(Entry) == 0); + + Page = PFN_FROM_SSE(Entry); + /* If there is a page, this must be because it's still dirty */ + ASSERT(Page != 0); + + /* Write the page */ + if (IS_DIRTY_SSE(Entry)) + MiWritePage(Segment, Offset->QuadPart, Page); + + MmReleasePageMemoryConsumer(MC_USER, Page); +} + +VOID +NTAPI +MmDereferenceSegment(PMM_SECTION_SEGMENT Segment) +{ + KIRQL OldIrql; + + /* Lock the PFN lock because we mess around with SectionObjectPointers */ + OldIrql = MiAcquirePfnLock(); + + if (InterlockedDecrement64(Segment->ReferenceCount) > 0) + { + /* Nothing to do yet */ + MiReleasePfnLock(OldIrql); + return; + } + + *Segment->Flags |= MM_SEGMENT_INDELETE; + + MiReleasePfnLock(OldIrql); + + /* Flush the segment */ + if (*Segment->Flags & MM_DATAFILE_SEGMENT) + { + /* Free the page table. This will flush any remaining dirty data */ + MmFreePageTablesSectionSegment(Segment, FreeSegmentPage); + + OldIrql = MiAcquirePfnLock(); + /* Delete the pointer on the file */ + ASSERT(Segment->FileObject->SectionObjectPointer->DataSectionObject == Segment); + Segment->FileObject->SectionObjectPointer->DataSectionObject = NULL; + MiReleasePfnLock(OldIrql); + ObDereferenceObject(Segment->FileObject); + + ExFreePoolWithTag(Segment, TAG_MM_SECTION_SEGMENT); + } + else + { + /* Most grotesque thing ever */ + PMM_IMAGE_SECTION_OBJECT ImageSectionObject = CONTAINING_RECORD(Segment->ReferenceCount, MM_IMAGE_SECTION_OBJECT, RefCount); PMM_SECTION_SEGMENT SectionSegments; ULONG NrSegments; ULONG i; - ImageSectionObject = (PMM_IMAGE_SECTION_OBJECT)FileObject->SectionObjectPointer->ImageSectionObject; + OldIrql = MiAcquirePfnLock(); + /* Delete the pointer on the file */ + ASSERT(ImageSectionObject->FileObject->SectionObjectPointer->ImageSectionObject == ImageSectionObject); + ImageSectionObject->FileObject->SectionObjectPointer->ImageSectionObject = NULL; + MiReleasePfnLock(OldIrql); + + ObDereferenceObject(ImageSectionObject->FileObject); + NrSegments = ImageSectionObject->NrSegments; SectionSegments = ImageSectionObject->Segments; for (i = 0; i < NrSegments; i++) { - if (SectionSegments[i].ReferenceCount != 0) + if (SectionSegments[i].Image.Characteristics & IMAGE_SCN_MEM_SHARED) { - DPRINT1("Image segment %lu still referenced (was %lu)\n", i, - SectionSegments[i].ReferenceCount); - KeBugCheck(MEMORY_MANAGEMENT); + MmpFreePageFileSegment(&SectionSegments[i]); } + MmFreePageTablesSectionSegment(&SectionSegments[i], NULL); } - ExFreePool(ImageSectionObject->Segments); - ExFreePool(ImageSectionObject); - FileObject->SectionObjectPointer->ImageSectionObject = NULL; - } - if (FileObject->SectionObjectPointer->DataSectionObject != NULL) - { - PMM_SECTION_SEGMENT Segment; - Segment = (PMM_SECTION_SEGMENT)FileObject->SectionObjectPointer-> - DataSectionObject; - - if (Segment->ReferenceCount != 0) - { - DPRINT1("Data segment still referenced\n"); - KeBugCheck(MEMORY_MANAGEMENT); - } - MmFreePageTablesSectionSegment(Segment, NULL); - ExFreePool(Segment); - FileObject->SectionObjectPointer->DataSectionObject = NULL; + ExFreePoolWithTag(ImageSectionObject->Segments, TAG_MM_SECTION_SEGMENT); + ExFreePoolWithTag(ImageSectionObject, TAG_MM_SECTION_SEGMENT); } } @@ -868,13 +1076,12 @@ MmSharePageEntrySectionSegment(PMM_SECTION_SEGMENT Segment, { KeBugCheck(MEMORY_MANAGEMENT); } - Entry = MAKE_SSE(PAGE_FROM_SSE(Entry), SHARE_COUNT_FROM_SSE(Entry) + 1); - MmSetPageEntrySectionSegment(Segment, Offset, Entry); + MmSetPageEntrySectionSegment(Segment, Offset, BUMPREF_SSE(Entry)); } BOOLEAN NTAPI -MmUnsharePageEntrySectionSegment(PROS_SECTION_OBJECT Section, +MmUnsharePageEntrySectionSegment(PMEMORY_AREA MemoryArea, PMM_SECTION_SEGMENT Segment, PLARGE_INTEGER Offset, BOOLEAN Dirty, @@ -882,7 +1089,8 @@ MmUnsharePageEntrySectionSegment(PROS_SECTION_OBJECT Section, ULONG_PTR *InEntry) { ULONG_PTR Entry = InEntry ? *InEntry : MmGetPageEntrySectionSegment(Segment, Offset); - BOOLEAN IsDirectMapped = FALSE; + PFN_NUMBER Page = PFN_FROM_SSE(Entry); + BOOLEAN IsDataMap = BooleanFlagOn(*Segment->Flags, MM_DATAFILE_SEGMENT); if (Entry == 0) { @@ -898,149 +1106,61 @@ MmUnsharePageEntrySectionSegment(PROS_SECTION_OBJECT Section, { KeBugCheck(MEMORY_MANAGEMENT); } - Entry = MAKE_SSE(PAGE_FROM_SSE(Entry), SHARE_COUNT_FROM_SSE(Entry) - 1); + Entry = DECREF_SSE(Entry); + if (Dirty) Entry = DIRTY_SSE(Entry); + + /* If we are paging-out, pruning the page for real will be taken care of in MmCheckDirtySegment */ + if ((SHARE_COUNT_FROM_SSE(Entry) > 0) || PageOut) + { + /* Update the page mapping in the segment and we're done */ + MmSetPageEntrySectionSegment(Segment, Offset, Entry); + return FALSE; + } + + /* We are pruning the last mapping on this page. See if we can keep it a bit more. */ + ASSERT(!PageOut); + + if (IsDataMap) + { + /* We can always keep memory in for data maps */ + MmSetPageEntrySectionSegment(Segment, Offset, Entry); + return FALSE; + } + + if (!BooleanFlagOn(Segment->Image.Characteristics, IMAGE_SCN_MEM_SHARED)) + { + /* So this must have been a read-only page. Keep it ! */ + ASSERT(Segment->WriteCopy); + ASSERT(!IS_DIRTY_SSE(Entry)); + ASSERT(MmGetSavedSwapEntryPage(Page) == 0); + MmSetPageEntrySectionSegment(Segment, Offset, Entry); + return FALSE; + } + /* - * If we reducing the share count of this entry to zero then set the entry - * to zero and tell the cache the page is no longer mapped. + * So this is a page for a shared section of a DLL. + * We can keep it if it is not dirty. */ - if (SHARE_COUNT_FROM_SSE(Entry) == 0) + SWAPENTRY SwapEntry = MmGetSavedSwapEntryPage(Page); + if ((SwapEntry == 0) && !IS_DIRTY_SSE(Entry)) { - PFILE_OBJECT FileObject; - SWAPENTRY SavedSwapEntry; - PFN_NUMBER Page; -#ifndef NEWCC - PROS_SHARED_CACHE_MAP SharedCacheMap; - BOOLEAN IsImageSection; - LARGE_INTEGER FileOffset; - - FileOffset.QuadPart = Offset->QuadPart + Segment->Image.FileOffset; - IsImageSection = Section->AllocationAttributes & SEC_IMAGE ? TRUE : FALSE; -#endif - - Page = PFN_FROM_SSE(Entry); - FileObject = Section->FileObject; - if (FileObject != NULL && - !(Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) - { - -#ifndef NEWCC - if ((FileOffset.QuadPart % PAGE_SIZE) == 0 && - (Offset->QuadPart + PAGE_SIZE <= Segment->RawLength.QuadPart || !IsImageSection)) - { - NTSTATUS Status; - SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; - IsDirectMapped = TRUE; -#ifndef NEWCC - Status = CcRosUnmapVacb(SharedCacheMap, FileOffset.QuadPart, Dirty); -#else - Status = STATUS_SUCCESS; -#endif - if (!NT_SUCCESS(Status)) - { - DPRINT1("CcRosUnmapVacb failed, status = %x\n", Status); - KeBugCheck(MEMORY_MANAGEMENT); - } - } -#endif - } - - SavedSwapEntry = MmGetSavedSwapEntryPage(Page); - if (SavedSwapEntry == 0) - { - if (!PageOut && - ((Segment->Flags & MM_PAGEFILE_SEGMENT) || - (Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED))) - { - /* - * FIXME: - * Try to page out this page and set the swap entry - * within the section segment. There exist no rmap entry - * for this page. The pager thread can't page out a - * page without a rmap entry. - */ - MmSetPageEntrySectionSegment(Segment, Offset, Entry); - if (InEntry) *InEntry = Entry; - MiSetPageEvent(NULL, NULL); - } - else - { - MmSetPageEntrySectionSegment(Segment, Offset, 0); - if (InEntry) *InEntry = 0; - MiSetPageEvent(NULL, NULL); - if (!IsDirectMapped) - { - MmReleasePageMemoryConsumer(MC_USER, Page); - } - } - } - else - { - if ((Segment->Flags & MM_PAGEFILE_SEGMENT) || - (Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) - { - if (!PageOut) - { - if (Dirty) - { - /* - * FIXME: - * We hold all locks. Nobody can do something with the current - * process and the current segment (also not within an other process). - */ - NTSTATUS Status; - Status = MmWriteToSwapPage(SavedSwapEntry, Page); - if (!NT_SUCCESS(Status)) - { - DPRINT1("MM: Failed to write to swap page (Status was 0x%.8X)\n", Status); - KeBugCheck(MEMORY_MANAGEMENT); - } - } - MmSetPageEntrySectionSegment(Segment, Offset, MAKE_SWAP_SSE(SavedSwapEntry)); - if (InEntry) *InEntry = MAKE_SWAP_SSE(SavedSwapEntry); - MmSetSavedSwapEntryPage(Page, 0); - MiSetPageEvent(NULL, NULL); - } - MmReleasePageMemoryConsumer(MC_USER, Page); - } - else - { - DPRINT1("Found a swapentry for a non private page in an image or data file sgment\n"); - KeBugCheck(MEMORY_MANAGEMENT); - } - } + MmSetPageEntrySectionSegment(Segment, Offset, Entry); + return FALSE; } - else + + /* No more processes are referencing this shared dirty page. Ditch it. */ + if (SwapEntry) { - if (InEntry) - *InEntry = Entry; - else - MmSetPageEntrySectionSegment(Segment, Offset, Entry); + MmSetSavedSwapEntryPage(Page, 0); + MmFreeSwapPage(SwapEntry); } - return(SHARE_COUNT_FROM_SSE(Entry) > 0); -} - -BOOLEAN MiIsPageFromCache(PMEMORY_AREA MemoryArea, - LONGLONG SegOffset) -{ -#ifndef NEWCC - if (!(MemoryArea->Data.SectionData.Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) - { - PROS_SHARED_CACHE_MAP SharedCacheMap; - PROS_VACB Vacb; - SharedCacheMap = MemoryArea->Data.SectionData.Section->FileObject->SectionObjectPointer->SharedCacheMap; - Vacb = CcRosLookupVacb(SharedCacheMap, SegOffset + MemoryArea->Data.SectionData.Segment->Image.FileOffset); - if (Vacb) - { - CcRosReleaseVacb(SharedCacheMap, Vacb, Vacb->Valid, FALSE, TRUE); - return TRUE; - } - } -#endif - return FALSE; + MmSetPageEntrySectionSegment(Segment, Offset, 0); + MmReleasePageMemoryConsumer(MC_USER, Page); + return TRUE; } +static NTSTATUS -NTAPI MiCopyFromUserPage(PFN_NUMBER DestPage, const VOID *SrcAddress) { PEPROCESS Process; @@ -1051,230 +1171,243 @@ MiCopyFromUserPage(PFN_NUMBER DestPage, const VOID *SrcAddress) DestAddress = MiMapPageInHyperSpace(Process, DestPage, &Irql); if (DestAddress == NULL) { - return(STATUS_NO_MEMORY); + return STATUS_NO_MEMORY; } ASSERT((ULONG_PTR)DestAddress % PAGE_SIZE == 0); ASSERT((ULONG_PTR)SrcAddress % PAGE_SIZE == 0); RtlCopyMemory(DestAddress, SrcAddress, PAGE_SIZE); MiUnmapPageInHyperSpace(Process, DestAddress, Irql); - return(STATUS_SUCCESS); + return STATUS_SUCCESS; } -#ifndef NEWCC +static NTSTATUS NTAPI -MiReadPage(PMEMORY_AREA MemoryArea, - LONGLONG SegOffset, - PPFN_NUMBER Page) -/* - * FUNCTION: Read a page for a section backed memory area. - * PARAMETERS: - * MemoryArea - Memory area to read the page for. - * Offset - Offset of the page to read. - * Page - Variable that receives a page contains the read data. - */ +MmMakeSegmentResident( + _In_ PMM_SECTION_SEGMENT Segment, + _In_ LONGLONG Offset, + _In_ ULONG Length, + _In_opt_ PLARGE_INTEGER ValidDataLength) { - LONGLONG BaseOffset; - LONGLONG FileOffset; - PVOID BaseAddress; - BOOLEAN UptoDate; - PROS_VACB Vacb; - PFILE_OBJECT FileObject; + /* Let's use a 64K granularity. */ + LONGLONG RangeStart, RangeEnd; NTSTATUS Status; - LONGLONG RawLength; - PROS_SHARED_CACHE_MAP SharedCacheMap; - BOOLEAN IsImageSection; - LONGLONG Length; + PFILE_OBJECT FileObject = Segment->FileObject; - FileObject = MemoryArea->Data.SectionData.Section->FileObject; - SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; - RawLength = MemoryArea->Data.SectionData.Segment->RawLength.QuadPart; - FileOffset = SegOffset + MemoryArea->Data.SectionData.Segment->Image.FileOffset; - IsImageSection = MemoryArea->Data.SectionData.Section->AllocationAttributes & SEC_IMAGE ? TRUE : FALSE; + /* Calculate our range, aligned on 64K if possible. */ + Status = RtlLongLongAdd(Offset, Length, &RangeEnd); + ASSERT(NT_SUCCESS(Status)); + if (!NT_SUCCESS(Status)) + return Status; - ASSERT(SharedCacheMap); + RangeStart = Offset - (Offset % _64K); + if (RangeEnd % _64K) + RangeEnd += _64K - (RangeEnd % _64K); - DPRINT("%S %I64x\n", FileObject->FileName.Buffer, FileOffset); - - /* - * If the file system is letting us go directly to the cache and the - * memory area was mapped at an offset in the file which is page aligned - * then get the related VACB. - */ - if (((FileOffset % PAGE_SIZE) == 0) && - ((SegOffset + PAGE_SIZE <= RawLength) || !IsImageSection) && - !(MemoryArea->Data.SectionData.Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) + /* Clamp if needed */ + if (!FlagOn(*Segment->Flags, MM_DATAFILE_SEGMENT)) { - - /* - * Get the related VACB; we use a lower level interface than - * filesystems do because it is safe for us to use an offset with an - * alignment less than the file system block size. - */ - Status = CcRosGetVacb(SharedCacheMap, - FileOffset, - &BaseOffset, - &BaseAddress, - &UptoDate, - &Vacb); - if (!NT_SUCCESS(Status)) - { - return(Status); - } - if (!UptoDate) - { - /* - * If the VACB isn't up to date then call the file - * system to read in the data. - */ - Status = CcReadVirtualAddress(Vacb); - if (!NT_SUCCESS(Status)) - { - CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE, FALSE); - return Status; - } - } - - /* Probe the page, since it's PDE might not be synced */ - (void)*((volatile char*)BaseAddress + FileOffset - BaseOffset); - - /* - * Retrieve the page from the view that we actually want. - */ - (*Page) = MmGetPhysicalAddress((char*)BaseAddress + - FileOffset - BaseOffset).LowPart >> PAGE_SHIFT; - - CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, TRUE); + if (RangeEnd > Segment->RawLength.QuadPart) + RangeEnd = Segment->RawLength.QuadPart; } - else + + /* Let's gooooooooo */ + for ( ; RangeStart < RangeEnd; RangeStart += _64K) { - PEPROCESS Process; - KIRQL Irql; - PVOID PageAddr; - LONGLONG VacbOffset; + /* First take a look at where we miss pages */ + ULONG ToReadPageBits = 0; + LONGLONG ChunkEnd = RangeStart + _64K; - /* - * Allocate a page, this is rather complicated by the possibility - * we might have to move other things out of memory - */ - MI_SET_USAGE(MI_USAGE_SECTION); - MI_SET_PROCESS2(PsGetCurrentProcess()->ImageFileName); - Status = MmRequestPageMemoryConsumer(MC_USER, TRUE, Page); - if (!NT_SUCCESS(Status)) + if (ChunkEnd > RangeEnd) + ChunkEnd = RangeEnd; + + MmLockSectionSegment(Segment); + for (LONGLONG ChunkOffset = RangeStart; ChunkOffset < ChunkEnd; ChunkOffset += PAGE_SIZE) { - return(Status); - } - Status = CcRosGetVacb(SharedCacheMap, - FileOffset, - &BaseOffset, - &BaseAddress, - &UptoDate, - &Vacb); - if (!NT_SUCCESS(Status)) - { - return(Status); - } - if (!UptoDate) - { - /* - * If the VACB isn't up to date then call the file - * system to read in the data. - */ - Status = CcReadVirtualAddress(Vacb); - if (!NT_SUCCESS(Status)) + LARGE_INTEGER CurrentOffset; + + CurrentOffset.QuadPart = ChunkOffset; + ULONG_PTR Entry = MmGetPageEntrySectionSegment(Segment, &CurrentOffset); + + /* Let any pending read proceed */ + while (MM_IS_WAIT_PTE(Entry)) { - CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE, FALSE); - return Status; + MmUnlockSectionSegment(Segment); + + KeDelayExecutionThread(KernelMode, FALSE, &TinyTime); + + MmLockSectionSegment(Segment); + Entry = MmGetPageEntrySectionSegment(Segment, &CurrentOffset); } + + if (Entry != 0) + { + /* There is a page here. Or a swap entry. Or whatever... */ + continue; + } + + ToReadPageBits |= 1UL << ((ChunkOffset - RangeStart) >> PAGE_SHIFT); + + /* Put a wait entry here */ + MmSetPageEntrySectionSegment(Segment, &CurrentOffset, MAKE_SWAP_SSE(MM_WAIT_ENTRY)); + } + MmUnlockSectionSegment(Segment); + + if (ToReadPageBits == 0) + { + /* Nothing to do for this chunk */ + continue; } - Process = PsGetCurrentProcess(); - PageAddr = MiMapPageInHyperSpace(Process, *Page, &Irql); - VacbOffset = BaseOffset + VACB_MAPPING_GRANULARITY - FileOffset; - Length = RawLength - SegOffset; - if (Length <= VacbOffset && Length <= PAGE_SIZE) + /* Now perform the actual read */ + LONGLONG ChunkOffset = RangeStart; + while (ChunkOffset < ChunkEnd) { - memcpy(PageAddr, (char*)BaseAddress + FileOffset - BaseOffset, Length); - } - else if (VacbOffset >= PAGE_SIZE) - { - memcpy(PageAddr, (char*)BaseAddress + FileOffset - BaseOffset, PAGE_SIZE); - } - else - { - memcpy(PageAddr, (char*)BaseAddress + FileOffset - BaseOffset, VacbOffset); - MiUnmapPageInHyperSpace(Process, PageAddr, Irql); - CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); - Status = CcRosGetVacb(SharedCacheMap, - FileOffset + VacbOffset, - &BaseOffset, - &BaseAddress, - &UptoDate, - &Vacb); - if (!NT_SUCCESS(Status)) + /* Move forward if there is a hole */ + ULONG BitSet; + if (!_BitScanForward(&BitSet, ToReadPageBits)) { - return(Status); + /* Nothing more to read */ + break; } - if (!UptoDate) + ToReadPageBits >>= BitSet; + ChunkOffset += BitSet * PAGE_SIZE; + ASSERT(ChunkOffset < ChunkEnd); + + /* Get the range we have to read */ + _BitScanForward(&BitSet, ~ToReadPageBits); + ULONG ReadLength = BitSet * PAGE_SIZE; + + ASSERT(ReadLength <= _64K); + + /* Clamp (This is for image mappings */ + if ((ChunkOffset + ReadLength) > ChunkEnd) + ReadLength = ChunkEnd - ChunkOffset; + + ASSERT(ReadLength != 0); + + /* Allocate a MDL */ + PMDL Mdl = IoAllocateMdl(NULL, ReadLength, FALSE, FALSE, NULL); + if (!Mdl) { - /* - * If the VACB isn't up to date then call the file - * system to read in the data. - */ - Status = CcReadVirtualAddress(Vacb); - if (!NT_SUCCESS(Status)) + /* Damn. Roll-back. */ + MmLockSectionSegment(Segment); + while (ChunkOffset < ChunkEnd) { - CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE, FALSE); - return Status; + if (ToReadPageBits & 1) + { + LARGE_INTEGER CurrentOffset; + CurrentOffset.QuadPart = ChunkOffset; + ASSERT(MM_IS_WAIT_PTE(MmGetPageEntrySectionSegment(Segment, &CurrentOffset))); + MmSetPageEntrySectionSegment(Segment, &CurrentOffset, 0); + } + ToReadPageBits >>= 1; + ChunkOffset += PAGE_SIZE; } + MmUnlockSectionSegment(Segment); + return STATUS_INSUFFICIENT_RESOURCES; } - PageAddr = MiMapPageInHyperSpace(Process, *Page, &Irql); - if (Length < PAGE_SIZE) + + /* Get our pages */ + PPFN_NUMBER Pages = MmGetMdlPfnArray(Mdl); + RtlZeroMemory(Pages, BYTES_TO_PAGES(ReadLength) * sizeof(PFN_NUMBER)); + for (UINT i = 0; i < BYTES_TO_PAGES(ReadLength); i++) { - memcpy((char*)PageAddr + VacbOffset, BaseAddress, Length - VacbOffset); + /* MmRequestPageMemoryConsumer succeeds or bugchecks */ + (void)MmRequestPageMemoryConsumer(MC_USER, FALSE, &Pages[i]); } - else + Mdl->MdlFlags |= MDL_PAGES_LOCKED | MDL_IO_PAGE_READ; + + LARGE_INTEGER FileOffset; + FileOffset.QuadPart = Segment->Image.FileOffset + ChunkOffset; + + /* Clamp to VDL */ + if (ValidDataLength && ((FileOffset.QuadPart + ReadLength) > ValidDataLength->QuadPart)) { - memcpy((char*)PageAddr + VacbOffset, BaseAddress, PAGE_SIZE - VacbOffset); + if (FileOffset.QuadPart > ValidDataLength->QuadPart) + { + /* Great, nothing to read. */ + goto AssignPagesToSegment; + } + + Mdl->Size = (FileOffset.QuadPart + ReadLength) - ValidDataLength->QuadPart; } + + KEVENT Event; + KeInitializeEvent(&Event, NotificationEvent, FALSE); + + /* Disable APCs */ + KIRQL OldIrql; + KeRaiseIrql(APC_LEVEL, &OldIrql); + + IO_STATUS_BLOCK Iosb; + Status = IoPageRead(FileObject, Mdl, &FileOffset, &Event, &Iosb); + if (Status == STATUS_PENDING) + { + KeWaitForSingleObject(&Event, WrPageIn, KernelMode, FALSE, NULL); + Status = Iosb.Status; + } + + if (Mdl->MdlFlags & MDL_MAPPED_TO_SYSTEM_VA) + { + MmUnmapLockedPages(Mdl->MappedSystemVa, Mdl); + } + + KeLowerIrql(OldIrql); + + if (Status == STATUS_END_OF_FILE) + { + DPRINT1("Got STATUS_END_OF_FILE at offset %I64d for file %wZ.\n", FileOffset.QuadPart, &FileObject->FileName); + Status = STATUS_SUCCESS; + } + + if (!NT_SUCCESS(Status)) + { + /* Damn. Roll back. */ + for (UINT i = 0; i < BYTES_TO_PAGES(ReadLength); i++) + MmReleasePageMemoryConsumer(MC_USER, Pages[i]); + + MmLockSectionSegment(Segment); + while (ChunkOffset < ChunkEnd) + { + if (ToReadPageBits & 1) + { + LARGE_INTEGER CurrentOffset; + CurrentOffset.QuadPart = ChunkOffset; + ASSERT(MM_IS_WAIT_PTE(MmGetPageEntrySectionSegment(Segment, &CurrentOffset))); + MmSetPageEntrySectionSegment(Segment, &CurrentOffset, 0); + } + ToReadPageBits >>= 1; + ChunkOffset += PAGE_SIZE; + } + MmUnlockSectionSegment(Segment); + IoFreeMdl(Mdl);; + return Status; + } + +AssignPagesToSegment: + MmLockSectionSegment(Segment); + + for (UINT i = 0; i < BYTES_TO_PAGES(ReadLength); i++) + { + LARGE_INTEGER CurrentOffset; + CurrentOffset.QuadPart = ChunkOffset + (i * PAGE_SIZE); + + ASSERT(MM_IS_WAIT_PTE(MmGetPageEntrySectionSegment(Segment, &CurrentOffset))); + + MmSetPageEntrySectionSegment(Segment, &CurrentOffset, MAKE_SSE(Pages[i] << PAGE_SHIFT, 0)); + } + + MmUnlockSectionSegment(Segment); + + IoFreeMdl(Mdl); + ToReadPageBits >>= BitSet; + ChunkOffset += BitSet * PAGE_SIZE; } - MiUnmapPageInHyperSpace(Process, PageAddr, Irql); - CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); } - return(STATUS_SUCCESS); + + return STATUS_SUCCESS; } -#else -NTSTATUS -NTAPI -MiReadPage(PMEMORY_AREA MemoryArea, - LONGLONG SegOffset, - PPFN_NUMBER Page) -/* - * FUNCTION: Read a page for a section backed memory area. - * PARAMETERS: - * MemoryArea - Memory area to read the page for. - * Offset - Offset of the page to read. - * Page - Variable that receives a page contains the read data. - */ -{ - MM_REQUIRED_RESOURCES Resources; - NTSTATUS Status; - - RtlZeroMemory(&Resources, sizeof(MM_REQUIRED_RESOURCES)); - - Resources.Context = MemoryArea->Data.SectionData.Section->FileObject; - Resources.FileOffset.QuadPart = SegOffset + - MemoryArea->Data.SectionData.Segment->Image.FileOffset; - Resources.Consumer = MC_USER; - Resources.Amount = PAGE_SIZE; - - DPRINT("%S, offset 0x%x, len 0x%x, page 0x%x\n", ((PFILE_OBJECT)Resources.Context)->FileName.Buffer, Resources.FileOffset.LowPart, Resources.Amount, Resources.Page[0]); - - Status = MiReadFilePage(MmGetKernelAddressSpace(), MemoryArea, &Resources); - *Page = Resources.Page[0]; - return Status; -} -#endif static VOID MmAlterViewAttributes(PMMSUPPORT AddressSpace, @@ -1293,7 +1426,7 @@ MmAlterViewAttributes(PMMSUPPORT AddressSpace, MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace, BaseAddress); ASSERT(MemoryArea != NULL); - Segment = MemoryArea->Data.SectionData.Segment; + Segment = MemoryArea->SectionData.Segment; MmLockSectionSegment(Segment); if ((Segment->WriteCopy) && @@ -1331,7 +1464,7 @@ MmAlterViewAttributes(PMMSUPPORT AddressSpace, PFN_NUMBER Page; Offset.QuadPart = (ULONG_PTR)Address - MA_GetStartingAddress(MemoryArea) - + MemoryArea->Data.SectionData.ViewOffset.QuadPart; + + MemoryArea->SectionData.ViewOffset; Entry = MmGetPageEntrySectionSegment(Segment, &Offset); /* * An MM_WAIT_ENTRY is ok in this case... It'll just count as @@ -1367,7 +1500,6 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, LARGE_INTEGER Offset; PFN_NUMBER Page; NTSTATUS Status; - PROS_SECTION_OBJECT Section; PMM_SECTION_SEGMENT Segment; ULONG_PTR Entry; ULONG_PTR Entry1; @@ -1385,12 +1517,12 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, */ if (MmIsPagePresent(Process, Address)) { - return(STATUS_SUCCESS); + return STATUS_SUCCESS; } if (MmIsDisabledPage(Process, Address)) { - return(STATUS_ACCESS_VIOLATION); + return STATUS_ACCESS_VIOLATION; } /* @@ -1398,17 +1530,16 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, */ if (MemoryArea->DeleteInProgress) { - return(STATUS_UNSUCCESSFUL); + return STATUS_UNSUCCESSFUL; } PAddress = MM_ROUND_DOWN(Address, PAGE_SIZE); Offset.QuadPart = (ULONG_PTR)PAddress - MA_GetStartingAddress(MemoryArea) - + MemoryArea->Data.SectionData.ViewOffset.QuadPart; + + MemoryArea->SectionData.ViewOffset; - Segment = MemoryArea->Data.SectionData.Segment; - Section = MemoryArea->Data.SectionData.Section; + Segment = MemoryArea->SectionData.Segment; Region = MmFindRegion((PVOID)MA_GetStartingAddress(MemoryArea), - &MemoryArea->Data.SectionData.RegionListHead, + &MemoryArea->SectionData.RegionListHead, Address, NULL); ASSERT(Region != NULL); @@ -1422,7 +1553,7 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, { /* Remove it */ Status = MmAlterRegion(AddressSpace, (PVOID)MA_GetStartingAddress(MemoryArea), - &MemoryArea->Data.SectionData.RegionListHead, + &MemoryArea->SectionData.RegionListHead, Address, PAGE_SIZE, Region->Type, Region->Protect & ~PAGE_GUARD, MmAlterViewAttributes); @@ -1464,7 +1595,7 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, MiWaitForPageEvent(NULL, NULL); MmLockAddressSpace(AddressSpace); DPRINT("Address 0x%p\n", Address); - return(STATUS_MM_RESTART_OPERATION); + return STATUS_MM_RESTART_OPERATION; } HasSwapEntry = MmIsPageSwapEntry(Process, Address); @@ -1474,37 +1605,24 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, { SWAPENTRY DummyEntry; - /* - * Is it a wait entry? - */ - if (HasSwapEntry) + MmGetPageFileMapping(Process, Address, &SwapEntry); + if (SwapEntry == MM_WAIT_ENTRY) { - MmGetPageFileMapping(Process, Address, &SwapEntry); - - if (SwapEntry == MM_WAIT_ENTRY) - { - MmUnlockSectionSegment(Segment); - MmUnlockAddressSpace(AddressSpace); - MiWaitForPageEvent(NULL, NULL); - MmLockAddressSpace(AddressSpace); - return STATUS_MM_RESTART_OPERATION; - } - - /* - * Must be private page we have swapped out. - */ - - /* - * Sanity check - */ - if (Segment->Flags & MM_PAGEFILE_SEGMENT) - { - DPRINT1("Found a swaped out private page in a pagefile section.\n"); - KeBugCheck(MEMORY_MANAGEMENT); - } - MmDeletePageFileMapping(Process, Address, &SwapEntry); + MmUnlockSectionSegment(Segment); + MmUnlockAddressSpace(AddressSpace); + MiWaitForPageEvent(NULL, NULL); + MmLockAddressSpace(AddressSpace); + return STATUS_MM_RESTART_OPERATION; } + /* + * Must be private page we have swapped out. + */ + + /* + * Sanity check + */ + MmDeletePageFileMapping(Process, Address, &SwapEntry); MmUnlockSectionSegment(Segment); /* Tell everyone else we are serving the fault. */ @@ -1541,7 +1659,7 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, { DPRINT("MmCreateVirtualMapping failed, not out of memory\n"); KeBugCheck(MEMORY_MANAGEMENT); - return(Status); + return Status; } /* @@ -1553,19 +1671,19 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, /* * Add the page to the process's working set */ - MmInsertRmap(Page, Process, Address); + if (Process) MmInsertRmap(Page, Process, Address); /* * Finish the operation */ MiSetPageEvent(Process, Address); DPRINT("Address 0x%p\n", Address); - return(STATUS_SUCCESS); + return STATUS_SUCCESS; } /* * Satisfying a page fault on a map of /Device/PhysicalMemory is easy */ - if (Section->AllocationAttributes & SEC_PHYSICALMEMORY) + if ((*Segment->Flags) & MM_PHYSICALMEMORY_SEGMENT) { MmUnlockSectionSegment(Segment); /* @@ -1581,7 +1699,7 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, { DPRINT("MmCreateVirtualMappingUnsafe failed, not out of memory\n"); KeBugCheck(MEMORY_MANAGEMENT); - return(Status); + return Status; } /* @@ -1589,7 +1707,7 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, */ MiSetPageEvent(Process, Address); DPRINT("Address 0x%p\n", Address); - return(STATUS_SUCCESS); + return STATUS_SUCCESS; } /* @@ -1599,85 +1717,57 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, if (Entry == 0) { - SWAPENTRY FakeSwapEntry; - /* - * If the entry is zero (and it can't change because we have - * locked the segment) then we need to load the page. + * If the entry is zero, then we need to load the page. */ - - /* - * Release all our locks and read in the page from disk - */ - MmSetPageEntrySectionSegment(Segment, &Offset, MAKE_SWAP_SSE(MM_WAIT_ENTRY)); - MmUnlockSectionSegment(Segment); - MmCreatePageFileMapping(Process, PAddress, MM_WAIT_ENTRY); - MmUnlockAddressSpace(AddressSpace); - - if ((Segment->Flags & MM_PAGEFILE_SEGMENT) || - ((Offset.QuadPart >= (LONGLONG)PAGE_ROUND_UP(Segment->RawLength.QuadPart) && - (Section->AllocationAttributes & SEC_IMAGE)))) + if ((Offset.QuadPart >= (LONGLONG)PAGE_ROUND_UP(Segment->RawLength.QuadPart)) && (MemoryArea->VadNode.u.VadFlags.VadType == VadImageMap)) { + /* We are beyond the data which is on file. Just get a new page. */ MI_SET_USAGE(MI_USAGE_SECTION); if (Process) MI_SET_PROCESS2(Process->ImageFileName); if (!Process) MI_SET_PROCESS2("Kernel Section"); - Status = MmRequestPageMemoryConsumer(MC_USER, TRUE, &Page); - if (!NT_SUCCESS(Status)) - { - DPRINT1("MmRequestPageMemoryConsumer failed (Status %x)\n", Status); - } + MmRequestPageMemoryConsumer(MC_USER, FALSE, &Page); + MmSetPageEntrySectionSegment(Segment, &Offset, MAKE_SSE(Page << PAGE_SHIFT, 1)); + MmUnlockSectionSegment(Segment); - } - else - { - Status = MiReadPage(MemoryArea, Offset.QuadPart, &Page); + Status = MmCreateVirtualMapping(Process, PAddress, Attributes, &Page, 1); if (!NT_SUCCESS(Status)) { - DPRINT1("MiReadPage failed (Status %x)\n", Status); + DPRINT1("Unable to create virtual mapping\n"); + KeBugCheck(MEMORY_MANAGEMENT); } - } - if (!NT_SUCCESS(Status)) - { - /* - * FIXME: What do we know in this case? - */ - /* - * Cleanup and release locks - */ - MmLockAddressSpace(AddressSpace); + ASSERT(MmIsPagePresent(Process, PAddress)); + if (Process) + MmInsertRmap(Page, Process, Address); + MiSetPageEvent(Process, Address); DPRINT("Address 0x%p\n", Address); - return(Status); + return STATUS_SUCCESS; } - /* Lock both segment and process address space while we proceed. */ - MmLockAddressSpace(AddressSpace); - MmLockSectionSegment(Segment); + MmUnlockSectionSegment(Segment); + MmUnlockAddressSpace(AddressSpace); - MmDeletePageFileMapping(Process, PAddress, &FakeSwapEntry); - DPRINT("CreateVirtualMapping Page %x Process %p PAddress %p Attributes %x\n", - Page, Process, PAddress, Attributes); - Status = MmCreateVirtualMapping(Process, - PAddress, - Attributes, - &Page, - 1); + /* The data must be paged in. Lock the file, so that the VDL doesn't get updated behind us. */ + FsRtlAcquireFileExclusive(Segment->FileObject); + + PFSRTL_COMMON_FCB_HEADER FcbHeader = Segment->FileObject->FsContext; + + Status = MmMakeSegmentResident(Segment, Offset.QuadPart, PAGE_SIZE, &FcbHeader->ValidDataLength); + + FsRtlReleaseFile(Segment->FileObject); + + /* Lock address space again */ + MmLockAddressSpace(AddressSpace); if (!NT_SUCCESS(Status)) { - DPRINT1("Unable to create virtual mapping\n"); - KeBugCheck(MEMORY_MANAGEMENT); + /* Damn */ + DPRINT1("Failed to page data in!\n"); + return STATUS_IN_PAGE_ERROR; } - ASSERT(MmIsPagePresent(Process, PAddress)); - MmInsertRmap(Page, Process, Address); - /* Set this section offset has being backed by our new page. */ - Entry = MAKE_SSE(Page << PAGE_SHIFT, 1); - MmSetPageEntrySectionSegment(Segment, &Offset, Entry); - MmUnlockSectionSegment(Segment); - - MiSetPageEvent(Process, Address); - DPRINT("Address 0x%p\n", Address); - return(STATUS_SUCCESS); + /* Everything went fine. Restart the operation */ + return STATUS_MM_RESTART_OPERATION; } else if (IS_SWAP_FROM_SSE(Entry)) { @@ -1741,7 +1831,7 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, /* Map the page into the process address space */ Status = MmCreateVirtualMapping(Process, PAddress, - Region->Protect, + Attributes, &Page, 1); if (!NT_SUCCESS(Status)) @@ -1749,7 +1839,8 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, DPRINT1("Unable to create virtual mapping\n"); KeBugCheck(MEMORY_MANAGEMENT); } - MmInsertRmap(Page, Process, Address); + if (Process) + MmInsertRmap(Page, Process, Address); /* * Mark the offset within the section as having valid, in-memory @@ -1761,7 +1852,7 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, MiSetPageEvent(Process, Address); DPRINT("Address 0x%p\n", Address); - return(STATUS_SUCCESS); + return STATUS_SUCCESS; } else { @@ -1778,7 +1869,9 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, DPRINT1("Unable to create virtual mapping\n"); KeBugCheck(MEMORY_MANAGEMENT); } - MmInsertRmap(Page, Process, Address); + + if (Process) + MmInsertRmap(Page, Process, Address); /* Take a reference on it */ MmSharePageEntrySectionSegment(Segment, &Offset); @@ -1786,7 +1879,7 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, MiSetPageEvent(Process, Address); DPRINT("Address 0x%p\n", Address); - return(STATUS_SUCCESS); + return STATUS_SUCCESS; } } @@ -1797,7 +1890,6 @@ MmAccessFaultSectionView(PMMSUPPORT AddressSpace, PVOID Address) { PMM_SECTION_SEGMENT Segment; - PROS_SECTION_OBJECT Section; PFN_NUMBER OldPage; PFN_NUMBER NewPage; NTSTATUS Status; @@ -1823,7 +1915,7 @@ MmAccessFaultSectionView(PMMSUPPORT AddressSpace, if (MmGetPageProtect(Process, Address) & PAGE_READWRITE) { DPRINT("Address 0x%p\n", Address); - return(STATUS_SUCCESS); + return STATUS_SUCCESS; } /* @@ -1831,12 +1923,11 @@ MmAccessFaultSectionView(PMMSUPPORT AddressSpace, */ PAddress = MM_ROUND_DOWN(Address, PAGE_SIZE); Offset.QuadPart = (ULONG_PTR)PAddress - MA_GetStartingAddress(MemoryArea) - + MemoryArea->Data.SectionData.ViewOffset.QuadPart; + + MemoryArea->SectionData.ViewOffset; - Segment = MemoryArea->Data.SectionData.Segment; - Section = MemoryArea->Data.SectionData.Section; + Segment = MemoryArea->SectionData.Segment; Region = MmFindRegion((PVOID)MA_GetStartingAddress(MemoryArea), - &MemoryArea->Data.SectionData.RegionListHead, + &MemoryArea->SectionData.RegionListHead, Address, NULL); ASSERT(Region != NULL); @@ -1848,7 +1939,7 @@ MmAccessFaultSectionView(PMMSUPPORT AddressSpace, Region->Protect == PAGE_EXECUTE_READWRITE))) { DPRINT("Address 0x%p\n", Address); - return(STATUS_ACCESS_VIOLATION); + return STATUS_ACCESS_VIOLATION; } /* Get the page mapping this section offset. */ @@ -1866,7 +1957,7 @@ MmAccessFaultSectionView(PMMSUPPORT AddressSpace, MmUnlockSectionSegment(Segment); /* This is a private page. We must only change the page protection. */ MmSetPageProtect(Process, PAddress, Region->Protect); - return(STATUS_SUCCESS); + return STATUS_SUCCESS; } /* @@ -1891,8 +1982,9 @@ MmAccessFaultSectionView(PMMSUPPORT AddressSpace, */ DPRINT("Swapping page (Old %x New %x)\n", OldPage, NewPage); MmDeleteVirtualMapping(Process, PAddress, NULL, NULL); - MmDeleteRmap(OldPage, Process, PAddress); - MmUnsharePageEntrySectionSegment(Section, Segment, &Offset, FALSE, FALSE, NULL); + if (Process) + MmDeleteRmap(OldPage, Process, PAddress); + MmUnsharePageEntrySectionSegment(MemoryArea, Segment, &Offset, FALSE, FALSE, NULL); MmUnlockSectionSegment(Segment); /* @@ -1907,615 +1999,15 @@ MmAccessFaultSectionView(PMMSUPPORT AddressSpace, { DPRINT1("MmCreateVirtualMapping failed, unable to create virtual mapping, not out of memory\n"); KeBugCheck(MEMORY_MANAGEMENT); - return(Status); + return Status; } - MmInsertRmap(NewPage, Process, PAddress); + + if (Process) + MmInsertRmap(NewPage, Process, PAddress); MiSetPageEvent(Process, Address); DPRINT("Address 0x%p\n", Address); - return(STATUS_SUCCESS); -} - -VOID -MmPageOutDeleteMapping(PVOID Context, PEPROCESS Process, PVOID Address) -{ - MM_SECTION_PAGEOUT_CONTEXT* PageOutContext; - BOOLEAN WasDirty; - PFN_NUMBER Page = 0; - - PageOutContext = (MM_SECTION_PAGEOUT_CONTEXT*)Context; - if (Process) - { - MmLockAddressSpace(&Process->Vm); - } - - MmDeleteVirtualMapping(Process, - Address, - &WasDirty, - &Page); - if (WasDirty) - { - PageOutContext->WasDirty = TRUE; - } - if (!PageOutContext->Private) - { - MmLockSectionSegment(PageOutContext->Segment); - MmUnsharePageEntrySectionSegment((PROS_SECTION_OBJECT)PageOutContext->Section, - PageOutContext->Segment, - &PageOutContext->Offset, - PageOutContext->WasDirty, - TRUE, - &PageOutContext->SectionEntry); - MmUnlockSectionSegment(PageOutContext->Segment); - } - if (Process) - { - MmUnlockAddressSpace(&Process->Vm); - } - - if (PageOutContext->Private) - { - MmReleasePageMemoryConsumer(MC_USER, Page); - } -} - -NTSTATUS -NTAPI -MmPageOutSectionView(PMMSUPPORT AddressSpace, - MEMORY_AREA* MemoryArea, - PVOID Address, ULONG_PTR Entry) -{ - PFN_NUMBER Page; - MM_SECTION_PAGEOUT_CONTEXT Context; - SWAPENTRY SwapEntry; - NTSTATUS Status; -#ifndef NEWCC - ULONGLONG FileOffset; - PFILE_OBJECT FileObject; - PROS_SHARED_CACHE_MAP SharedCacheMap = NULL; - BOOLEAN IsImageSection; -#endif - BOOLEAN DirectMapped; - PEPROCESS Process = MmGetAddressSpaceOwner(AddressSpace); - KIRQL OldIrql; - - Address = (PVOID)PAGE_ROUND_DOWN(Address); - - /* - * Get the segment and section. - */ - Context.Segment = MemoryArea->Data.SectionData.Segment; - Context.Section = MemoryArea->Data.SectionData.Section; - Context.SectionEntry = Entry; - Context.CallingProcess = Process; - - Context.Offset.QuadPart = (ULONG_PTR)Address - MA_GetStartingAddress(MemoryArea) - + MemoryArea->Data.SectionData.ViewOffset.QuadPart; - - DirectMapped = FALSE; - - MmLockSectionSegment(Context.Segment); - -#ifndef NEWCC - FileOffset = Context.Offset.QuadPart + Context.Segment->Image.FileOffset; - IsImageSection = Context.Section->AllocationAttributes & SEC_IMAGE ? TRUE : FALSE; - FileObject = Context.Section->FileObject; - - if (FileObject != NULL && - !(Context.Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) - { - SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; - - /* - * If the file system is letting us go directly to the cache and the - * memory area was mapped at an offset in the file which is page aligned - * then note this is a direct mapped page. - */ - if ((FileOffset % PAGE_SIZE) == 0 && - (Context.Offset.QuadPart + PAGE_SIZE <= Context.Segment->RawLength.QuadPart || !IsImageSection)) - { - DirectMapped = TRUE; - } - } -#endif - - - /* - * This should never happen since mappings of physical memory are never - * placed in the rmap lists. - */ - if (Context.Section->AllocationAttributes & SEC_PHYSICALMEMORY) - { - DPRINT1("Trying to page out from physical memory section address 0x%p " - "process %p\n", Address, - Process ? Process->UniqueProcessId : 0); - KeBugCheck(MEMORY_MANAGEMENT); - } - - /* - * Get the section segment entry and the physical address. - */ - if (!MmIsPagePresent(Process, Address)) - { - DPRINT1("Trying to page out not-present page at (%p,0x%p).\n", - Process ? Process->UniqueProcessId : 0, Address); - KeBugCheck(MEMORY_MANAGEMENT); - } - Page = MmGetPfnForProcess(Process, Address); - SwapEntry = MmGetSavedSwapEntryPage(Page); - - /* - * Check the reference count to ensure this page can be paged out - */ - if (MmGetReferenceCountPage(Page) != 1) - { - DPRINT("Cannot page out locked section page: 0x%lu (RefCount: %lu)\n", - Page, MmGetReferenceCountPage(Page)); - MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, Entry); - MmUnlockSectionSegment(Context.Segment); - return STATUS_UNSUCCESSFUL; - } - - /* - * Prepare the context structure for the rmap delete call. - */ - MmUnlockSectionSegment(Context.Segment); - Context.WasDirty = FALSE; - if (IS_SWAP_FROM_SSE(Entry) || PFN_FROM_SSE(Entry) != Page) - { - Context.Private = TRUE; - } - else - { - Context.Private = FALSE; - } - - /* - * Take an additional reference to the page or the VACB. - */ - if (DirectMapped && !Context.Private) - { - if(!MiIsPageFromCache(MemoryArea, Context.Offset.QuadPart)) - { - DPRINT1("Direct mapped non private page is not associated with the cache.\n"); - KeBugCheck(MEMORY_MANAGEMENT); - } - } - else - { - OldIrql = MiAcquirePfnLock(); - MmReferencePage(Page); - MiReleasePfnLock(OldIrql); - } - - MmDeleteAllRmaps(Page, (PVOID)&Context, MmPageOutDeleteMapping); - - /* Since we passed in a surrogate, we'll get back the page entry - * state in our context. This is intended to make intermediate - * decrements of share count not release the wait entry. - */ - Entry = Context.SectionEntry; - - /* - * If this wasn't a private page then we should have reduced the entry to - * zero by deleting all the rmaps. - */ - if (!Context.Private && Entry != 0) - { - if (!(Context.Segment->Flags & MM_PAGEFILE_SEGMENT) && - !(Context.Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) - { - KeBugCheckEx(MEMORY_MANAGEMENT, Entry, (ULONG_PTR)Process, (ULONG_PTR)Address, 0); - } - } - - /* - * If the page wasn't dirty then we can just free it as for a readonly page. - * Since we unmapped all the mappings above we know it will not suddenly - * become dirty. - * If the page is from a pagefile section and has no swap entry, - * we can't free the page at this point. - */ - SwapEntry = MmGetSavedSwapEntryPage(Page); - if (Context.Segment->Flags & MM_PAGEFILE_SEGMENT) - { - if (Context.Private) - { - DPRINT1("Found a %s private page (address %p) in a pagefile segment.\n", - Context.WasDirty ? "dirty" : "clean", Address); - KeBugCheckEx(MEMORY_MANAGEMENT, SwapEntry, (ULONG_PTR)Process, (ULONG_PTR)Address, 0); - } - if (!Context.WasDirty && SwapEntry != 0) - { - MmSetSavedSwapEntryPage(Page, 0); - MmLockSectionSegment(Context.Segment); - MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, MAKE_SWAP_SSE(SwapEntry)); - MmUnlockSectionSegment(Context.Segment); - MmReleasePageMemoryConsumer(MC_USER, Page); - MiSetPageEvent(NULL, NULL); - return(STATUS_SUCCESS); - } - } - else if (Context.Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED) - { - if (Context.Private) - { - DPRINT1("Found a %s private page (address %p) in a shared section segment.\n", - Context.WasDirty ? "dirty" : "clean", Address); - KeBugCheckEx(MEMORY_MANAGEMENT, Page, (ULONG_PTR)Process, (ULONG_PTR)Address, 0); - } - if (!Context.WasDirty || SwapEntry != 0) - { - MmSetSavedSwapEntryPage(Page, 0); - if (SwapEntry != 0) - { - MmLockSectionSegment(Context.Segment); - MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, MAKE_SWAP_SSE(SwapEntry)); - MmUnlockSectionSegment(Context.Segment); - } - MmReleasePageMemoryConsumer(MC_USER, Page); - MiSetPageEvent(NULL, NULL); - return(STATUS_SUCCESS); - } - } - else if (!Context.Private && DirectMapped) - { - if (SwapEntry != 0) - { - DPRINT1("Found a swapentry for a non private and direct mapped page (address %p)\n", - Address); - KeBugCheckEx(MEMORY_MANAGEMENT, STATUS_UNSUCCESSFUL, SwapEntry, (ULONG_PTR)Process, (ULONG_PTR)Address); - } -#ifndef NEWCC - Status = CcRosUnmapVacb(SharedCacheMap, FileOffset, FALSE); -#else - Status = STATUS_SUCCESS; -#endif -#ifndef NEWCC - if (!NT_SUCCESS(Status)) - { - DPRINT1("CcRosUnmapVacb failed, status = %x\n", Status); - KeBugCheckEx(MEMORY_MANAGEMENT, Status, (ULONG_PTR)SharedCacheMap, (ULONG_PTR)FileOffset, (ULONG_PTR)Address); - } -#endif - MiSetPageEvent(NULL, NULL); - return(STATUS_SUCCESS); - } - else if (!Context.WasDirty && !DirectMapped && !Context.Private) - { - if (SwapEntry != 0) - { - DPRINT1("Found a swap entry for a non dirty, non private and not direct mapped page (address %p)\n", - Address); - KeBugCheckEx(MEMORY_MANAGEMENT, SwapEntry, Page, (ULONG_PTR)Process, (ULONG_PTR)Address); - } - MmReleasePageMemoryConsumer(MC_USER, Page); - MiSetPageEvent(NULL, NULL); - return(STATUS_SUCCESS); - } - else if (!Context.WasDirty && Context.Private && SwapEntry != 0) - { - DPRINT("Not dirty and private and not swapped (%p:%p)\n", Process, Address); - MmSetSavedSwapEntryPage(Page, 0); - MmLockAddressSpace(AddressSpace); - Status = MmCreatePageFileMapping(Process, - Address, - SwapEntry); - MmUnlockAddressSpace(AddressSpace); - if (!NT_SUCCESS(Status)) - { - DPRINT1("Status %x Swapping out %p:%p\n", Status, Process, Address); - KeBugCheckEx(MEMORY_MANAGEMENT, Status, (ULONG_PTR)Process, (ULONG_PTR)Address, SwapEntry); - } - MmReleasePageMemoryConsumer(MC_USER, Page); - MiSetPageEvent(NULL, NULL); - return(STATUS_SUCCESS); - } - - /* - * If necessary, allocate an entry in the paging file for this page - */ - if (SwapEntry == 0) - { - SwapEntry = MmAllocSwapPage(); - if (SwapEntry == 0) - { - MmShowOutOfSpaceMessagePagingFile(); - MmLockAddressSpace(AddressSpace); - /* - * For private pages restore the old mappings. - */ - if (Context.Private) - { - Status = MmCreateVirtualMapping(Process, - Address, - MemoryArea->Protect, - &Page, - 1); - MmSetDirtyPage(Process, Address); - MmInsertRmap(Page, - Process, - Address); - } - else - { - ULONG_PTR OldEntry; - - MmLockSectionSegment(Context.Segment); - - /* - * For non-private pages if the page wasn't direct mapped then - * set it back into the section segment entry so we don't loose - * our copy. Otherwise it will be handled by the cache manager. - */ - Status = MmCreateVirtualMapping(Process, - Address, - MemoryArea->Protect, - &Page, - 1); - MmSetDirtyPage(Process, Address); - MmInsertRmap(Page, - Process, - Address); - // If we got here, the previous entry should have been a wait - Entry = MAKE_SSE(Page << PAGE_SHIFT, 1); - OldEntry = MmGetPageEntrySectionSegment(Context.Segment, &Context.Offset); - ASSERT(OldEntry == 0 || OldEntry == MAKE_SWAP_SSE(MM_WAIT_ENTRY)); - MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, Entry); - MmUnlockSectionSegment(Context.Segment); - } - MmUnlockAddressSpace(AddressSpace); - MiSetPageEvent(NULL, NULL); - return(STATUS_PAGEFILE_QUOTA); - } - } - - /* - * Write the page to the pagefile - */ - Status = MmWriteToSwapPage(SwapEntry, Page); - if (!NT_SUCCESS(Status)) - { - DPRINT1("MM: Failed to write to swap page (Status was 0x%.8X)\n", - Status); - /* - * As above: undo our actions. - * FIXME: Also free the swap page. - */ - MmLockAddressSpace(AddressSpace); - if (Context.Private) - { - Status = MmCreateVirtualMapping(Process, - Address, - MemoryArea->Protect, - &Page, - 1); - MmSetDirtyPage(Process, Address); - MmInsertRmap(Page, - Process, - Address); - } - else - { - MmLockSectionSegment(Context.Segment); - Status = MmCreateVirtualMapping(Process, - Address, - MemoryArea->Protect, - &Page, - 1); - MmSetDirtyPage(Process, Address); - MmInsertRmap(Page, - Process, - Address); - Entry = MAKE_SSE(Page << PAGE_SHIFT, 1); - MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, Entry); - MmUnlockSectionSegment(Context.Segment); - } - MmUnlockAddressSpace(AddressSpace); - MiSetPageEvent(NULL, NULL); - return(STATUS_UNSUCCESSFUL); - } - - /* - * Otherwise we have succeeded. - */ - DPRINT("MM: Wrote section page 0x%.8X to swap!\n", Page << PAGE_SHIFT); - MmSetSavedSwapEntryPage(Page, 0); - if (Context.Segment->Flags & MM_PAGEFILE_SEGMENT || - Context.Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED) - { - MmLockSectionSegment(Context.Segment); - MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, MAKE_SWAP_SSE(SwapEntry)); - MmUnlockSectionSegment(Context.Segment); - } - else - { - MmReleasePageMemoryConsumer(MC_USER, Page); - } - - if (Context.Private) - { - MmLockAddressSpace(AddressSpace); - MmLockSectionSegment(Context.Segment); - Status = MmCreatePageFileMapping(Process, - Address, - SwapEntry); - /* We had placed a wait entry upon entry ... replace it before leaving */ - MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, Entry); - MmUnlockSectionSegment(Context.Segment); - MmUnlockAddressSpace(AddressSpace); - if (!NT_SUCCESS(Status)) - { - DPRINT1("Status %x Creating page file mapping for %p:%p\n", Status, Process, Address); - KeBugCheckEx(MEMORY_MANAGEMENT, Status, (ULONG_PTR)Process, (ULONG_PTR)Address, SwapEntry); - } - } - else - { - MmLockAddressSpace(AddressSpace); - MmLockSectionSegment(Context.Segment); - Entry = MAKE_SWAP_SSE(SwapEntry); - /* We had placed a wait entry upon entry ... replace it before leaving */ - MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, Entry); - MmUnlockSectionSegment(Context.Segment); - MmUnlockAddressSpace(AddressSpace); - } - - MiSetPageEvent(NULL, NULL); - return(STATUS_SUCCESS); -} - -NTSTATUS -NTAPI -MmWritePageSectionView(PMMSUPPORT AddressSpace, - PMEMORY_AREA MemoryArea, - PVOID Address, - ULONG PageEntry) -{ - LARGE_INTEGER Offset; - PROS_SECTION_OBJECT Section; - PMM_SECTION_SEGMENT Segment; - PFN_NUMBER Page; - SWAPENTRY SwapEntry; - ULONG_PTR Entry; - BOOLEAN Private; - NTSTATUS Status; - PFILE_OBJECT FileObject; -#ifndef NEWCC - PROS_SHARED_CACHE_MAP SharedCacheMap = NULL; -#endif - BOOLEAN DirectMapped; - BOOLEAN IsImageSection; - PEPROCESS Process = MmGetAddressSpaceOwner(AddressSpace); - - Address = (PVOID)PAGE_ROUND_DOWN(Address); - - Offset.QuadPart = (ULONG_PTR)Address - MA_GetStartingAddress(MemoryArea) - + MemoryArea->Data.SectionData.ViewOffset.QuadPart; - - /* - * Get the segment and section. - */ - Segment = MemoryArea->Data.SectionData.Segment; - Section = MemoryArea->Data.SectionData.Section; - IsImageSection = Section->AllocationAttributes & SEC_IMAGE ? TRUE : FALSE; - - FileObject = Section->FileObject; - DirectMapped = FALSE; - if (FileObject != NULL && - !(Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) - { -#ifndef NEWCC - SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; -#endif - - /* - * If the file system is letting us go directly to the cache and the - * memory area was mapped at an offset in the file which is page aligned - * then note this is a direct mapped page. - */ - if (((Offset.QuadPart + Segment->Image.FileOffset) % PAGE_SIZE) == 0 && - (Offset.QuadPart + PAGE_SIZE <= Segment->RawLength.QuadPart || !IsImageSection)) - { - DirectMapped = TRUE; - } - } - - /* - * This should never happen since mappings of physical memory are never - * placed in the rmap lists. - */ - if (Section->AllocationAttributes & SEC_PHYSICALMEMORY) - { - DPRINT1("Trying to write back page from physical memory mapped at %p " - "process %p\n", Address, - Process ? Process->UniqueProcessId : 0); - KeBugCheck(MEMORY_MANAGEMENT); - } - - /* - * Get the section segment entry and the physical address. - */ - Entry = MmGetPageEntrySectionSegment(Segment, &Offset); - if (!MmIsPagePresent(Process, Address)) - { - DPRINT1("Trying to page out not-present page at (%p,0x%p).\n", - Process ? Process->UniqueProcessId : 0, Address); - KeBugCheck(MEMORY_MANAGEMENT); - } - Page = MmGetPfnForProcess(Process, Address); - SwapEntry = MmGetSavedSwapEntryPage(Page); - - /* - * Check for a private (COWed) page. - */ - if (IS_SWAP_FROM_SSE(Entry) || PFN_FROM_SSE(Entry) != Page) - { - Private = TRUE; - } - else - { - Private = FALSE; - } - - /* - * Speculatively set all mappings of the page to clean. - */ - MmSetCleanAllRmaps(Page); - - /* - * If this page was direct mapped from the cache then the cache manager - * will take care of writing it back to disk. - */ - if (DirectMapped && !Private) - { - //LARGE_INTEGER SOffset; - ASSERT(SwapEntry == 0); - //SOffset.QuadPart = Offset.QuadPart + Segment->Image.FileOffset; -#ifndef NEWCC - CcRosMarkDirtyFile(SharedCacheMap, Offset.QuadPart); -#endif - MmLockSectionSegment(Segment); - MmSetPageEntrySectionSegment(Segment, &Offset, PageEntry); - MmUnlockSectionSegment(Segment); - MiSetPageEvent(NULL, NULL); - return(STATUS_SUCCESS); - } - - /* - * If necessary, allocate an entry in the paging file for this page - */ - if (SwapEntry == 0) - { - SwapEntry = MmAllocSwapPage(); - if (SwapEntry == 0) - { - MmSetDirtyAllRmaps(Page); - MiSetPageEvent(NULL, NULL); - return(STATUS_PAGEFILE_QUOTA); - } - MmSetSavedSwapEntryPage(Page, SwapEntry); - } - - /* - * Write the page to the pagefile - */ - Status = MmWriteToSwapPage(SwapEntry, Page); - if (!NT_SUCCESS(Status)) - { - DPRINT1("MM: Failed to write to swap page (Status was 0x%.8X)\n", - Status); - MmSetDirtyAllRmaps(Page); - MiSetPageEvent(NULL, NULL); - return(STATUS_UNSUCCESSFUL); - } - - /* - * Otherwise we have succeeded. - */ - DPRINT("MM: Wrote section page 0x%.8X to swap!\n", Page << PAGE_SHIFT); - MiSetPageEvent(NULL, NULL); - return(STATUS_SUCCESS); + return STATUS_SUCCESS; } NTSTATUS @@ -2536,7 +2028,7 @@ MmProtectSectionView(PMMSUPPORT AddressSpace, Length = (ULONG)MaxLength; Region = MmFindRegion((PVOID)MA_GetStartingAddress(MemoryArea), - &MemoryArea->Data.SectionData.RegionListHead, + &MemoryArea->SectionData.RegionListHead, BaseAddress, NULL); ASSERT(Region != NULL); @@ -2548,11 +2040,11 @@ MmProtectSectionView(PMMSUPPORT AddressSpace, *OldProtect = Region->Protect; Status = MmAlterRegion(AddressSpace, (PVOID)MA_GetStartingAddress(MemoryArea), - &MemoryArea->Data.SectionData.RegionListHead, + &MemoryArea->SectionData.RegionListHead, BaseAddress, Length, Region->Type, Protect, MmAlterViewAttributes); - return(Status); + return Status; } NTSTATUS NTAPI @@ -2563,21 +2055,19 @@ MmQuerySectionView(PMEMORY_AREA MemoryArea, { PMM_REGION Region; PVOID RegionBaseAddress; - PROS_SECTION_OBJECT Section; PMM_SECTION_SEGMENT Segment; Region = MmFindRegion((PVOID)MA_GetStartingAddress(MemoryArea), - &MemoryArea->Data.SectionData.RegionListHead, + &MemoryArea->SectionData.RegionListHead, Address, &RegionBaseAddress); if (Region == NULL) { return STATUS_UNSUCCESSFUL; } - Section = MemoryArea->Data.SectionData.Section; - if (Section->AllocationAttributes & SEC_IMAGE) + if (MemoryArea->VadNode.u.VadFlags.VadType == VadImageMap) { - Segment = MemoryArea->Data.SectionData.Segment; + Segment = MemoryArea->SectionData.Segment; Info->AllocationBase = (PUCHAR)MA_GetStartingAddress(MemoryArea) - Segment->Image.VirtualAddress; Info->Type = MEM_IMAGE; } @@ -2587,61 +2077,19 @@ MmQuerySectionView(PMEMORY_AREA MemoryArea, Info->Type = MEM_MAPPED; } Info->BaseAddress = RegionBaseAddress; - Info->AllocationProtect = MemoryArea->Protect; + Info->AllocationProtect = MmProtectToValue[MemoryArea->VadNode.u.VadFlags.Protection]; Info->RegionSize = Region->Length; Info->State = MEM_COMMIT; Info->Protect = Region->Protect; *ResultLength = sizeof(MEMORY_BASIC_INFORMATION); - return(STATUS_SUCCESS); -} - -VOID -NTAPI -MmpFreePageFileSegment(PMM_SECTION_SEGMENT Segment) -{ - ULONG Length; - LARGE_INTEGER Offset; - ULONG_PTR Entry; - SWAPENTRY SavedSwapEntry; - PFN_NUMBER Page; - - Page = 0; - - MmLockSectionSegment(Segment); - - Length = PAGE_ROUND_UP(Segment->Length.QuadPart); - for (Offset.QuadPart = 0; Offset.QuadPart < Length; Offset.QuadPart += PAGE_SIZE) - { - Entry = MmGetPageEntrySectionSegment(Segment, &Offset); - if (Entry) - { - MmSetPageEntrySectionSegment(Segment, &Offset, 0); - if (IS_SWAP_FROM_SSE(Entry)) - { - MmFreeSwapPage(SWAPENTRY_FROM_SSE(Entry)); - } - else - { - Page = PFN_FROM_SSE(Entry); - SavedSwapEntry = MmGetSavedSwapEntryPage(Page); - if (SavedSwapEntry != 0) - { - MmSetSavedSwapEntryPage(Page, 0); - MmFreeSwapPage(SavedSwapEntry); - } - MmReleasePageMemoryConsumer(MC_USER, Page); - } - } - } - - MmUnlockSectionSegment(Segment); + return STATUS_SUCCESS; } VOID NTAPI MmpDeleteSection(PVOID ObjectBody) { - PROS_SECTION_OBJECT Section = (PROS_SECTION_OBJECT)ObjectBody; + PSECTION Section = ObjectBody; /* Check if it's an ARM3, or ReactOS section */ if (!MiIsRosSectionObject(Section)) @@ -2651,12 +2099,9 @@ MmpDeleteSection(PVOID ObjectBody) } DPRINT("MmpDeleteSection(ObjectBody %p)\n", ObjectBody); - if (Section->AllocationAttributes & SEC_IMAGE) + if (Section->u.Flags.Image) { - ULONG i; - ULONG NrSegments; - ULONG RefCount; - PMM_SECTION_SEGMENT SectionSegments; + PMM_IMAGE_SECTION_OBJECT ImageSectionObject = (PMM_IMAGE_SECTION_OBJECT)Section->Segment; /* * NOTE: Section->ImageSection can be NULL for short time @@ -2664,76 +2109,26 @@ MmpDeleteSection(PVOID ObjectBody) * until the image section is properly initialized we shouldn't * process further here. */ - if (Section->ImageSection == NULL) + if (Section->Segment == NULL) return; - SectionSegments = Section->ImageSection->Segments; - NrSegments = Section->ImageSection->NrSegments; - - for (i = 0; i < NrSegments; i++) - { - if (SectionSegments[i].Image.Characteristics & IMAGE_SCN_MEM_SHARED) - { - MmLockSectionSegment(&SectionSegments[i]); - } - RefCount = InterlockedDecrementUL(&SectionSegments[i].ReferenceCount); - if (SectionSegments[i].Image.Characteristics & IMAGE_SCN_MEM_SHARED) - { - MmUnlockSectionSegment(&SectionSegments[i]); - if (RefCount == 0) - { - MmpFreePageFileSegment(&SectionSegments[i]); - } - } - } + /* We just dereference the first segment */ + ASSERT(ImageSectionObject->RefCount > 0); + MmDereferenceSegment(ImageSectionObject->Segments); } -#ifdef NEWCC - else if (Section->Segment && Section->Segment->Flags & MM_DATAFILE_SEGMENT) - { - ULONG RefCount = 0; - PMM_SECTION_SEGMENT Segment = Section->Segment; - - if (Segment && - (RefCount = InterlockedDecrementUL(&Segment->ReferenceCount)) == 0) - { - DPRINT("Freeing section segment\n"); - Section->Segment = NULL; - MmFinalizeSegment(Segment); - } - else - { - DPRINT("RefCount %d\n", RefCount); - } - } -#endif else { + PMM_SECTION_SEGMENT Segment = (PMM_SECTION_SEGMENT)Section->Segment; + /* * NOTE: Section->Segment can be NULL for short time * during the section creating. */ - if (Section->Segment == NULL) + if (Segment == NULL) return; - if (Section->Segment->Flags & MM_PAGEFILE_SEGMENT) - { - MmpFreePageFileSegment(Section->Segment); - MmFreePageTablesSectionSegment(Section->Segment, NULL); - ExFreePool(Section->Segment); - Section->Segment = NULL; - } - else - { - (void)InterlockedDecrementUL(&Section->Segment->ReferenceCount); - } - } - if (Section->FileObject != NULL) - { -#ifndef NEWCC - CcRosDereferenceCache(Section->FileObject); -#endif - ObDereferenceObject(Section->FileObject); - Section->FileObject = NULL; + Segment->SectionCount--; + MmDereferenceSegment(Segment); } } @@ -2752,35 +2147,76 @@ NTSTATUS NTAPI MmCreatePhysicalMemorySection(VOID) { - PROS_SECTION_OBJECT PhysSection; + PSECTION PhysSection; NTSTATUS Status; OBJECT_ATTRIBUTES Obj; UNICODE_STRING Name = RTL_CONSTANT_STRING(L"\\Device\\PhysicalMemory"); LARGE_INTEGER SectionSize; HANDLE Handle; + PMM_SECTION_SEGMENT Segment; /* * Create the section mapping physical memory */ - SectionSize.QuadPart = 0xFFFFFFFF; + SectionSize.QuadPart = MmHighestPhysicalPage * PAGE_SIZE; InitializeObjectAttributes(&Obj, &Name, OBJ_PERMANENT | OBJ_KERNEL_EXCLUSIVE, NULL, NULL); - Status = MmCreateSection((PVOID)&PhysSection, - SECTION_ALL_ACCESS, - &Obj, - &SectionSize, - PAGE_EXECUTE_READWRITE, - SEC_PHYSICALMEMORY, - NULL, - NULL); + /* + * Create the Object + */ + Status = ObCreateObject(KernelMode, + MmSectionObjectType, + &Obj, + ExGetPreviousMode(), + NULL, + sizeof(*PhysSection), + 0, + 0, + (PVOID*)&PhysSection); if (!NT_SUCCESS(Status)) { - DPRINT1("Failed to create PhysicalMemory section\n"); - KeBugCheck(MEMORY_MANAGEMENT); + DPRINT1("MmCreatePhysicalMemorySection: failed to create object (0x%lx)\n", Status); + return Status; } + + /* + * Initialize it + */ + RtlZeroMemory(PhysSection, sizeof(*PhysSection)); + + /* Mark this as a "ROS Section" */ + PhysSection->u.Flags.filler = 1; + PhysSection->InitialPageProtection = PAGE_EXECUTE_READWRITE; + PhysSection->u.Flags.PhysicalMemory = 1; + PhysSection->SizeOfSection = SectionSize; + Segment = ExAllocatePoolWithTag(NonPagedPool, sizeof(MM_SECTION_SEGMENT), + TAG_MM_SECTION_SEGMENT); + if (Segment == NULL) + { + ObDereferenceObject(PhysSection); + return STATUS_NO_MEMORY; + } + RtlZeroMemory(Segment, sizeof(MM_SECTION_SEGMENT)); + PhysSection->Segment = (PSEGMENT)Segment; + Segment->RefCount = 1; + + Segment->ReferenceCount = &Segment->RefCount; + Segment->Flags = &Segment->SegFlags; + + ExInitializeFastMutex(&Segment->Lock); + Segment->Image.FileOffset = 0; + Segment->Protection = PAGE_EXECUTE_READWRITE; + Segment->RawLength = SectionSize; + Segment->Length = SectionSize; + Segment->SegFlags = MM_PHYSICALMEMORY_SEGMENT; + Segment->WriteCopy = FALSE; + Segment->Image.VirtualAddress = 0; + Segment->Image.Characteristics = 0; + MiInitializeSectionPageTable(Segment); + Status = ObInsertObject(PhysSection, NULL, SECTION_ALL_ACCESS, @@ -2790,12 +2226,11 @@ MmCreatePhysicalMemorySection(VOID) if (!NT_SUCCESS(Status)) { ObDereferenceObject(PhysSection); + return Status; } ObCloseHandle(Handle, KernelMode); - PhysSection->AllocationAttributes |= SEC_PHYSICALMEMORY; - PhysSection->Segment->Flags &= ~MM_PAGEFILE_SEGMENT; - return(STATUS_SUCCESS); + return STATUS_SUCCESS; } CODE_SEG("INIT") @@ -2816,7 +2251,7 @@ MmInitSectionImplementation(VOID) RtlZeroMemory(&ObjectTypeInitializer, sizeof(ObjectTypeInitializer)); RtlInitUnicodeString(&Name, L"Section"); ObjectTypeInitializer.Length = sizeof(ObjectTypeInitializer); - ObjectTypeInitializer.DefaultPagedPoolCharge = sizeof(ROS_SECTION_OBJECT); + ObjectTypeInitializer.DefaultPagedPoolCharge = sizeof(SECTION); ObjectTypeInitializer.PoolType = PagedPool; ObjectTypeInitializer.UseDefaultObject = TRUE; ObjectTypeInitializer.GenericMapping = MmpSectionMapping; @@ -2828,103 +2263,29 @@ MmInitSectionImplementation(VOID) MmCreatePhysicalMemorySection(); - return(STATUS_SUCCESS); + return STATUS_SUCCESS; } +static NTSTATUS NTAPI -MmCreatePageFileSection(PROS_SECTION_OBJECT *SectionObject, - ACCESS_MASK DesiredAccess, - POBJECT_ATTRIBUTES ObjectAttributes, - PLARGE_INTEGER UMaximumSize, - ULONG SectionPageProtection, - ULONG AllocationAttributes) -/* - * Create a section which is backed by the pagefile - */ -{ - LARGE_INTEGER MaximumSize; - PROS_SECTION_OBJECT Section; - PMM_SECTION_SEGMENT Segment; - NTSTATUS Status; - - if (UMaximumSize == NULL) - { - DPRINT1("MmCreatePageFileSection: (UMaximumSize == NULL)\n"); - return(STATUS_INVALID_PARAMETER); - } - MaximumSize = *UMaximumSize; - - /* - * Create the section - */ - Status = ObCreateObject(ExGetPreviousMode(), - MmSectionObjectType, - ObjectAttributes, - ExGetPreviousMode(), - NULL, - sizeof(ROS_SECTION_OBJECT), - 0, - 0, - (PVOID*)(PVOID)&Section); - if (!NT_SUCCESS(Status)) - { - DPRINT1("MmCreatePageFileSection: failed to create object (0x%lx)\n", Status); - return(Status); - } - - /* - * Initialize it - */ - RtlZeroMemory(Section, sizeof(ROS_SECTION_OBJECT)); - Section->Type = 'SC'; - Section->Size = 'TN'; - Section->SectionPageProtection = SectionPageProtection; - Section->AllocationAttributes = AllocationAttributes; - Section->MaximumSize = MaximumSize; - Segment = ExAllocatePoolWithTag(NonPagedPool, sizeof(MM_SECTION_SEGMENT), - TAG_MM_SECTION_SEGMENT); - if (Segment == NULL) - { - ObDereferenceObject(Section); - return(STATUS_NO_MEMORY); - } - RtlZeroMemory(Segment, sizeof(MM_SECTION_SEGMENT)); - Section->Segment = Segment; - Segment->ReferenceCount = 1; - ExInitializeFastMutex(&Segment->Lock); - Segment->Image.FileOffset = 0; - Segment->Protection = SectionPageProtection; - Segment->RawLength.QuadPart = MaximumSize.u.LowPart; - Segment->Length.QuadPart = PAGE_ROUND_UP(MaximumSize.u.LowPart); - Segment->Flags = MM_PAGEFILE_SEGMENT; - Segment->WriteCopy = FALSE; - Segment->Image.VirtualAddress = 0; - Segment->Image.Characteristics = 0; - *SectionObject = Section; - MiInitializeSectionPageTable(Segment); - return(STATUS_SUCCESS); -} - -NTSTATUS -NTAPI -MmCreateDataFileSection(PROS_SECTION_OBJECT *SectionObject, +MmCreateDataFileSection(PSECTION *SectionObject, ACCESS_MASK DesiredAccess, POBJECT_ATTRIBUTES ObjectAttributes, PLARGE_INTEGER UMaximumSize, ULONG SectionPageProtection, ULONG AllocationAttributes, - PFILE_OBJECT FileObject) + PFILE_OBJECT FileObject, + BOOLEAN GotFileHandle) /* * Create a section backed by a data file */ { - PROS_SECTION_OBJECT Section; + PSECTION Section; NTSTATUS Status; LARGE_INTEGER MaximumSize; PMM_SECTION_SEGMENT Segment; - FILE_STANDARD_INFORMATION FileInfo; - ULONG Length; + KIRQL OldIrql; /* * Create the section @@ -2934,81 +2295,81 @@ MmCreateDataFileSection(PROS_SECTION_OBJECT *SectionObject, ObjectAttributes, ExGetPreviousMode(), NULL, - sizeof(ROS_SECTION_OBJECT), + sizeof(*Section), 0, 0, (PVOID*)&Section); if (!NT_SUCCESS(Status)) { - ObDereferenceObject(FileObject); - return(Status); + return Status; } /* * Initialize it */ - RtlZeroMemory(Section, sizeof(ROS_SECTION_OBJECT)); - Section->Type = 'SC'; - Section->Size = 'TN'; - Section->SectionPageProtection = SectionPageProtection; - Section->AllocationAttributes = AllocationAttributes; + RtlZeroMemory(Section, sizeof(*Section)); - /* - * FIXME: This is propably not entirely correct. We can't look into - * the standard FCB header because it might not be initialized yet - * (as in case of the EXT2FS driver by Manoj Paul Joseph where the - * standard file information is filled on first request). - */ - Status = IoQueryFileInformation(FileObject, - FileStandardInformation, - sizeof(FILE_STANDARD_INFORMATION), - &FileInfo, - &Length); - if (!NT_SUCCESS(Status)) - { - ObDereferenceObject(Section); - ObDereferenceObject(FileObject); - return Status; - } + /* Mark this as a "ROS" section */ + Section->u.Flags.filler = 1; + Section->InitialPageProtection = SectionPageProtection; + Section->u.Flags.File = 1; - /* - * FIXME: Revise this once a locking order for file size changes is - * decided - */ - if ((UMaximumSize != NULL) && (UMaximumSize->QuadPart != 0)) + if (AllocationAttributes & SEC_NO_CHANGE) + Section->u.Flags.NoChange = 1; + if (AllocationAttributes & SEC_RESERVE) + Section->u.Flags.Reserve = 1; + + if (!GotFileHandle) { + ASSERT(UMaximumSize != NULL); + // ASSERT(UMaximumSize->QuadPart != 0); MaximumSize = *UMaximumSize; } else { - MaximumSize = FileInfo.EndOfFile; - /* Mapping zero-sized files isn't allowed. */ - if (MaximumSize.QuadPart == 0) - { - ObDereferenceObject(Section); - ObDereferenceObject(FileObject); - return STATUS_MAPPED_FILE_SIZE_ZERO; - } - } - - if (MaximumSize.QuadPart > FileInfo.EndOfFile.QuadPart) - { - Status = IoSetInformation(FileObject, - FileEndOfFileInformation, - sizeof(LARGE_INTEGER), - &MaximumSize); + LARGE_INTEGER FileSize; + Status = FsRtlGetFileSize(FileObject, &FileSize); if (!NT_SUCCESS(Status)) { ObDereferenceObject(Section); - ObDereferenceObject(FileObject); - return(STATUS_SECTION_NOT_EXTENDED); + return Status; + } + + /* + * FIXME: Revise this once a locking order for file size changes is + * decided + */ + if ((UMaximumSize != NULL) && (UMaximumSize->QuadPart != 0)) + { + MaximumSize = *UMaximumSize; + } + else + { + MaximumSize = FileSize; + /* Mapping zero-sized files isn't allowed. */ + if (MaximumSize.QuadPart == 0) + { + ObDereferenceObject(Section); + return STATUS_MAPPED_FILE_SIZE_ZERO; + } + } + + if (MaximumSize.QuadPart > FileSize.QuadPart) + { + Status = IoSetInformation(FileObject, + FileEndOfFileInformation, + sizeof(LARGE_INTEGER), + &MaximumSize); + if (!NT_SUCCESS(Status)) + { + ObDereferenceObject(Section); + return STATUS_SECTION_NOT_EXTENDED; + } } } - if (FileObject->SectionObjectPointer == NULL || - FileObject->SectionObjectPointer->SharedCacheMap == NULL) + if (FileObject->SectionObjectPointer == NULL) { ObDereferenceObject(Section); - ObDereferenceObject(FileObject); return STATUS_INVALID_FILE_FOR_SECTION; } @@ -3019,37 +2380,62 @@ MmCreateDataFileSection(PROS_SECTION_OBJECT *SectionObject, if (Status != STATUS_SUCCESS) { ObDereferenceObject(Section); - ObDereferenceObject(FileObject); - return(Status); + return Status; + } + + /* Lock the PFN lock while messing with Section Object pointers */ + OldIrql = MiAcquirePfnLock(); + Segment = FileObject->SectionObjectPointer->DataSectionObject; + + while (Segment && (Segment->SegFlags & (MM_SEGMENT_INDELETE | MM_SEGMENT_INCREATE))) + { + MiReleasePfnLock(OldIrql); + KeDelayExecutionThread(KernelMode, FALSE, &TinyTime); + OldIrql = MiAcquirePfnLock(); + Segment = FileObject->SectionObjectPointer->DataSectionObject; } /* * If this file hasn't been mapped as a data file before then allocate a * section segment to describe the data file mapping */ - if (FileObject->SectionObjectPointer->DataSectionObject == NULL) + if (Segment == NULL) { Segment = ExAllocatePoolWithTag(NonPagedPool, sizeof(MM_SECTION_SEGMENT), TAG_MM_SECTION_SEGMENT); if (Segment == NULL) { //KeSetEvent((PVOID)&FileObject->Lock, IO_NO_INCREMENT, FALSE); + MiReleasePfnLock(OldIrql); ObDereferenceObject(Section); - ObDereferenceObject(FileObject); - return(STATUS_NO_MEMORY); + return STATUS_NO_MEMORY; } - Section->Segment = Segment; - Segment->ReferenceCount = 1; + + /* We are creating it */ + RtlZeroMemory(Segment, sizeof(*Segment)); + Segment->SegFlags = MM_DATAFILE_SEGMENT | MM_SEGMENT_INCREATE; + Segment->RefCount = 1; + + FileObject->SectionObjectPointer->DataSectionObject = Segment; + + /* We're safe to release the lock now */ + MiReleasePfnLock(OldIrql); + + Section->Segment = (PSEGMENT)Segment; + + /* Self-referencing segment */ + Segment->Flags = &Segment->SegFlags; + Segment->ReferenceCount = &Segment->RefCount; + + Segment->SectionCount = 1; + ExInitializeFastMutex(&Segment->Lock); - /* - * Set the lock before assigning the segment to the file object - */ - ExAcquireFastMutex(&Segment->Lock); - FileObject->SectionObjectPointer->DataSectionObject = (PVOID)Segment; + Segment->FileObject = FileObject; + ObReferenceObject(FileObject); Segment->Image.FileOffset = 0; Segment->Protection = SectionPageProtection; - Segment->Flags = MM_DATAFILE_SEGMENT; + Segment->Image.Characteristics = 0; Segment->WriteCopy = (SectionPageProtection & (PAGE_WRITECOPY | PAGE_EXECUTE_WRITECOPY)); if (AllocationAttributes & SEC_RESERVE) @@ -3062,20 +2448,21 @@ MmCreateDataFileSection(PROS_SECTION_OBJECT *SectionObject, Segment->Length.QuadPart = PAGE_ROUND_UP(Segment->RawLength.QuadPart); } Segment->Image.VirtualAddress = 0; - Segment->Locked = TRUE; MiInitializeSectionPageTable(Segment); + + /* We're good to use it now */ + OldIrql = MiAcquirePfnLock(); + Segment->SegFlags &= ~MM_SEGMENT_INCREATE; + MiReleasePfnLock(OldIrql); } else { - /* - * If the file is already mapped as a data file then we may need - * to extend it - */ - Segment = - (PMM_SECTION_SEGMENT)FileObject->SectionObjectPointer-> - DataSectionObject; - Section->Segment = Segment; - (void)InterlockedIncrementUL(&Segment->ReferenceCount); + Section->Segment = (PSEGMENT)Segment; + Segment->RefCount++; + InterlockedIncrementUL(&Segment->SectionCount); + + MiReleasePfnLock(OldIrql); + MmLockSectionSegment(Segment); if (MaximumSize.QuadPart > Segment->RawLength.QuadPart && @@ -3084,16 +2471,14 @@ MmCreateDataFileSection(PROS_SECTION_OBJECT *SectionObject, Segment->RawLength.QuadPart = MaximumSize.QuadPart; Segment->Length.QuadPart = PAGE_ROUND_UP(Segment->RawLength.QuadPart); } + + MmUnlockSectionSegment(Segment); } - MmUnlockSectionSegment(Segment); - Section->FileObject = FileObject; - Section->MaximumSize = MaximumSize; -#ifndef NEWCC - CcRosReferenceCache(FileObject); -#endif + Section->SizeOfSection = MaximumSize; + //KeSetEvent((PVOID)&FileObject->Lock, IO_NO_INCREMENT, FALSE); *SectionObject = Section; - return(STATUS_SUCCESS); + return STATUS_SUCCESS; } /* @@ -3150,7 +2535,6 @@ ExeFmtpAllocateSegments(IN ULONG NrSegments) return Segments; } - static NTSTATUS NTAPI @@ -3193,28 +2577,18 @@ ExeFmtpReadFile(IN PVOID File, BufferSize = Length + OffsetAdjustment; BufferSize = PAGE_ROUND_UP(BufferSize); - /* Flush data since we're about to perform a non-cached read */ - CcFlushCache(FileObject->SectionObjectPointer, - &FileOffset, - BufferSize, - &Iosb); - /* * It's ok to use paged pool, because this is a temporary buffer only used in * the loading of executables. The assumption is that MmCreateSection is * always called at low IRQLs and that these buffers don't survive a brief * initialization phase */ - Buffer = ExAllocatePoolWithTag(PagedPool, - BufferSize, - 'rXmM'); + Buffer = ExAllocatePoolWithTag(PagedPool, BufferSize, 'rXmM'); if (!Buffer) { return STATUS_INSUFFICIENT_RESOURCES; } - UsedSize = 0; - Status = MiSimpleRead(FileObject, &FileOffset, Buffer, BufferSize, TRUE, &Iosb); UsedSize = (ULONG)Iosb.Information; @@ -3617,7 +2991,6 @@ ExeFmtpCreateImageSection(PFILE_OBJECT FileObject, */ for (i = 0; i < RTL_NUMBER_OF(ExeFmtpLoaders); ++ i) { - RtlZeroMemory(ImageSectionObject, sizeof(*ImageSectionObject)); Flags = 0; Status = ExeFmtpLoaders[i](FileHeader, @@ -3656,6 +3029,7 @@ ExeFmtpCreateImageSection(PFILE_OBJECT FileObject, return Status; ASSERT(ImageSectionObject->Segments != NULL); + ASSERT(ImageSectionObject->RefCount > 0); /* * Some defaults @@ -3716,16 +3090,22 @@ ExeFmtpCreateImageSection(PFILE_OBJECT FileObject, for ( i = 0; i < ImageSectionObject->NrSegments; ++ i ) { ExInitializeFastMutex(&ImageSectionObject->Segments[i].Lock); - ImageSectionObject->Segments[i].ReferenceCount = 1; + ImageSectionObject->Segments[i].ReferenceCount = &ImageSectionObject->RefCount; + ImageSectionObject->Segments[i].Flags = &ImageSectionObject->SegFlags; MiInitializeSectionPageTable(&ImageSectionObject->Segments[i]); + ImageSectionObject->Segments[i].FileObject = FileObject; } + ASSERT(ImageSectionObject->RefCount > 0); + + ImageSectionObject->FileObject = FileObject; + ASSERT(NT_SUCCESS(Status)); return Status; } NTSTATUS -MmCreateImageSection(PROS_SECTION_OBJECT *SectionObject, +MmCreateImageSection(PSECTION *SectionObject, ACCESS_MASK DesiredAccess, POBJECT_ATTRIBUTES ObjectAttributes, PLARGE_INTEGER UMaximumSize, @@ -3733,22 +3113,20 @@ MmCreateImageSection(PROS_SECTION_OBJECT *SectionObject, ULONG AllocationAttributes, PFILE_OBJECT FileObject) { - PROS_SECTION_OBJECT Section; + PSECTION Section; NTSTATUS Status; - PMM_SECTION_SEGMENT SectionSegments; PMM_IMAGE_SECTION_OBJECT ImageSectionObject; - ULONG i; + KIRQL OldIrql; + if (FileObject == NULL) return STATUS_INVALID_FILE_FOR_SECTION; -#ifndef NEWCC - if (!CcIsFileCached(FileObject)) + if (FileObject->SectionObjectPointer == NULL) { DPRINT1("Denying section creation due to missing cache initialization\n"); return STATUS_INVALID_FILE_FOR_SECTION; } -#endif /* * Create the section @@ -3758,43 +3136,73 @@ MmCreateImageSection(PROS_SECTION_OBJECT *SectionObject, ObjectAttributes, ExGetPreviousMode(), NULL, - sizeof(ROS_SECTION_OBJECT), + sizeof(*Section), 0, 0, (PVOID*)(PVOID)&Section); if (!NT_SUCCESS(Status)) { - ObDereferenceObject(FileObject); - return(Status); + return Status; } /* * Initialize it */ - RtlZeroMemory(Section, sizeof(ROS_SECTION_OBJECT)); - Section->Type = 'SC'; - Section->Size = 'TN'; - Section->SectionPageProtection = SectionPageProtection; - Section->AllocationAttributes = AllocationAttributes; + RtlZeroMemory(Section, sizeof(*Section)); - if (FileObject->SectionObjectPointer->ImageSectionObject == NULL) + /* Mark this as a "ROS" Section */ + Section->u.Flags.filler = 1; + + Section->InitialPageProtection = SectionPageProtection; + Section->u.Flags.File = 1; + Section->u.Flags.Image = 1; + if (AllocationAttributes & SEC_NO_CHANGE) + Section->u.Flags.NoChange = 1; + + OldIrql = MiAcquirePfnLock(); + + /* Wait for it to be properly created or deleted */ + ImageSectionObject = FileObject->SectionObjectPointer->ImageSectionObject; + while(ImageSectionObject && (ImageSectionObject->SegFlags & (MM_SEGMENT_INDELETE | MM_SEGMENT_INCREATE))) + { + MiReleasePfnLock(OldIrql); + + KeDelayExecutionThread(KernelMode, FALSE, &TinyTime); + + OldIrql = MiAcquirePfnLock(); + ImageSectionObject = FileObject->SectionObjectPointer->ImageSectionObject; + } + + if (ImageSectionObject == NULL) { NTSTATUS StatusExeFmt; - ImageSectionObject = ExAllocatePoolWithTag(PagedPool, sizeof(MM_IMAGE_SECTION_OBJECT), TAG_MM_SECTION_SEGMENT); + ImageSectionObject = ExAllocatePoolZero(NonPagedPool, sizeof(MM_IMAGE_SECTION_OBJECT), TAG_MM_SECTION_SEGMENT); if (ImageSectionObject == NULL) { - ObDereferenceObject(FileObject); + MiReleasePfnLock(OldIrql); ObDereferenceObject(Section); - return(STATUS_NO_MEMORY); + return STATUS_NO_MEMORY; } - RtlZeroMemory(ImageSectionObject, sizeof(MM_IMAGE_SECTION_OBJECT)); + ImageSectionObject->SegFlags = MM_SEGMENT_INCREATE; + ImageSectionObject->RefCount = 1; + FileObject->SectionObjectPointer->ImageSectionObject = ImageSectionObject; + + MiReleasePfnLock(OldIrql); + + /* Purge the cache */ + CcFlushCache(FileObject->SectionObjectPointer, NULL, 0, NULL); StatusExeFmt = ExeFmtpCreateImageSection(FileObject, ImageSectionObject); if (!NT_SUCCESS(StatusExeFmt)) { + /* Unset */ + OldIrql = MiAcquirePfnLock(); + FileObject->SectionObjectPointer->ImageSectionObject = NULL; + MiReleasePfnLock(OldIrql); + if(ImageSectionObject->Segments != NULL) ExFreePool(ImageSectionObject->Segments); @@ -3809,12 +3217,12 @@ MmCreateImageSection(PROS_SECTION_OBJECT *SectionObject, ExFreePoolWithTag(ImageSectionObject, TAG_MM_SECTION_SEGMENT); ObDereferenceObject(Section); - ObDereferenceObject(FileObject); - return(Status); + return Status; } - Section->ImageSection = ImageSectionObject; + Section->Segment = (PSEGMENT)ImageSectionObject; ASSERT(ImageSectionObject->Segments); + ASSERT(ImageSectionObject->RefCount > 0); /* * Lock the file @@ -3822,85 +3230,64 @@ MmCreateImageSection(PROS_SECTION_OBJECT *SectionObject, Status = MmspWaitForFileLock(FileObject); if (!NT_SUCCESS(Status)) { + /* Unset */ + OldIrql = MiAcquirePfnLock(); + FileObject->SectionObjectPointer->ImageSectionObject = NULL; + MiReleasePfnLock(OldIrql); + ExFreePool(ImageSectionObject->Segments); ExFreePool(ImageSectionObject); ObDereferenceObject(Section); - ObDereferenceObject(FileObject); - return(Status); + return Status; } - if (NULL != InterlockedCompareExchangePointer(&FileObject->SectionObjectPointer->ImageSectionObject, - ImageSectionObject, NULL)) - { - /* - * An other thread has initialized the same image in the background - */ - ExFreePool(ImageSectionObject->Segments); - ExFreePool(ImageSectionObject); - ImageSectionObject = FileObject->SectionObjectPointer->ImageSectionObject; - Section->ImageSection = ImageSectionObject; - SectionSegments = ImageSectionObject->Segments; + OldIrql = MiAcquirePfnLock(); + ImageSectionObject->SegFlags &= ~MM_SEGMENT_INCREATE; - for (i = 0; i < ImageSectionObject->NrSegments; i++) - { - (void)InterlockedIncrementUL(&SectionSegments[i].ReferenceCount); - } - } + /* Take a ref on the file on behalf of the newly created structure */ + ObReferenceObject(FileObject); + + MiReleasePfnLock(OldIrql); Status = StatusExeFmt; } else { - /* - * Lock the file - */ - Status = MmspWaitForFileLock(FileObject); - if (Status != STATUS_SUCCESS) - { - ObDereferenceObject(Section); - ObDereferenceObject(FileObject); - return(Status); - } + /* Take one ref */ + ImageSectionObject->RefCount++; - ImageSectionObject = FileObject->SectionObjectPointer->ImageSectionObject; - Section->ImageSection = ImageSectionObject; - SectionSegments = ImageSectionObject->Segments; + MiReleasePfnLock(OldIrql); - /* - * Otherwise just reference all the section segments - */ - for (i = 0; i < ImageSectionObject->NrSegments; i++) - { - (void)InterlockedIncrementUL(&SectionSegments[i].ReferenceCount); - } + Section->Segment = (PSEGMENT)ImageSectionObject; Status = STATUS_SUCCESS; } - Section->FileObject = FileObject; -#ifndef NEWCC - CcRosReferenceCache(FileObject); -#endif //KeSetEvent((PVOID)&FileObject->Lock, IO_NO_INCREMENT, FALSE); *SectionObject = Section; - return(Status); + ASSERT(ImageSectionObject->RefCount > 0); + + return Status; } static NTSTATUS -MmMapViewOfSegment(PMMSUPPORT AddressSpace, - PROS_SECTION_OBJECT Section, - PMM_SECTION_SEGMENT Segment, - PVOID* BaseAddress, - SIZE_T ViewSize, - ULONG Protect, - ULONG ViewOffset, - ULONG AllocationType) +MmMapViewOfSegment( + PMMSUPPORT AddressSpace, + BOOLEAN AsImage, + PMM_SECTION_SEGMENT Segment, + PVOID* BaseAddress, + SIZE_T ViewSize, + ULONG Protect, + LONGLONG ViewOffset, + ULONG AllocationType) { PMEMORY_AREA MArea; NTSTATUS Status; ULONG Granularity; + ASSERT(ViewSize != 0); + if (Segment->WriteCopy) { /* We have to do this because the not present fault @@ -3944,23 +3331,22 @@ MmMapViewOfSegment(PMMSUPPORT AddressSpace, { DPRINT1("Mapping between 0x%p and 0x%p failed (%X).\n", (*BaseAddress), (char*)(*BaseAddress) + ViewSize, Status); - return(Status); + return Status; } - ObReferenceObject((PVOID)Section); + InterlockedIncrement64(Segment->ReferenceCount); - MArea->Data.SectionData.Segment = Segment; - MArea->Data.SectionData.Section = Section; - MArea->Data.SectionData.ViewOffset.QuadPart = ViewOffset; - if (Section->AllocationAttributes & SEC_IMAGE) + MArea->SectionData.Segment = Segment; + MArea->SectionData.ViewOffset = ViewOffset; + if (AsImage) { MArea->VadNode.u.VadFlags.VadType = VadImageMap; } - MmInitializeRegion(&MArea->Data.SectionData.RegionListHead, + MmInitializeRegion(&MArea->SectionData.RegionListHead, ViewSize, 0, Protect); - return(STATUS_SUCCESS); + return STATUS_SUCCESS; } @@ -3969,13 +3355,8 @@ MmFreeSectionPage(PVOID Context, MEMORY_AREA* MemoryArea, PVOID Address, PFN_NUMBER Page, SWAPENTRY SwapEntry, BOOLEAN Dirty) { ULONG_PTR Entry; -#ifndef NEWCC - PFILE_OBJECT FileObject; - PROS_SHARED_CACHE_MAP SharedCacheMap; -#endif LARGE_INTEGER Offset; SWAPENTRY SavedSwapEntry; - PROS_SECTION_OBJECT Section; PMM_SECTION_SEGMENT Segment; PMMSUPPORT AddressSpace; PEPROCESS Process; @@ -3986,10 +3367,9 @@ MmFreeSectionPage(PVOID Context, MEMORY_AREA* MemoryArea, PVOID Address, Address = (PVOID)PAGE_ROUND_DOWN(Address); Offset.QuadPart = ((ULONG_PTR)Address - MA_GetStartingAddress(MemoryArea)) + - MemoryArea->Data.SectionData.ViewOffset.QuadPart; + MemoryArea->SectionData.ViewOffset; - Section = MemoryArea->Data.SectionData.Section; - Segment = MemoryArea->Data.SectionData.Segment; + Segment = MemoryArea->SectionData.Segment; Entry = MmGetPageEntrySectionSegment(Segment, &Offset); while (Entry && MM_IS_WAIT_PTE(Entry)) @@ -4005,18 +3385,12 @@ MmFreeSectionPage(PVOID Context, MEMORY_AREA* MemoryArea, PVOID Address, } /* - * For a dirty, datafile, non-private page mark it as dirty in the - * cache manager. + * For a dirty, datafile, non-private page, there shoulkd be no swap entry */ - if (Segment->Flags & MM_DATAFILE_SEGMENT) + if (*Segment->Flags & MM_DATAFILE_SEGMENT) { if (Page == PFN_FROM_SSE(Entry) && Dirty) { -#ifndef NEWCC - FileObject = MemoryArea->Data.SectionData.Section->FileObject; - SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; - CcRosMarkDirtyFile(SharedCacheMap, Offset.QuadPart + Segment->Image.FileOffset); -#endif ASSERT(SwapEntry == 0); } } @@ -4026,11 +3400,6 @@ MmFreeSectionPage(PVOID Context, MEMORY_AREA* MemoryArea, PVOID Address, /* * Sanity check */ - if (Segment->Flags & MM_PAGEFILE_SEGMENT) - { - DPRINT1("Found a swap entry for a page in a pagefile section.\n"); - KeBugCheck(MEMORY_MANAGEMENT); - } MmFreeSwapPage(SwapEntry); } else if (Page != 0) @@ -4038,14 +3407,8 @@ MmFreeSectionPage(PVOID Context, MEMORY_AREA* MemoryArea, PVOID Address, if (IS_SWAP_FROM_SSE(Entry) || Page != PFN_FROM_SSE(Entry)) { - /* - * Sanity check - */ - if (Segment->Flags & MM_PAGEFILE_SEGMENT) - { - DPRINT1("Found a private page in a pagefile section.\n"); - KeBugCheck(MEMORY_MANAGEMENT); - } + ASSERT(Process != NULL); + /* * Just dereference private pages */ @@ -4060,8 +3423,13 @@ MmFreeSectionPage(PVOID Context, MEMORY_AREA* MemoryArea, PVOID Address, } else { - MmDeleteRmap(Page, Process, Address); - MmUnsharePageEntrySectionSegment(Section, Segment, &Offset, Dirty, FALSE, NULL); + if (Process) + { + MmDeleteRmap(Page, Process, Address); + } + + /* We don't dirtify for System Space Maps. We let Cc manage that */ + MmUnsharePageEntrySectionSegment(MemoryArea, Segment, &Offset, Process ? Dirty : FALSE, FALSE, NULL); } } } @@ -4072,7 +3440,6 @@ MmUnmapViewOfSegment(PMMSUPPORT AddressSpace, { NTSTATUS Status; PMEMORY_AREA MemoryArea; - PROS_SECTION_OBJECT Section; PMM_SECTION_SEGMENT Segment; PLIST_ENTRY CurrentEntry; PMM_REGION CurrentRegion; @@ -4082,11 +3449,10 @@ MmUnmapViewOfSegment(PMMSUPPORT AddressSpace, BaseAddress); if (MemoryArea == NULL) { - return(STATUS_UNSUCCESSFUL); + return STATUS_UNSUCCESSFUL; } - Section = MemoryArea->Data.SectionData.Section; - Segment = MemoryArea->Data.SectionData.Segment; + Segment = MemoryArea->SectionData.Segment; #ifdef NEWCC if (Segment->Flags & MM_DATAFILE_SEGMENT) @@ -4103,7 +3469,7 @@ MmUnmapViewOfSegment(PMMSUPPORT AddressSpace, MmLockSectionSegment(Segment); - RegionListHead = &MemoryArea->Data.SectionData.RegionListHead; + RegionListHead = &MemoryArea->SectionData.RegionListHead; while (!IsListEmpty(RegionListHead)) { CurrentEntry = RemoveHeadList(RegionListHead); @@ -4111,7 +3477,7 @@ MmUnmapViewOfSegment(PMMSUPPORT AddressSpace, ExFreePoolWithTag(CurrentRegion, TAG_MM_REGION); } - if (Section->AllocationAttributes & SEC_PHYSICALMEMORY) + if ((*Segment->Flags) & MM_PHYSICALMEMORY_SEGMENT) { Status = MmFreeMemoryArea(AddressSpace, MemoryArea, @@ -4126,10 +3492,11 @@ MmUnmapViewOfSegment(PMMSUPPORT AddressSpace, AddressSpace); } MmUnlockSectionSegment(Segment); - ObDereferenceObject(Section); - return(Status); + MmDereferenceSegment(Segment); + return Status; } +/* This functions must be called with a locked address space */ NTSTATUS NTAPI MiRosUnmapViewOfSection(IN PEPROCESS Process, @@ -4139,7 +3506,6 @@ MiRosUnmapViewOfSection(IN PEPROCESS Process, NTSTATUS Status; PMEMORY_AREA MemoryArea; PMMSUPPORT AddressSpace; - PROS_SECTION_OBJECT Section; PVOID ImageBaseAddress = 0; DPRINT("Opening memory area Process %p BaseAddress %p\n", @@ -4149,22 +3515,24 @@ MiRosUnmapViewOfSection(IN PEPROCESS Process, AddressSpace = Process ? &Process->Vm : MmGetKernelAddressSpace(); - MmLockAddressSpace(AddressSpace); MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace, BaseAddress); if (MemoryArea == NULL || - ((MemoryArea->Type != MEMORY_AREA_SECTION_VIEW) && - (MemoryArea->Type != MEMORY_AREA_CACHE)) || +#ifdef NEWCC + ((MemoryArea->Type != MEMORY_AREA_SECTION_VIEW) && (MemoryArea->Type != MEMORY_AREA_CACHE)) || +#else + (MemoryArea->Type != MEMORY_AREA_SECTION_VIEW) || +#endif MemoryArea->DeleteInProgress) + { if (MemoryArea) ASSERT(MemoryArea->Type != MEMORY_AREA_OWNED_BY_ARM3); - MmUnlockAddressSpace(AddressSpace); + + DPRINT1("Unable to find memory area at address %p.\n", BaseAddress); return STATUS_NOT_MAPPED_VIEW; } - Section = MemoryArea->Data.SectionData.Section; - - if ((Section != NULL) && (Section->AllocationAttributes & SEC_IMAGE)) + if (MemoryArea->VadNode.u.VadFlags.VadType == VadImageMap) { ULONG i; ULONG NrSegments; @@ -4172,8 +3540,8 @@ MiRosUnmapViewOfSection(IN PEPROCESS Process, PMM_SECTION_SEGMENT SectionSegments; PMM_SECTION_SEGMENT Segment; - Segment = MemoryArea->Data.SectionData.Segment; - ImageSectionObject = Section->ImageSection; + Segment = MemoryArea->SectionData.Segment; + ImageSectionObject = ImageSectionObjectFromSegment(Segment); SectionSegments = ImageSectionObject->Segments; NrSegments = ImageSectionObject->NrSegments; @@ -4219,12 +3587,10 @@ MiRosUnmapViewOfSection(IN PEPROCESS Process, } } - MmUnlockAddressSpace(AddressSpace); - /* Notify debugger */ if (ImageBaseAddress && !SkipDebuggerNotify) DbgkUnMapViewOfSection(ImageBaseAddress); - return(STATUS_SUCCESS); + return STATUS_SUCCESS; } @@ -4319,132 +3685,117 @@ NtQuerySection( return Status; } - if (MiIsRosSectionObject(Section)) + switch(SectionInformationClass) { - PROS_SECTION_OBJECT RosSection = (PROS_SECTION_OBJECT)Section; - - switch (SectionInformationClass) + case SectionBasicInformation: { - case SectionBasicInformation: + SECTION_BASIC_INFORMATION Sbi; + + Sbi.Size = Section->SizeOfSection; + Sbi.BaseAddress = (PVOID)Section->Address.StartingVpn; + + Sbi.Attributes = 0; + if (Section->u.Flags.File) + Sbi.Attributes |= SEC_FILE; + if (Section->u.Flags.Image) + Sbi.Attributes |= SEC_IMAGE; + + /* Those are not set ************* + if (Section->u.Flags.Commit) + Sbi.Attributes |= SEC_COMMIT; + if (Section->u.Flags.Reserve) + Sbi.Attributes |= SEC_RESERVE; + **********************************/ + + if (Section->u.Flags.Image) { - PSECTION_BASIC_INFORMATION Sbi = (PSECTION_BASIC_INFORMATION)SectionInformation; - - _SEH2_TRY + if (MiIsRosSectionObject(Section)) { - Sbi->Attributes = RosSection->AllocationAttributes; - if (RosSection->AllocationAttributes & SEC_IMAGE) - { - Sbi->BaseAddress = 0; - Sbi->Size.QuadPart = 0; - } - else - { - Sbi->BaseAddress = (PVOID)RosSection->Segment->Image.VirtualAddress; - Sbi->Size.QuadPart = RosSection->Segment->Length.QuadPart; - } - - if (ResultLength != NULL) - { - *ResultLength = sizeof(SECTION_BASIC_INFORMATION); - } - Status = STATUS_SUCCESS; - } - _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER) - { - Status = _SEH2_GetExceptionCode(); - } - _SEH2_END; - - break; - } - - case SectionImageInformation: - { - PSECTION_IMAGE_INFORMATION Sii = (PSECTION_IMAGE_INFORMATION)SectionInformation; - - _SEH2_TRY - { - if (RosSection->AllocationAttributes & SEC_IMAGE) - { - PMM_IMAGE_SECTION_OBJECT ImageSectionObject; - ImageSectionObject = RosSection->ImageSection; - - *Sii = ImageSectionObject->ImageInformation; - } - - if (ResultLength != NULL) - { - *ResultLength = sizeof(SECTION_IMAGE_INFORMATION); - } - Status = STATUS_SUCCESS; - } - _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER) - { - Status = _SEH2_GetExceptionCode(); - } - _SEH2_END; - - break; - } - } - } - else - { - switch(SectionInformationClass) - { - case SectionBasicInformation: - { - SECTION_BASIC_INFORMATION Sbi; - - Sbi.Size = Section->SizeOfSection; - Sbi.BaseAddress = (PVOID)Section->Address.StartingVpn; - - Sbi.Attributes = 0; - if (Section->u.Flags.Image) - Sbi.Attributes |= SEC_IMAGE; - if (Section->u.Flags.Commit) - Sbi.Attributes |= SEC_COMMIT; - if (Section->u.Flags.Reserve) - Sbi.Attributes |= SEC_RESERVE; - if (Section->u.Flags.File) - Sbi.Attributes |= SEC_FILE; - if (Section->u.Flags.Image) - Sbi.Attributes |= SEC_IMAGE; - - /* FIXME : Complete/test the list of flags passed back from NtCreateSection */ - - _SEH2_TRY - { - *((SECTION_BASIC_INFORMATION*)SectionInformation) = Sbi; - if (ResultLength) - *ResultLength = sizeof(Sbi); - } - _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER) - { - Status = _SEH2_GetExceptionCode(); - } - _SEH2_END; - break; - } - case SectionImageInformation: - { - if (!Section->u.Flags.Image) - { - Status = STATUS_SECTION_NOT_IMAGE; + PMM_IMAGE_SECTION_OBJECT ImageSectionObject = ((PMM_IMAGE_SECTION_OBJECT)Section->Segment); + Sbi.BaseAddress = 0; + Sbi.Size.QuadPart = ImageSectionObject->ImageInformation.ImageFileSize; } else { - /* Currently not supported */ + /* Not supported yet */ ASSERT(FALSE); } - break; } + else if (MiIsRosSectionObject(Section)) + { + Sbi.BaseAddress = (PVOID)((PMM_SECTION_SEGMENT)Section->Segment)->Image.VirtualAddress; + Sbi.Size.QuadPart = ((PMM_SECTION_SEGMENT)Section->Segment)->RawLength.QuadPart; + } + else + { + DPRINT1("Unimplemented code path!"); + } + + _SEH2_TRY + { + *((SECTION_BASIC_INFORMATION*)SectionInformation) = Sbi; + if (ResultLength != NULL) + { + *ResultLength = sizeof(Sbi); + } + } + _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER) + { + Status = _SEH2_GetExceptionCode(); + } + _SEH2_END; + break; } + case SectionImageInformation: + { + if (!Section->u.Flags.Image) + { + Status = STATUS_SECTION_NOT_IMAGE; + } + else if (MiIsRosSectionObject(Section)) + { + PMM_IMAGE_SECTION_OBJECT ImageSectionObject = ((PMM_IMAGE_SECTION_OBJECT)Section->Segment); + + _SEH2_TRY + { + PSECTION_IMAGE_INFORMATION Sii = (PSECTION_IMAGE_INFORMATION)SectionInformation; + *Sii = ImageSectionObject->ImageInformation; + if (ResultLength != NULL) + { + *ResultLength = sizeof(*Sii); + } + } + _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER) + { + Status = _SEH2_GetExceptionCode(); + } + _SEH2_END; + } + else + { + _SEH2_TRY + { + PSECTION_IMAGE_INFORMATION Sii = (PSECTION_IMAGE_INFORMATION)SectionInformation; + *Sii = *Section->Segment->u2.ImageInformation; + if (ResultLength != NULL) + *ResultLength = sizeof(*Sii); + } + _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER) + { + Status = _SEH2_GetExceptionCode(); + } + _SEH2_END; + } + break; + } + default: + DPRINT1("Unknown SectionInformationClass: %d\n", SectionInformationClass); + Status = STATUS_NOT_SUPPORTED; } ObDereferenceObject(Section); - return(Status); + return Status; } /********************************************************************** @@ -4508,9 +3859,8 @@ MmMapViewOfSection(IN PVOID SectionObject, IN ULONG AllocationType, IN ULONG Protect) { - PROS_SECTION_OBJECT Section; + PSECTION Section; PMMSUPPORT AddressSpace; - ULONG ViewOffset; NTSTATUS Status = STATUS_SUCCESS; BOOLEAN NotAtBase = FALSE; @@ -4539,14 +3889,15 @@ MmMapViewOfSection(IN PVOID SectionObject, /* FIXME: We should keep this, but it would break code checking equality */ Protect &= ~PAGE_NOCACHE; - Section = (PROS_SECTION_OBJECT)SectionObject; + Section = SectionObject; AddressSpace = &Process->Vm; - AllocationType |= (Section->AllocationAttributes & SEC_NO_CHANGE); + if (Section->u.Flags.NoChange) + AllocationType |= SEC_NO_CHANGE; MmLockAddressSpace(AddressSpace); - if (Section->AllocationAttributes & SEC_IMAGE) + if (Section->u.Flags.Image) { ULONG i; ULONG NrSegments; @@ -4555,10 +3906,12 @@ MmMapViewOfSection(IN PVOID SectionObject, PMM_IMAGE_SECTION_OBJECT ImageSectionObject; PMM_SECTION_SEGMENT SectionSegments; - ImageSectionObject = Section->ImageSection; + ImageSectionObject = ((PMM_IMAGE_SECTION_OBJECT)Section->Segment); SectionSegments = ImageSectionObject->Segments; NrSegments = ImageSectionObject->NrSegments; + ASSERT(ImageSectionObject->RefCount > 0); + ImageBase = (ULONG_PTR)*BaseAddress; if (ImageBase == 0) { @@ -4600,14 +3953,14 @@ MmMapViewOfSection(IN PVOID SectionObject, if ((*BaseAddress) != NULL) { MmUnlockAddressSpace(AddressSpace); - return(STATUS_CONFLICTING_ADDRESSES); + return STATUS_CONFLICTING_ADDRESSES; } /* Otherwise find a gap to map the image. */ ImageBase = (ULONG_PTR)MmFindGap(AddressSpace, PAGE_ROUND_UP(ImageSize), MM_VIRTMEM_GRANULARITY, FALSE); if (ImageBase == 0) { MmUnlockAddressSpace(AddressSpace); - return(STATUS_CONFLICTING_ADDRESSES); + return STATUS_CONFLICTING_ADDRESSES; } /* Remember that we loaded image at a different base address */ NotAtBase = TRUE; @@ -4619,18 +3972,27 @@ MmMapViewOfSection(IN PVOID SectionObject, ((char*)ImageBase + (ULONG_PTR)SectionSegments[i].Image.VirtualAddress); MmLockSectionSegment(&SectionSegments[i]); Status = MmMapViewOfSegment(AddressSpace, - Section, + TRUE, &SectionSegments[i], &SBaseAddress, - SectionSegments[i].Length.LowPart, + SectionSegments[i].Length.QuadPart, SectionSegments[i].Protection, 0, 0); MmUnlockSectionSegment(&SectionSegments[i]); if (!NT_SUCCESS(Status)) { + /* roll-back */ + while (i--) + { + SBaseAddress = ((char*)ImageBase + (ULONG_PTR)SectionSegments[i].Image.VirtualAddress); + MmLockSectionSegment(&SectionSegments[i]); + MmUnmapViewOfSegment(AddressSpace, SBaseAddress); + MmUnlockSectionSegment(&SectionSegments[i]); + } + MmUnlockAddressSpace(AddressSpace); - return(Status); + return Status; } } @@ -4639,23 +4001,28 @@ MmMapViewOfSection(IN PVOID SectionObject, } else { + PMM_SECTION_SEGMENT Segment = (PMM_SECTION_SEGMENT)Section->Segment; + LONGLONG ViewOffset; + + ASSERT(Segment->RefCount > 0); + /* check for write access */ if ((Protect & (PAGE_READWRITE|PAGE_EXECUTE_READWRITE)) && - !(Section->SectionPageProtection & (PAGE_READWRITE|PAGE_EXECUTE_READWRITE))) + !(Section->InitialPageProtection & (PAGE_READWRITE|PAGE_EXECUTE_READWRITE))) { MmUnlockAddressSpace(AddressSpace); return STATUS_SECTION_PROTECTION; } /* check for read access */ if ((Protect & (PAGE_READONLY|PAGE_WRITECOPY|PAGE_EXECUTE_READ|PAGE_EXECUTE_WRITECOPY)) && - !(Section->SectionPageProtection & (PAGE_READONLY|PAGE_READWRITE|PAGE_WRITECOPY|PAGE_EXECUTE_READ|PAGE_EXECUTE_READWRITE|PAGE_EXECUTE_WRITECOPY))) + !(Section->InitialPageProtection & (PAGE_READONLY|PAGE_READWRITE|PAGE_WRITECOPY|PAGE_EXECUTE_READ|PAGE_EXECUTE_READWRITE|PAGE_EXECUTE_WRITECOPY))) { MmUnlockAddressSpace(AddressSpace); return STATUS_SECTION_PROTECTION; } /* check for execute access */ if ((Protect & (PAGE_EXECUTE|PAGE_EXECUTE_READ|PAGE_EXECUTE_READWRITE|PAGE_EXECUTE_WRITECOPY)) && - !(Section->SectionPageProtection & (PAGE_EXECUTE|PAGE_EXECUTE_READ|PAGE_EXECUTE_READWRITE|PAGE_EXECUTE_WRITECOPY))) + !(Section->InitialPageProtection & (PAGE_EXECUTE|PAGE_EXECUTE_READ|PAGE_EXECUTE_READWRITE|PAGE_EXECUTE_WRITECOPY))) { MmUnlockAddressSpace(AddressSpace); return STATUS_SECTION_PROTECTION; @@ -4667,45 +4034,47 @@ MmMapViewOfSection(IN PVOID SectionObject, } else { - ViewOffset = SectionOffset->u.LowPart; + ViewOffset = SectionOffset->QuadPart; } if ((ViewOffset % PAGE_SIZE) != 0) { MmUnlockAddressSpace(AddressSpace); - return(STATUS_MAPPED_ALIGNMENT); + return STATUS_MAPPED_ALIGNMENT; } if ((*ViewSize) == 0) { - (*ViewSize) = Section->MaximumSize.u.LowPart - ViewOffset; + (*ViewSize) = Section->SizeOfSection.QuadPart - ViewOffset; } - else if (((*ViewSize)+ViewOffset) > Section->MaximumSize.u.LowPart) + else if ((ExGetPreviousMode() == UserMode) && + (((*ViewSize)+ViewOffset) > Section->SizeOfSection.QuadPart) && + (!Section->u.Flags.Reserve)) { - (*ViewSize) = Section->MaximumSize.u.LowPart - ViewOffset; + /* Dubious */ + (*ViewSize) = MIN(Section->SizeOfSection.QuadPart - ViewOffset, SIZE_T_MAX - PAGE_SIZE); } *ViewSize = PAGE_ROUND_UP(*ViewSize); - MmLockSectionSegment(Section->Segment); + MmLockSectionSegment(Segment); Status = MmMapViewOfSegment(AddressSpace, - Section, - Section->Segment, + FALSE, + Segment, BaseAddress, *ViewSize, Protect, ViewOffset, AllocationType & (MEM_TOP_DOWN|SEC_NO_CHANGE)); - MmUnlockSectionSegment(Section->Segment); + MmUnlockSectionSegment(Segment); if (!NT_SUCCESS(Status)) { MmUnlockAddressSpace(AddressSpace); - return(Status); + return Status; } } MmUnlockAddressSpace(AddressSpace); - ASSERT(*BaseAddress == ALIGN_DOWN_POINTER_BY(*BaseAddress, MM_VIRTMEM_GRANULARITY)); if (NotAtBase) Status = STATUS_IMAGE_NOT_AT_BASE; @@ -4722,6 +4091,9 @@ BOOLEAN NTAPI MmCanFileBeTruncated (IN PSECTION_OBJECT_POINTERS SectionObjectPointer, IN PLARGE_INTEGER NewFileSize) { + BOOLEAN Ret; + PMM_SECTION_SEGMENT Segment; + /* Check whether an ImageSectionObject exists */ if (SectionObjectPointer->ImageSectionObject != NULL) { @@ -4729,111 +4101,64 @@ MmCanFileBeTruncated (IN PSECTION_OBJECT_POINTERS SectionObjectPointer, return FALSE; } - if (SectionObjectPointer->DataSectionObject != NULL) + Segment = MiGrabDataSection(SectionObjectPointer); + if (!Segment) { - PMM_SECTION_SEGMENT Segment; - - Segment = (PMM_SECTION_SEGMENT)SectionObjectPointer-> - DataSectionObject; - - if (Segment->ReferenceCount != 0) - { -#ifdef NEWCC - CC_FILE_SIZES FileSizes; - CcpLock(); - if (SectionObjectPointer->SharedCacheMap && (Segment->ReferenceCount > CcpCountCacheSections((PNOCC_CACHE_MAP)SectionObjectPointer->SharedCacheMap))) - { - CcpUnlock(); - /* Check size of file */ - if (SectionObjectPointer->SharedCacheMap) - { - if (!CcGetFileSizes(Segment->FileObject, &FileSizes)) - { - return FALSE; - } - - if (NewFileSize->QuadPart <= FileSizes.FileSize.QuadPart) - { - return FALSE; - } - } - } - else - CcpUnlock(); -#else - /* Check size of file */ - if (SectionObjectPointer->SharedCacheMap) - { - PROS_SHARED_CACHE_MAP SharedCacheMap = SectionObjectPointer->SharedCacheMap; - if (NewFileSize->QuadPart <= SharedCacheMap->FileSize.QuadPart) - { - return FALSE; - } - } -#endif - } - else - { - /* Something must gone wrong - * how can we have a Section but no - * reference? */ - DPRINT("ERROR: DataSectionObject without reference!\n"); - } + /* There is no data section. It's fine to do anything. */ + return TRUE; } + MmLockSectionSegment(Segment); + if ((Segment->SectionCount == 0) || + ((Segment->SectionCount == 1) && (SectionObjectPointer->SharedCacheMap != NULL))) + { + /* If the cache is the only one holding a reference to the segment, then it's fine to resize */ + Ret = TRUE; + } + else + { + /* We can't shrink, but we can extend */ + Ret = NewFileSize->QuadPart >= Segment->RawLength.QuadPart; +#if DBG + if (!Ret) + { + DPRINT1("Cannot truncate data: New Size %I64d, Segment Size %I64d\n", NewFileSize->QuadPart, Segment->RawLength.QuadPart); + } +#endif + } + MmUnlockSectionSegment(Segment); + MmDereferenceSegment(Segment); + DPRINT("FIXME: didn't check for outstanding write probes\n"); - return TRUE; + return Ret; } - - - /* * @implemented */ BOOLEAN NTAPI MmFlushImageSection (IN PSECTION_OBJECT_POINTERS SectionObjectPointer, - IN MMFLUSH_TYPE FlushType) + IN MMFLUSH_TYPE FlushType) { - BOOLEAN Result = TRUE; -#ifdef NEWCC - PMM_SECTION_SEGMENT Segment; -#endif - switch(FlushType) { - case MmFlushForDelete: - if (SectionObjectPointer->ImageSectionObject || - SectionObjectPointer->DataSectionObject) + case MmFlushForDelete: + case MmFlushForWrite: { - return FALSE; - } -#ifndef NEWCC - CcRosRemoveIfClosed(SectionObjectPointer); -#endif - return TRUE; - case MmFlushForWrite: - { - DPRINT("MmFlushImageSection(%d)\n", FlushType); -#ifdef NEWCC - Segment = (PMM_SECTION_SEGMENT)SectionObjectPointer->DataSectionObject; -#endif + BOOLEAN Ret = TRUE; + KIRQL OldIrql = MiAcquirePfnLock(); - if (SectionObjectPointer->ImageSectionObject) - { - DPRINT1("SectionObject has ImageSection\n"); - return FALSE; - } + if (SectionObjectPointer->ImageSectionObject) + { + PMM_IMAGE_SECTION_OBJECT ImageSectionObject = SectionObjectPointer->ImageSectionObject; + if (!(ImageSectionObject->SegFlags & MM_SEGMENT_INDELETE)) + Ret = FALSE; + } -#ifdef NEWCC - CcpLock(); - Result = !SectionObjectPointer->SharedCacheMap || (Segment->ReferenceCount == CcpCountCacheSections((PNOCC_CACHE_MAP)SectionObjectPointer->SharedCacheMap)); - CcpUnlock(); - DPRINT("Result %d\n", Result); -#endif - return Result; - } + MiReleasePfnLock(OldIrql); + return Ret; + } } return FALSE; } @@ -4841,14 +4166,36 @@ MmFlushImageSection (IN PSECTION_OBJECT_POINTERS SectionObjectPointer, /* * @implemented */ -NTSTATUS NTAPI +NTSTATUS +NTAPI MmMapViewInSystemSpace (IN PVOID SectionObject, OUT PVOID * MappedBase, IN OUT PSIZE_T ViewSize) { - PROS_SECTION_OBJECT Section; + LARGE_INTEGER SectionOffset; + + SectionOffset.QuadPart = 0; + + return MmMapViewInSystemSpaceEx(SectionObject, MappedBase, ViewSize, &SectionOffset, 0); +} + +NTSTATUS +NTAPI +MmMapViewInSystemSpaceEx ( + _In_ PVOID SectionObject, + _Outptr_result_bytebuffer_ (*ViewSize) PVOID *MappedBase, + _Inout_ PSIZE_T ViewSize, + _Inout_ PLARGE_INTEGER SectionOffset, + _In_ ULONG_PTR Flags + ) +{ + PSECTION Section = SectionObject; + PMM_SECTION_SEGMENT Segment; PMMSUPPORT AddressSpace; NTSTATUS Status; + + UNREFERENCED_PARAMETER(Flags); + PAGED_CODE(); if (MiIsRosSectionObject(SectionObject) == FALSE) @@ -4856,62 +4203,83 @@ MmMapViewInSystemSpace (IN PVOID SectionObject, return MiMapViewInSystemSpace(SectionObject, &MmSession, MappedBase, - ViewSize); + ViewSize, + SectionOffset); } - DPRINT("MmMapViewInSystemSpace() called\n"); + DPRINT("MmMapViewInSystemSpaceEx() called\n"); + + Section = SectionObject; + Segment = (PMM_SECTION_SEGMENT)Section->Segment; + + if (*ViewSize == 0) + { + LONGLONG MapSizeLL; + + /* Page-align the mapping */ + SectionOffset->LowPart = PAGE_ROUND_DOWN(SectionOffset->LowPart); + + if (!NT_SUCCESS(RtlLongLongSub(Section->SizeOfSection.QuadPart, SectionOffset->QuadPart, &MapSizeLL))) + return STATUS_INVALID_VIEW_SIZE; + + if (!NT_SUCCESS(RtlLongLongToSIZET(MapSizeLL, ViewSize))) + return STATUS_INVALID_VIEW_SIZE; + } + else + { + LONGLONG HelperLL; + + /* Get the map end */ + if (!NT_SUCCESS(RtlLongLongAdd(SectionOffset->QuadPart, *ViewSize, &HelperLL))) + return STATUS_INVALID_VIEW_SIZE; + + /* Round it up, if needed */ + if (HelperLL % PAGE_SIZE) + { + if (!NT_SUCCESS(RtlLongLongAdd(HelperLL, PAGE_SIZE - (HelperLL % PAGE_SIZE), &HelperLL))) + return STATUS_INVALID_VIEW_SIZE; + } + + /* Now that we have the mapping end, we can align down its start */ + SectionOffset->LowPart = PAGE_ROUND_DOWN(SectionOffset->LowPart); + + /* Get the new size */ + if (!NT_SUCCESS(RtlLongLongSub(HelperLL, SectionOffset->QuadPart, &HelperLL))) + return STATUS_INVALID_VIEW_SIZE; + + if (!NT_SUCCESS(RtlLongLongToSIZET(HelperLL, ViewSize))) + return STATUS_INVALID_VIEW_SIZE; + } - Section = (PROS_SECTION_OBJECT)SectionObject; AddressSpace = MmGetKernelAddressSpace(); MmLockAddressSpace(AddressSpace); - - if ((*ViewSize) == 0) - { - (*ViewSize) = Section->MaximumSize.u.LowPart; - } - else if ((*ViewSize) > Section->MaximumSize.u.LowPart) - { - (*ViewSize) = Section->MaximumSize.u.LowPart; - } - - MmLockSectionSegment(Section->Segment); - + MmLockSectionSegment(Segment); Status = MmMapViewOfSegment(AddressSpace, - Section, - Section->Segment, + Section->u.Flags.Image, + Segment, MappedBase, *ViewSize, PAGE_READWRITE, - 0, - 0); + SectionOffset->QuadPart, + SEC_RESERVE); - MmUnlockSectionSegment(Section->Segment); + MmUnlockSectionSegment(Segment); MmUnlockAddressSpace(AddressSpace); return Status; } +/* This function must be called with adress space lock held */ NTSTATUS NTAPI MiRosUnmapViewInSystemSpace(IN PVOID MappedBase) { - PMMSUPPORT AddressSpace; - NTSTATUS Status; - DPRINT("MmUnmapViewInSystemSpace() called\n"); - AddressSpace = MmGetKernelAddressSpace(); - - MmLockAddressSpace(AddressSpace); - - Status = MmUnmapViewOfSegment(AddressSpace, MappedBase); - - MmUnlockAddressSpace(AddressSpace); - - return Status; + return MmUnmapViewOfSegment(MmGetKernelAddressSpace(), MappedBase); } /********************************************************************** @@ -4981,7 +4349,8 @@ MmCreateSection (OUT PVOID * Section, { NTSTATUS Status; ULONG Protection; - PROS_SECTION_OBJECT *SectionObject = (PROS_SECTION_OBJECT *)Section; + PSECTION *SectionObject = (PSECTION *)Section; + BOOLEAN FileLock = FALSE; /* Check if an ARM3 section is being created instead */ if (!(AllocationAttributes & (SEC_IMAGE | SEC_PHYSICALMEMORY))) @@ -5020,11 +4389,14 @@ MmCreateSection (OUT PVOID * Section, return STATUS_INVALID_PARAMETER_6; } - /* Did the caller pass an object? */ + /* Did the caller pass a file object ? */ if (FileObject) { /* Reference the object directly */ ObReferenceObject(FileObject); + + /* We don't create image mappings with file objects */ + AllocationAttributes &= ~SEC_IMAGE; } else { @@ -5040,6 +4412,29 @@ MmCreateSection (OUT PVOID * Section, DPRINT1("Failed to get a handle to the FO: %lx\n", Status); return Status; } + + /* Lock the file */ + Status = FsRtlAcquireToCreateMappedSection(FileObject, SectionPageProtection); + if (!NT_SUCCESS(Status)) + { + ObDereferenceObject(FileObject); + return Status; + } + + FileLock = TRUE; + + /* Deny access if there are writes on the file */ +#if 0 + if ((AllocationAttributes & SEC_IMAGE) && (Status == STATUS_FILE_LOCKED_WITH_WRITERS)) + { + DPRINT1("Cannot create image maps with writers open on the file!\n"); + Status = STATUS_ACCESS_DENIED; + goto Quit; + } +#else + if ((AllocationAttributes & SEC_IMAGE) && (Status == STATUS_FILE_LOCKED_WITH_WRITERS)) + DPRINT1("Creating image map with writers open on the file!\n"); +#endif } } else @@ -5048,63 +4443,6 @@ MmCreateSection (OUT PVOID * Section, if (AllocationAttributes & SEC_IMAGE) return STATUS_INVALID_FILE_FOR_SECTION; } -#ifndef NEWCC // A hack for initializing caching. - // This is needed only in the old case. - if (FileHandle) - { - IO_STATUS_BLOCK Iosb; - NTSTATUS Status; - CHAR Buffer; - LARGE_INTEGER ByteOffset; - ByteOffset.QuadPart = 0; - Status = ZwReadFile(FileHandle, - NULL, - NULL, - NULL, - &Iosb, - &Buffer, - sizeof(Buffer), - &ByteOffset, - NULL); - if (!NT_SUCCESS(Status) && Status != STATUS_END_OF_FILE) - { - DPRINT1("CC failure: %lx\n", Status); - if (FileObject) - ObDereferenceObject(FileObject); - return Status; - } - // Caching is initialized... - - // Hack of the hack: actually, it might not be initialized if FSD init on effective right and if file is null-size - // In such case, force cache by initiating a write IRP - if (Status == STATUS_END_OF_FILE && !(AllocationAttributes & SEC_IMAGE) && FileObject != NULL && - (FileObject->SectionObjectPointer == NULL || FileObject->SectionObjectPointer->SharedCacheMap == NULL)) - { - Buffer = 0xdb; - Status = ZwWriteFile(FileHandle, - NULL, - NULL, - NULL, - &Iosb, - &Buffer, - sizeof(Buffer), - &ByteOffset, - NULL); - if (NT_SUCCESS(Status)) - { - LARGE_INTEGER Zero; - Zero.QuadPart = 0LL; - - Status = IoSetInformation(FileObject, - FileEndOfFileInformation, - sizeof(LARGE_INTEGER), - &Zero); - ASSERT(NT_SUCCESS(Status)); - } - } - } -#endif - if (AllocationAttributes & SEC_IMAGE) { Status = MmCreateImageSection(SectionObject, @@ -5116,7 +4454,7 @@ MmCreateSection (OUT PVOID * Section, FileObject); } #ifndef NEWCC - else if (FileHandle != NULL) + else if (FileObject != NULL) { Status = MmCreateDataFileSection(SectionObject, DesiredAccess, @@ -5124,7 +4462,8 @@ MmCreateSection (OUT PVOID * Section, MaximumSize, SectionPageProtection, AllocationAttributes, - FileObject); + FileObject, + FileHandle != NULL); } #else else if (FileHandle != NULL || FileObject != NULL) @@ -5140,22 +4479,594 @@ MmCreateSection (OUT PVOID * Section, #endif else { - if ((AllocationAttributes & SEC_PHYSICALMEMORY) == 0) - { - DPRINT1("Invalid path: %lx %p %p\n", AllocationAttributes, FileObject, FileHandle); - } -// ASSERT(AllocationAttributes & SEC_PHYSICALMEMORY); - Status = MmCreatePageFileSection(SectionObject, - DesiredAccess, - ObjectAttributes, - MaximumSize, - SectionPageProtection, - AllocationAttributes); - if (FileObject) - ObDereferenceObject(FileObject); + /* All cases should be handled above */ + Status = STATUS_INVALID_PARAMETER; } + if (FileLock) + FsRtlReleaseFile(FileObject); + if (FileObject) + ObDereferenceObject(FileObject); + return Status; } +BOOLEAN +NTAPI +MmArePagesResident( + _In_ PEPROCESS Process, + _In_ PVOID Address, + _In_ ULONG Length) +{ + PMEMORY_AREA MemoryArea; + BOOLEAN Ret = TRUE; + PMM_SECTION_SEGMENT Segment; + LARGE_INTEGER SegmentOffset, RangeEnd; + PMMSUPPORT AddressSpace = Process ? &Process->Vm : MmGetKernelAddressSpace(); + + MmLockAddressSpace(AddressSpace); + + MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace, Address); + if (MemoryArea == NULL) + { + MmUnlockAddressSpace(AddressSpace); + return FALSE; + } + + /* Only supported in old Mm for now */ + ASSERT(MemoryArea->Type == MEMORY_AREA_SECTION_VIEW); + /* For file mappings */ + ASSERT(MemoryArea->VadNode.u.VadFlags.VadType != VadImageMap); + + Segment = MemoryArea->SectionData.Segment; + MmLockSectionSegment(Segment); + + SegmentOffset.QuadPart = PAGE_ROUND_DOWN(Address) - MA_GetStartingAddress(MemoryArea) + + MemoryArea->SectionData.ViewOffset; + RangeEnd.QuadPart = PAGE_ROUND_UP((ULONG_PTR)Address + Length) - MA_GetStartingAddress(MemoryArea) + + MemoryArea->SectionData.ViewOffset; + + while (SegmentOffset.QuadPart < RangeEnd.QuadPart) + { + ULONG_PTR Entry = MmGetPageEntrySectionSegment(Segment, &SegmentOffset); + if ((Entry == 0) || IS_SWAP_FROM_SSE(Entry)) + { + Ret = FALSE; + break; + } + SegmentOffset.QuadPart += PAGE_SIZE; + } + + MmUnlockSectionSegment(Segment); + + MmUnlockAddressSpace(AddressSpace); + return Ret; +} + +NTSTATUS +NTAPI +MmRosFlushVirtualMemory( + _In_ PEPROCESS Process, + _Inout_ PVOID* Address, + _Inout_ PSIZE_T Length, + _Out_ PIO_STATUS_BLOCK Iosb) +{ + PMEMORY_AREA MemoryArea; + PMM_SECTION_SEGMENT Segment; + LARGE_INTEGER SegmentOffset, RangeEnd; + PMMSUPPORT AddressSpace = Process ? &Process->Vm : MmGetKernelAddressSpace(); + PVOID CurrentAddress; + + PAGED_CODE(); + + MmLockAddressSpace(AddressSpace); + + DPRINT("Flushing Process %p at %p --> 0x%x", Process, *Address, *Length); + + MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace, *Address); + if ((MemoryArea == NULL) || (MemoryArea->Type != MEMORY_AREA_SECTION_VIEW) || + (MemoryArea->VadNode.u.VadFlags.VadType == VadImageMap)) + { + DPRINT1("Unable to find memory area at address %p.\n", Address); + MmUnlockAddressSpace(AddressSpace); + return STATUS_NOT_MAPPED_VIEW; + } + + Segment = MemoryArea->SectionData.Segment; + + SegmentOffset.QuadPart = PAGE_ROUND_DOWN(*Address) - MA_GetStartingAddress(MemoryArea) + + MemoryArea->SectionData.ViewOffset; + RangeEnd.QuadPart = PAGE_ROUND_UP((ULONG_PTR)*Address + *Length) - MA_GetStartingAddress(MemoryArea) + + MemoryArea->SectionData.ViewOffset; + + CurrentAddress = *Address; + + MmUnlockAddressSpace(AddressSpace); + + MmLockSectionSegment(Segment); + + Iosb->Information = 0; + while (SegmentOffset.QuadPart < RangeEnd.QuadPart) + { + ULONG_PTR Entry = MmGetPageEntrySectionSegment(Segment, &SegmentOffset); + + /* Let any pending read proceed */ + while (MM_IS_WAIT_PTE(Entry)) + { + MmUnlockSectionSegment(Segment); + MiWaitForPageEvent(NULL, NULL); + MmLockSectionSegment(Segment); + Entry = MmGetPageEntrySectionSegment(Segment, &SegmentOffset); + } + + /* We are called from Cc, this can't be backed by the page files */ + ASSERT(!IS_SWAP_FROM_SSE(Entry)); + + /* At this point, there may be a valid page there */ + if (Entry != 0) + { + /* This will write the page to disk, if needed */ + MmCheckDirtySegment(Segment, &SegmentOffset, Process ? MmIsDirtyPage(Process, CurrentAddress) : FALSE, FALSE); + Iosb->Information += PAGE_SIZE; + } + SegmentOffset.QuadPart += PAGE_SIZE; + CurrentAddress = (PVOID)((ULONG_PTR)CurrentAddress + PAGE_SIZE); + } + + MmUnlockSectionSegment(Segment); + + return STATUS_SUCCESS; +} + +/* Like CcPurgeCache but for the in-memory segment */ +BOOLEAN +NTAPI +MmPurgeSegment( + _In_ PSECTION_OBJECT_POINTERS SectionObjectPointer, + _In_opt_ PLARGE_INTEGER Offset, + _In_ ULONG Length) +{ + LARGE_INTEGER PurgeStart, PurgeEnd; + PMM_SECTION_SEGMENT Segment; + + Segment = MiGrabDataSection(SectionObjectPointer); + if (!Segment) + { + /* Nothing to purge */ + return STATUS_SUCCESS; + } + + PurgeStart.QuadPart = Offset ? Offset->QuadPart : 0LL; + if (Length && Offset) + { + if (!NT_SUCCESS(RtlLongLongAdd(PurgeStart.QuadPart, Length, &PurgeEnd.QuadPart))) + return FALSE; + } + + MmLockSectionSegment(Segment); + + if (!Length || !Offset) + { + /* We must calculate the length for ourselves */ + /* FIXME: All of this is suboptimal */ + ULONG ElemCount = RtlNumberGenericTableElements(&Segment->PageTable); + /* No page. Nothing to purge */ + if (!ElemCount) + { + MmUnlockSectionSegment(Segment); + MmDereferenceSegment(Segment); + return TRUE; + } + + PCACHE_SECTION_PAGE_TABLE PageTable = RtlGetElementGenericTable(&Segment->PageTable, ElemCount - 1); + PurgeEnd.QuadPart = PageTable->FileOffset.QuadPart + _countof(PageTable->PageEntries) * PAGE_SIZE; + } + + while (PurgeStart.QuadPart < PurgeEnd.QuadPart) + { + ULONG_PTR Entry = MmGetPageEntrySectionSegment(Segment, &PurgeStart); + + if (Entry == 0) + { + PurgeStart.QuadPart += PAGE_SIZE; + continue; + } + + if (IS_SWAP_FROM_SSE(Entry)) + { + ASSERT(SWAPENTRY_FROM_SSE(Entry) == MM_WAIT_ENTRY); + /* The page is currently being read. Meaning someone will need it soon. Bad luck */ + MmUnlockSectionSegment(Segment); + MmDereferenceSegment(Segment); + return FALSE; + } + + if (IS_WRITE_SSE(Entry)) + { + /* We're trying to purge an entry which is being written. Restart this loop iteration */ + MmUnlockSectionSegment(Segment); + KeDelayExecutionThread(KernelMode, FALSE, &TinyTime); + MmLockSectionSegment(Segment); + continue; + } + + if (SHARE_COUNT_FROM_SSE(Entry) > 0) + { + /* This page is currently in use. Bad luck */ + MmUnlockSectionSegment(Segment); + MmDereferenceSegment(Segment); + return FALSE; + } + + /* We can let this page go */ + MmSetPageEntrySectionSegment(Segment, &PurgeStart, 0); + MmReleasePageMemoryConsumer(MC_USER, PFN_FROM_SSE(Entry)); + + PurgeStart.QuadPart += PAGE_SIZE; + } + + /* This page is currently in use. Bad luck */ + MmUnlockSectionSegment(Segment); + MmDereferenceSegment(Segment); + return TRUE; +} + +NTSTATUS +NTAPI +MmMakeDataSectionResident( + _In_ PSECTION_OBJECT_POINTERS SectionObjectPointer, + _In_ LONGLONG Offset, + _In_ ULONG Length, + _In_ PLARGE_INTEGER ValidDataLength) +{ + PMM_SECTION_SEGMENT Segment = MiGrabDataSection(SectionObjectPointer); + + /* There must be a segment for this call */ + ASSERT(Segment); + + NTSTATUS Status = MmMakeSegmentResident(Segment, Offset, Length, ValidDataLength); + + MmDereferenceSegment(Segment); + + return Status; +} + +NTSTATUS +NTAPI +MmFlushSegment( + _In_ PSECTION_OBJECT_POINTERS SectionObjectPointer, + _In_opt_ PLARGE_INTEGER Offset, + _In_ ULONG Length, + _In_opt_ PIO_STATUS_BLOCK Iosb) +{ + LARGE_INTEGER FlushStart, FlushEnd; + NTSTATUS Status; + + if (Offset) + { + FlushStart = *Offset; + Status = RtlLongLongAdd(FlushStart.QuadPart, Length, &FlushEnd.QuadPart); + if (!NT_SUCCESS(Status)) + return Status; + } + + if (Iosb) + Iosb->Information = 0; + + PMM_SECTION_SEGMENT Segment = MiGrabDataSection(SectionObjectPointer); + if (!Segment) + { + /* Nothing to flush */ + if (Iosb) + Iosb->Status = STATUS_SUCCESS; + return STATUS_SUCCESS; + } + + ASSERT(*Segment->Flags & MM_DATAFILE_SEGMENT); + + MmLockSectionSegment(Segment); + + if (!Offset) + { + FlushStart.QuadPart = 0; + + /* FIXME: All of this is suboptimal */ + ULONG ElemCount = RtlNumberGenericTableElements(&Segment->PageTable); + /* No page. Nothing to flush */ + if (!ElemCount) + { + MmUnlockSectionSegment(Segment); + MmDereferenceSegment(Segment); + if (Iosb) + { + Iosb->Status = STATUS_SUCCESS; + Iosb->Information = 0; + } + return STATUS_SUCCESS; + } + + PCACHE_SECTION_PAGE_TABLE PageTable = RtlGetElementGenericTable(&Segment->PageTable, ElemCount - 1); + FlushEnd.QuadPart = PageTable->FileOffset.QuadPart + _countof(PageTable->PageEntries) * PAGE_SIZE; + } + + FlushStart.QuadPart >>= PAGE_SHIFT; + FlushStart.QuadPart <<= PAGE_SHIFT; + + while (FlushStart.QuadPart < FlushEnd.QuadPart) + { + ULONG_PTR Entry = MmGetPageEntrySectionSegment(Segment, &FlushStart); + + if (IS_DIRTY_SSE(Entry)) + { + MmCheckDirtySegment(Segment, &FlushStart, FALSE, FALSE); + + if (Iosb) + Iosb->Information += PAGE_SIZE; + } + + FlushStart.QuadPart += PAGE_SIZE; + } + + MmUnlockSectionSegment(Segment); + + MmDereferenceSegment(Segment); + + if (Iosb) + Iosb->Status = STATUS_SUCCESS; + + return STATUS_SUCCESS; +} + +_Requires_exclusive_lock_held_(Segment->Lock) +BOOLEAN +NTAPI +MmCheckDirtySegment( + PMM_SECTION_SEGMENT Segment, + PLARGE_INTEGER Offset, + BOOLEAN ForceDirty, + BOOLEAN PageOut) +{ + ULONG_PTR Entry; + NTSTATUS Status; + PFN_NUMBER Page; + + ASSERT(Segment->Locked); + + ASSERT((Offset->QuadPart % PAGE_SIZE) == 0); + + DPRINT("Checking segment for file %wZ at offset 0x%I64X.\n", &Segment->FileObject->FileName, Offset->QuadPart); + + Entry = MmGetPageEntrySectionSegment(Segment, Offset); + if (Entry == 0) + return FALSE; + + Page = PFN_FROM_SSE(Entry); + if ((IS_DIRTY_SSE(Entry)) || ForceDirty) + { + BOOLEAN DirtyAgain; + + /* + * We got a dirty entry. This path is for the shared data, + * be-it regular file maps or shared sections of DLLs + */ + ASSERT(!Segment->WriteCopy); + ASSERT(FlagOn(*Segment->Flags, MM_DATAFILE_SEGMENT) || FlagOn(Segment->Image.Characteristics, IMAGE_SCN_MEM_SHARED)); + + /* Insert the cleaned entry back. Mark it as write in progress, and clear the dirty bit. */ + Entry = MAKE_SSE(PAGE_FROM_SSE(Entry), SHARE_COUNT_FROM_SSE(Entry) + 1); + Entry = WRITE_SSE(Entry); + MmSetPageEntrySectionSegment(Segment, Offset, Entry); + + /* Tell the other users that we are clean again */ + MmSetCleanAllRmaps(Page); + + MmUnlockSectionSegment(Segment); + + if (FlagOn(*Segment->Flags, MM_DATAFILE_SEGMENT)) + { + /* We have to write it back to the file. Tell the FS driver who we are */ + if (PageOut) + IoSetTopLevelIrp((PIRP)FSRTL_MOD_WRITE_TOP_LEVEL_IRP); + + /* Go ahead and write the page */ + DPRINT("Writing page at offset %I64d for file %wZ, Pageout: %s\n", + Offset->QuadPart, &Segment->FileObject->FileName, PageOut ? "TRUE" : "FALSE"); + Status = MiWritePage(Segment, Offset->QuadPart, Page); + + if (PageOut) + IoSetTopLevelIrp(NULL); + } + else + { + /* This must only be called by the page-out path */ + ASSERT(PageOut); + + /* And this must be for a shared section in a DLL */ + ASSERT(Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED); + + SWAPENTRY SwapEntry = MmGetSavedSwapEntryPage(Page); + if (!SwapEntry) + { + SwapEntry = MmAllocSwapPage(); + } + + if (SwapEntry) + { + Status = MmWriteToSwapPage(SwapEntry, Page); + if (NT_SUCCESS(Status)) + { + MmSetSavedSwapEntryPage(Page, SwapEntry); + } + else + { + MmFreeSwapPage(SwapEntry); + } + } + else + { + DPRINT1("Failed to allocate a swap page!\n"); + Status = STATUS_INSUFFICIENT_RESOURCES; + } + } + + MmLockSectionSegment(Segment); + + /* Get the entry again */ + Entry = MmGetPageEntrySectionSegment(Segment, Offset); + ASSERT(PFN_FROM_SSE(Entry) == Page); + + if (!NT_SUCCESS(Status)) + { + /* Damn, this failed. Consider this page as still dirty */ + DPRINT1("MiWritePage FAILED: Status 0x%08x!\n", Status); + DirtyAgain = TRUE; + } + else + { + /* Check if someone dirtified this page while we were not looking */ + DirtyAgain = IS_DIRTY_SSE(Entry) || MmIsDirtyPageRmap(Page); + } + + /* Drop the reference we got, deleting the write altogether. */ + Entry = MAKE_SSE(Page << PAGE_SHIFT, SHARE_COUNT_FROM_SSE(Entry) - 1); + if (DirtyAgain) + { + Entry = DIRTY_SSE(Entry); + } + MmSetPageEntrySectionSegment(Segment, Offset, Entry); + } + + /* Were this page hanging there just for the sake of being present ? */ + if (!IS_DIRTY_SSE(Entry) && (SHARE_COUNT_FROM_SSE(Entry) == 0) && PageOut) + { + ULONG_PTR NewEntry = 0; + /* Restore the swap entry here */ + if (!FlagOn(*Segment->Flags, MM_DATAFILE_SEGMENT)) + { + SWAPENTRY SwapEntry = MmGetSavedSwapEntryPage(Page); + if (SwapEntry) + NewEntry = MAKE_SWAP_SSE(SwapEntry); + } + + /* Yes. Release it */ + MmSetPageEntrySectionSegment(Segment, Offset, NewEntry); + MmReleasePageMemoryConsumer(MC_USER, Page); + /* Tell the caller we released the page */ + return TRUE; + } + + return FALSE; +} + +NTSTATUS +NTAPI +MmMakePagesDirty( + _In_ PEPROCESS Process, + _In_ PVOID Address, + _In_ ULONG Length) +{ + PMEMORY_AREA MemoryArea; + PMM_SECTION_SEGMENT Segment; + LARGE_INTEGER SegmentOffset, RangeEnd; + PMMSUPPORT AddressSpace = Process ? &Process->Vm : MmGetKernelAddressSpace(); + + MmLockAddressSpace(AddressSpace); + + MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace, Address); + if (MemoryArea == NULL) + { + DPRINT1("Unable to find memory area at address %p.\n", Address); + MmUnlockAddressSpace(AddressSpace); + return STATUS_NOT_MAPPED_VIEW; + } + + /* Only supported in old Mm for now */ + ASSERT(MemoryArea->Type == MEMORY_AREA_SECTION_VIEW); + /* For file mappings */ + ASSERT(MemoryArea->VadNode.u.VadFlags.VadType != VadImageMap); + + Segment = MemoryArea->SectionData.Segment; + MmLockSectionSegment(Segment); + + SegmentOffset.QuadPart = PAGE_ROUND_DOWN(Address) - MA_GetStartingAddress(MemoryArea) + + MemoryArea->SectionData.ViewOffset; + RangeEnd.QuadPart = PAGE_ROUND_UP((ULONG_PTR)Address + Length) - MA_GetStartingAddress(MemoryArea) + + MemoryArea->SectionData.ViewOffset; + + DPRINT("MmMakePagesResident: Segment %p, 0x%I64x -> 0x%I64x\n", Segment, SegmentOffset.QuadPart, RangeEnd.QuadPart); + + while (SegmentOffset.QuadPart < RangeEnd.QuadPart) + { + ULONG_PTR Entry = MmGetPageEntrySectionSegment(Segment, &SegmentOffset); + + /* Let any pending read proceed */ + while (MM_IS_WAIT_PTE(Entry)) + { + MmUnlockSectionSegment(Segment); + MmUnlockAddressSpace(AddressSpace); + MiWaitForPageEvent(NULL, NULL); + MmLockAddressSpace(AddressSpace); + MmLockSectionSegment(Segment); + Entry = MmGetPageEntrySectionSegment(Segment, &SegmentOffset); + } + + /* We are called from Cc, this can't be backed by the page files */ + ASSERT(!IS_SWAP_FROM_SSE(Entry)); + + /* If there is no page there, there is nothing to make dirty */ + if (Entry != 0) + { + /* Dirtify the entry */ + MmSetPageEntrySectionSegment(Segment, &SegmentOffset, DIRTY_SSE(Entry)); + } + + SegmentOffset.QuadPart += PAGE_SIZE; + } + + MmUnlockSectionSegment(Segment); + + MmUnlockAddressSpace(AddressSpace); + return STATUS_SUCCESS; +} + +NTSTATUS +NTAPI +MmExtendSection( + _In_ PVOID _Section, + _Inout_ PLARGE_INTEGER NewSize) +{ + PSECTION Section = _Section; + + /* It makes no sense to extend an image mapping */ + if (Section->u.Flags.Image) + return STATUS_SECTION_NOT_EXTENDED; + + /* Nor is it possible to extend a page file mapping */ + if (!Section->u.Flags.File) + return STATUS_SECTION_NOT_EXTENDED; + + if (!MiIsRosSectionObject(Section)) + return STATUS_NOT_IMPLEMENTED; + + /* We just extend the sizes. Shrinking is a no-op ? */ + if (NewSize->QuadPart > Section->SizeOfSection.QuadPart) + { + PMM_SECTION_SEGMENT Segment = (PMM_SECTION_SEGMENT)Section->Segment; + Section->SizeOfSection = *NewSize; + + if (!Section->u.Flags.Reserve) + { + MmLockSectionSegment(Segment); + if (Segment->RawLength.QuadPart < NewSize->QuadPart) + { + Segment->RawLength = *NewSize; + Segment->Length.QuadPart = (NewSize->QuadPart + PAGE_SIZE - 1) & ~((LONGLONG)PAGE_SIZE); + } + MmUnlockSectionSegment(Segment); + } + } + + return STATUS_SUCCESS; +} + /* EOF */ diff --git a/ntoskrnl/mm/shutdown.c b/ntoskrnl/mm/shutdown.c index c80cc527a30..89beca88fac 100644 --- a/ntoskrnl/mm/shutdown.c +++ b/ntoskrnl/mm/shutdown.c @@ -21,6 +21,44 @@ VOID MiShutdownSystem(VOID) { ULONG i; + PFN_NUMBER Page; + BOOLEAN Dirty; + + /* Loop through all the pages owned by the legacy Mm and page them out, if needed. */ + /* We do it twice, since flushing can cause the FS to dirtify new pages */ + do + { + Dirty = FALSE; + + Page = MmGetLRUFirstUserPage(); + while (Page) + { + LARGE_INTEGER SegmentOffset; + PMM_SECTION_SEGMENT Segment = MmGetSectionAssociation(Page, &SegmentOffset); + + if (Segment) + { + if ((*Segment->Flags) & MM_DATAFILE_SEGMENT) + { + MmLockSectionSegment(Segment); + + ULONG_PTR Entry = MmGetPageEntrySectionSegment(Segment, &SegmentOffset); + + if (!IS_SWAP_FROM_SSE(Entry) && IS_DIRTY_SSE(Entry)) + { + Dirty = TRUE; + MmCheckDirtySegment(Segment, &SegmentOffset, FALSE, TRUE); + } + + MmUnlockSectionSegment(Segment); + } + + MmDereferenceSegment(Segment); + } + + Page = MmGetLRUNextUserPage(Page, FALSE); + } + } while (Dirty); /* Loop through all the paging files */ for (i = 0; i < MmNumberOfPagingFiles; i++) diff --git a/ntoskrnl/ntos.cmake b/ntoskrnl/ntos.cmake index 79317e880db..2a098336288 100644 --- a/ntoskrnl/ntos.cmake +++ b/ntoskrnl/ntos.cmake @@ -28,7 +28,11 @@ if(NEWCC) ${REACTOS_SOURCE_DIR}/ntoskrnl/cache/lazyrite.c ${REACTOS_SOURCE_DIR}/ntoskrnl/cache/logsup.c ${REACTOS_SOURCE_DIR}/ntoskrnl/cache/mdlsup.c - ${REACTOS_SOURCE_DIR}/ntoskrnl/cache/pinsup.c) + ${REACTOS_SOURCE_DIR}/ntoskrnl/cache/pinsup.c + ${REACTOS_SOURCE_DIR}/ntoskrnl/cache/section/fault.c + ${REACTOS_SOURCE_DIR}/ntoskrnl/cache/section/swapout.c + ${REACTOS_SOURCE_DIR}/ntoskrnl/cache/section/data.c + ${REACTOS_SOURCE_DIR}/ntoskrnl/cache/section/reqtools.c) else() list(APPEND SOURCE ${REACTOS_SOURCE_DIR}/ntoskrnl/cc/cacheman.c @@ -42,11 +46,7 @@ endif() list(APPEND SOURCE ${REACTOS_SOURCE_DIR}/ntoskrnl/cache/section/io.c - ${REACTOS_SOURCE_DIR}/ntoskrnl/cache/section/data.c - ${REACTOS_SOURCE_DIR}/ntoskrnl/cache/section/fault.c - ${REACTOS_SOURCE_DIR}/ntoskrnl/cache/section/reqtools.c ${REACTOS_SOURCE_DIR}/ntoskrnl/cache/section/sptab.c - ${REACTOS_SOURCE_DIR}/ntoskrnl/cache/section/swapout.c ${REACTOS_SOURCE_DIR}/ntoskrnl/config/cmalloc.c ${REACTOS_SOURCE_DIR}/ntoskrnl/config/cmapi.c ${REACTOS_SOURCE_DIR}/ntoskrnl/config/cmboot.c diff --git a/ntoskrnl/po/power.c b/ntoskrnl/po/power.c index 9490da817ce..125edbc11df 100644 --- a/ntoskrnl/po/power.c +++ b/ntoskrnl/po/power.c @@ -4,7 +4,7 @@ * FILE: ntoskrnl/po/power.c * PURPOSE: Power Manager * PROGRAMMERS: Casper S. Hornstrup (chorns@users.sourceforge.net) - * Hervé Poussineau (hpoussin@reactos.com) + * Herv� Poussineau (hpoussin@reactos.com) */ /* INCLUDES ******************************************************************/ @@ -1072,18 +1072,19 @@ NtSetSystemPowerState(IN POWER_ACTION SystemAction, /* Check if we're still in an invalid status */ if (!NT_SUCCESS(Status)) break; + /* Flush all volumes and the registry */ + DPRINT("Flushing volumes\n"); + PopFlushVolumes(PopAction.Shutdown); + #ifndef NEWCC /* Flush dirty cache pages */ /* XXX: Is that still mandatory? As now we'll wait on lazy writer to complete? */ - CcRosFlushDirtyPages(-1, &Dummy, FALSE, FALSE); //HACK: We really should wait here! + CcRosFlushDirtyPages(MAXULONG, &Dummy, TRUE, FALSE); + DPRINT("Cache flushed %lu pages\n", Dummy); #else Dummy = 0; #endif - /* Flush all volumes and the registry */ - DPRINT("Flushing volumes, cache flushed %lu pages\n", Dummy); - PopFlushVolumes(PopAction.Shutdown); - /* Set IRP for drivers */ PopAction.IrpMinor = IRP_MN_SET_POWER; if (PopAction.Shutdown) diff --git a/ntoskrnl/ps/process.c b/ntoskrnl/ps/process.c index 179fd1d8fa4..0e6f112a53b 100644 --- a/ntoskrnl/ps/process.c +++ b/ntoskrnl/ps/process.c @@ -358,7 +358,7 @@ PspCreateProcess(OUT PHANDLE ProcessHandle, PEPROCESS Process, Parent; PVOID ExceptionPortObject; PDEBUG_OBJECT DebugObject; - PSECTION_OBJECT SectionObject; + PSECTION SectionObject; NTSTATUS Status, AccessStatus; ULONG_PTR DirectoryTableBase[2] = {0,0}; KAFFINITY Affinity; diff --git a/sdk/include/ndk/mmtypes.h b/sdk/include/ndk/mmtypes.h index ca62ebdf5ce..3e9fabf1c02 100644 --- a/sdk/include/ndk/mmtypes.h +++ b/sdk/include/ndk/mmtypes.h @@ -623,19 +623,6 @@ typedef struct _SEGMENT_OBJECT PMMSUBSECTION_FLAGS MmSubSectionFlags; } SEGMENT_OBJECT, *PSEGMENT_OBJECT; -// -// Section Object -// -typedef struct _SECTION_OBJECT -{ - PVOID StartingVa; - PVOID EndingVa; - PVOID Parent; - PVOID LeftChild; - PVOID RightChild; - PSEGMENT_OBJECT Segment; -} SECTION_OBJECT, *PSECTION_OBJECT; - // // Generic Address Range Structure //