diff --git a/src/kernel/memory.cpp b/src/kernel/memory.cpp index 3eb86119..246f0d12 100644 --- a/src/kernel/memory.cpp +++ b/src/kernel/memory.cpp @@ -72,6 +72,9 @@ constexpr uint64_t DEFAULT_FLEXIBLE_MEMORY_SIZE = 1ull * 1024ull * 1024ull * 102 static uint64_t g_flexible_memory_size = DEFAULT_FLEXIBLE_MEMORY_SIZE; static bool g_flexible_memory_size_frozen = false; static Graphics::RenderContext* g_gpu_resources = nullptr; +#if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS) +static void (*g_test_gpu_range)(bool, uint64_t, size_t) = nullptr; +#endif static Graphics::RenderContext& GetGpuResources() { EXIT_IF(g_gpu_resources == nullptr); @@ -83,6 +86,9 @@ static bool IsGpuAddressRange(uint64_t vaddr, uint64_t size) { } static void MapGpuRange(uint64_t vaddr, uint64_t size) { +#if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS) + if (g_test_gpu_range != nullptr) g_test_gpu_range(true, vaddr, size); +#endif if (g_gpu_resources == nullptr || !IsGpuAddressRange(vaddr, size)) { return; } @@ -90,6 +96,9 @@ static void MapGpuRange(uint64_t vaddr, uint64_t size) { } static void UnmapGpuRange(uint64_t vaddr, uint64_t size) { +#if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS) + if (g_test_gpu_range != nullptr) g_test_gpu_range(false, vaddr, size); +#endif if (g_gpu_resources == nullptr || !IsGpuAddressRange(vaddr, size)) { return; } @@ -2528,20 +2537,7 @@ static bool FreeGuestMemoryOwner(uint64_t vaddr, uint64_t size) { g_virtual_ranges->Remove(vaddr, size); } -static int UnmapMemoryRange(uint64_t vaddr, size_t len) { - if (len == 0 || UINT64_MAX - vaddr < len) { - return KERNEL_ERROR_EINVAL; - } - - VirtualRanges::Range range {}; - if (!g_virtual_ranges->Query(vaddr, 0, &range)) { - return KERNEL_ERROR_EACCES; - } - const auto chunk_len = std::min(len, range.size - (vaddr - range.start)); - if (chunk_len < len) { - const int ret = UnmapMemoryRange(vaddr, chunk_len); - return ret == OK ? UnmapMemoryRange(vaddr + chunk_len, len - chunk_len) : ret; - } +static int UnmapMemoryOwner(const VirtualRanges::Range& range, uint64_t vaddr, size_t len) { if (IsReservedRangeType(range.type)) { if (!g_guest_address_space->ReleaseFree(vaddr, len)) { return KERNEL_ERROR_EACCES; @@ -2605,6 +2601,13 @@ static int UnmapMemoryRange(uint64_t vaddr, size_t len) { return OK; } +static int UnmapMemoryRange(const VirtualRanges::Range& range, uint64_t vaddr, size_t len) { + UnmapGpuRange(vaddr, len); + const int result = UnmapMemoryOwner(range, vaddr, len); + if (result != OK && IsCommittedRangeType(range.type)) MapGpuRange(vaddr, len); + return result; +} + // The ABI entry point and batch operations hold g_memory_operation_mutex. static int UnmapMemory(uint64_t vaddr, size_t len) { LOGF("\t start = 0x%016" PRIx64 "\n" @@ -2618,8 +2621,11 @@ static int UnmapMemory(uint64_t vaddr, size_t len) { if (!g_virtual_ranges->QuerySpan(vaddr, len, &ranges)) { return KERNEL_ERROR_EACCES; } - UnmapGpuRange(vaddr, len); - return UnmapMemoryRange(vaddr, len); + for (const auto& range: ranges) { + const int result = UnmapMemoryRange(range, range.start, range.size); + if (result != OK) return result; + } + return OK; } int KYTY_SYSV_ABI KernelMunmap(uint64_t vaddr, size_t len) { @@ -2628,6 +2634,31 @@ int KYTY_SYSV_ABI KernelMunmap(uint64_t vaddr, size_t len) { return UnmapMemory(vaddr, len); } +static bool IsAmmMemoryRange(uint64_t vaddr, size_t size) { + constexpr uint64_t page_size = 0x4000; + return size != 0 && ((vaddr | size) & (page_size - 1)) == 0 && + vaddr >= kExtendedMemoryBase && vaddr < kExtendedMemoryBase + kExtendedMemorySize && + size <= kExtendedMemoryBase + kExtendedMemorySize - vaddr; +} + +int UnmapAmmMemory(uint64_t vaddr, size_t size) { + std::lock_guard memory_operation_lock(g_memory_operation_mutex); + if (!IsAmmMemoryRange(vaddr, size)) return KERNEL_ERROR_EINVAL; + + const auto end = vaddr + size; + while (vaddr < end) { + VirtualRanges::Range range {}; + // AMM unmap accepts holes, including a completely unmapped range. + if (!g_virtual_ranges->Query(vaddr, 1, &range) || range.start >= end) break; + vaddr = std::max(vaddr, range.start); + const auto chunk = std::min(end, range.start + range.size) - vaddr; + const int result = UnmapMemoryRange(range, vaddr, chunk); + if (result != OK) return result; + vaddr += chunk; + } + return OK; +} + size_t KYTY_SYSV_ABI KernelGetDirectMemorySize() { PRINT_NAME(); @@ -3229,11 +3260,7 @@ int KYTY_SYSV_ABI KernelMapDirectMemory2(void** addr, size_t len, int type, int int MapAutomaticMemory(uint64_t vaddr, size_t size, int type, int prot) { std::lock_guard memory_operation_lock(g_memory_operation_mutex); constexpr uint64_t page_size = 0x4000; - if (size == 0 || (vaddr & (page_size - 1)) != 0 || (size & (page_size - 1)) != 0 || - vaddr < kExtendedMemoryBase || vaddr >= kExtendedMemoryBase + kExtendedMemorySize || - size > kExtendedMemoryBase + kExtendedMemorySize - vaddr) { - return KERNEL_ERROR_EINVAL; - } + if (!IsAmmMemoryRange(vaddr, size)) return KERNEL_ERROR_EINVAL; PhysicalMemory::PhysicalRanges ranges; if (!g_physical_memory->ReserveAutomatic(size, &ranges)) { @@ -3571,6 +3598,10 @@ int KYTY_SYSV_ABI KernelReserveVirtualRange(void** addr, size_t len, int flags, } #if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS) +void TestSetGpuRangeCallback(void (*callback)(bool, uint64_t, size_t)) { + g_test_gpu_range = callback; +} + void TestBeforeNextBackingMap(callback_func_t callback) { g_test_before_backing_map = callback; } diff --git a/src/kernel/memory.h b/src/kernel/memory.h index c3eb032a..af1231f0 100644 --- a/src/kernel/memory.h +++ b/src/kernel/memory.h @@ -112,6 +112,7 @@ void SetFlexibleMemorySize(uint64_t size); int AllocateDirectMemory(int64_t search_start, int64_t search_end, size_t size, size_t alignment, int memory_type, int64_t* phys_addr_out, bool automatic = false); int MapAutomaticMemory(uint64_t vaddr, size_t size, int type, int prot); +int UnmapAmmMemory(uint64_t vaddr, size_t size); bool TryWriteBacking(uint64_t vaddr, const void* data, uint64_t size); bool TryReadBacking(uint64_t vaddr, void* data, uint64_t size); bool TryReadBufferBacking(uint64_t vaddr, void* data, uint64_t size); @@ -192,6 +193,7 @@ bool ProtectGuestHostMemory(uint64_t vaddr, uint64_t size, Common::VirtualMemory bool FreeGuestMemory(uint64_t vaddr, uint64_t size); #if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS) +void TestSetGpuRangeCallback(void (*callback)(bool mapped, uint64_t addr, size_t size)); void TestBeforeNextBackingMap(callback_func_t callback); void TestSetBackingReadCallback(callback_func_t callback); void TestFailNextPhysicalMemoryUnmap(); diff --git a/src/libs/libAmpr.cpp b/src/libs/libAmpr.cpp index fc4ae4e7..276c76a8 100644 --- a/src/libs/libAmpr.cpp +++ b/src/libs/libAmpr.cpp @@ -1211,7 +1211,7 @@ static int ExecuteCommand(const CommandBufferState::Command& entry) { const auto& command = *payload; int result = OK; if (command.kind == AmmCommandKind::Unmap) { - result = LibKernel::Memory::KernelMunmap(command.va, command.size); + result = LibKernel::Memory::UnmapAmmMemory(command.va, command.size); } else { result = ExecuteAmmMapCommand(command); } diff --git a/tests/VirtualMemoryAllocationTests.cpp b/tests/VirtualMemoryAllocationTests.cpp index db066e01..369b9088 100644 --- a/tests/VirtualMemoryAllocationTests.cpp +++ b/tests/VirtualMemoryAllocationTests.cpp @@ -109,6 +109,18 @@ struct TestFailure {}; int g_failed_tests = 0; +struct GpuRangeEvent { + bool mapped; + uint64_t address; + size_t size; + bool operator==(const GpuRangeEvent&) const = default; +}; +std::vector g_gpu_range_events; + +void ObserveGpuRange(bool mapped, uint64_t address, size_t size) { + g_gpu_range_events.push_back({mapped, address, size}); +} + [[noreturn]] void Fail(const char* test, const std::string& message) { std::fflush(stdout); std::fprintf(stderr, "VirtualMemoryAllocationTests: %s failed: %s\n", test, message.c_str()); @@ -2718,6 +2730,14 @@ void TestAutomaticMemoryReuseAndAliases() { constexpr uint64_t base = kExtendedMemoryBase + 0x20000000; const auto end = static_cast(KernelGetDirectMemorySize()); int64_t physical = -1; + CheckOk(test, UnmapAmmMemory(base, block), "AMM unmap empty range"); + Check(test, KernelMunmap(base, block) == Libs::LibKernel::KERNEL_ERROR_EACCES, + "kernel unmap unexpectedly accepted an empty range"); + for (const auto [address, size] : {std::pair{base + 1, page}, {base, page - 1}, + {kExtendedMemoryBase - page, page}, + {kExtendedMemoryBase + kExtendedMemorySize - page, page * 2}}) + Check(test, UnmapAmmMemory(address, size) == Libs::LibKernel::KERNEL_ERROR_EINVAL, + "AMM unmap accepted an unaligned or out-of-domain range"); CheckOk(test, AllocateDirectMemory(0, end, block, block, 0, &physical, true), "allocate automatic backing"); CheckOk(test, MapAutomaticMemory(base, block, SceKernelMtypeC, SceKernelProtCpuRw), @@ -2740,7 +2760,18 @@ void TestAutomaticMemoryReuseAndAliases() { MapAutomaticMemory(base + block * 4, page, SceKernelMtypeC, SceKernelProtCpuRw) == Libs::LibKernel::KERNEL_ERROR_EAGAIN, "restored mappings remained available to the automatic allocator"); - CheckOk(test, KernelMunmap(base, block), "KernelMunmap(original)"); + TestFailPhysicalMemoryUnmapAfter(0); + g_gpu_range_events.clear(); + TestSetGpuRangeCallback(ObserveGpuRange); + const int unmap_result = UnmapAmmMemory(base - page, block + page * 2); + TestSetGpuRangeCallback(nullptr); + CheckFailed(test, unmap_result, "AMM unmap owner failure"); + Check(test, g_gpu_range_events == std::vector{{false, base, block}, {true, base, block}}, + "failed AMM unmap did not restore exactly the surviving GPU range"); + Check(test, *reinterpret_cast(base + page * 3) == 0x0123456789abcdefull, + "failed AMM unmap did not restore the original backing"); + CheckOk(test, UnmapAmmMemory(base - page, block + page * 2), + "AMM unmap original with leading and trailing holes"); Check(test, MapAutomaticMemory(base, block, SceKernelMtypeC, SceKernelProtCpuRw) == Libs::LibKernel::KERNEL_ERROR_EAGAIN, @@ -2751,20 +2782,22 @@ void TestAutomaticMemoryReuseAndAliases() { Query(test, base).offset == static_cast(physical) && Query(test, base + page).offset == static_cast(physical + page * 4), "automatic allocation did not skip the union of overlapping aliases"); - CheckOk(test, KernelMunmap(reinterpret_cast(first_alias), page * 3), - "KernelMunmap(first alias)"); + CheckOk(test, UnmapAmmMemory(reinterpret_cast(first_alias), page * 3), + "AMM unmap first alias"); CheckOk(test, MapAutomaticMemory(base + block * 4, page, SceKernelMtypeC, SceKernelProtCpuRw), "MapAutomaticMemory(partial alias release)"); Check(test, Query(test, base + block * 4).offset == static_cast(physical + page), "partial alias release did not reclaim its unreferenced page"); - CheckOk(test, KernelMunmap(reinterpret_cast(second_alias), page), - "KernelMunmap(partial second alias)"); + CheckOk(test, UnmapAmmMemory(reinterpret_cast(first_alias), block + page), + "AMM unmap holes and partial second alias"); CheckOk(test, MapAutomaticMemory(base + block * 5, page, SceKernelMtypeC, SceKernelProtCpuRw), "MapAutomaticMemory(partial unmap)"); Check(test, Query(test, base + block * 5).offset == static_cast(physical + page * 2), "partial unmap did not reclaim the matching physical page"); Check(test, *reinterpret_cast(base + block * 3 + page) == 0x0123456789abcdefull, "automatic reuse changed a surviving alias"); + CheckOk(test, UnmapAmmMemory(base + block * 4, block + page), "AMM unmap multiple sparse mappings"); + CheckOk(test, UnmapAmmMemory(base + block * 4, block + page), "AMM repeat unmap after final release"); CheckOk(test, KernelCheckedReleaseDirectMemory(physical, block), "KernelCheckedReleaseDirectMemory(donation)"); ExpectUnmapped(test, base); @@ -2826,6 +2859,17 @@ void TestAutomaticMemoryFragmentedMapRollback() { "automatic allocation beyond donated capacity"); CheckOk(test, MapAutomaticMemory(base, block * 2, SceKernelMtypeC, SceKernelProtCpuRw), "reuse both donations after final alias removal"); + TestFailPhysicalMemoryUnmapAfter(1); + g_gpu_range_events.clear(); + TestSetGpuRangeCallback(ObserveGpuRange); + const int unmap_result = KernelMunmap(base, block * 2); + TestSetGpuRangeCallback(nullptr); + CheckFailed(test, unmap_result, "unmap second backing failure"); + ExpectUnmapped(test, base); + Check(test, Query(test, base + block).offset == static_cast(second) && + g_gpu_range_events == std::vector{ + {false, base, block}, {false, base + block, block}, {true, base + block, block}}, + "failed strict unmap restored a freed prefix or lost the surviving GPU range"); CheckOk(test, KernelCheckedReleaseDirectMemory(first, block), "release first donation"); CheckOk(test, KernelCheckedReleaseDirectMemory(gap, block), "release ordinary allocation"); CheckOk(test, KernelCheckedReleaseDirectMemory(second, block), "release second donation");