From df3d027ebe56d81f71c25f0f0007f7245091c936 Mon Sep 17 00:00:00 2001 From: Mouad-Scriptz-0 Date: Mon, 12 Oct 2026 01:35:55 +0000 Subject: [PATCH] perf(libkernel): find guest allocations and direct mappings by address sceKernelVirtualQuery, sceKernelMunmap, sceKernelReleaseDirectMemory and the reservation paths walked a copy of the whole allocation registry or of every direct mapping under one lock on each call. The registry now answers lookups from its ordered map, applies range replacements as patches instead of rebuilding the map, and direct mappings carry an index by physical offset so a release finds only the mappings it covers. Minecraft (PPSA17221) holds about 7,000 direct mappings and spent its world generation in these walks. Co-Authored-By: Claude Fable 5.1 --- .../prx/libc/include/GuestAllocations.hpp | 1 + core/libs/prx/libc/src/GuestAllocations.cpp | 81 ++++++++++++++----- .../libkernel/DirectMemory/DirectMemory.cpp | 40 +++++---- .../prx/libkernel/DirectMemory/Export.cpp | 10 +-- core/libs/tests/GuestMemory.cpp | 53 ++++++++++++ 5 files changed, 136 insertions(+), 49 deletions(-) diff --git a/core/libs/prx/libc/include/GuestAllocations.hpp b/core/libs/prx/libc/include/GuestAllocations.hpp index e016be850..231826bb9 100644 --- a/core/libs/prx/libc/include/GuestAllocations.hpp +++ b/core/libs/prx/libc/include/GuestAllocations.hpp @@ -69,6 +69,7 @@ void GuestAllocationsProtect_nid_postfix(void* mutation, const void* pointer, st void GuestAllocationsUnmap_nid_postfix(void* mutation, const void* pointer, std::size_t bytes, const std::function& apply); Lease GuestAllocationsAcquire_nid_postfix(); Lease GuestAllocationsAcquireAll_nid_postfix(); +std::shared_ptr GuestAllocationsLookup_nid_postfix(std::uintptr_t address, bool next); // Changes whenever a mutation ends, so callers can cache facts about guest mappings between changes. std::uint64_t GuestAllocationsGeneration_nid_postfix(); // Precise invalidation for such caches: the callback runs, after the generation changed, for every diff --git a/core/libs/prx/libc/src/GuestAllocations.cpp b/core/libs/prx/libc/src/GuestAllocations.cpp index 33791e180..1faac8313 100644 --- a/core/libs/prx/libc/src/GuestAllocations.cpp +++ b/core/libs/prx/libc/src/GuestAllocations.cpp @@ -382,10 +382,14 @@ bool GuestAllocationsCovers_nid_postfix(void*, const void* pointer, std::size_t require(address != 0 && bytes != 0 && bytes <= std::numeric_limits::max() - address, "invalid guest allocation range"); const auto end = address + bytes; auto cursor = address; - for (const auto& [base, range] : registry().ranges) { - const auto finish = base + range->bytes; + const auto& ranges = registry().ranges; + auto it = ranges.upper_bound(address); + if (it != ranges.begin()) --it; + for (; it != ranges.end(); ++it) { + const auto base = it->first; + const auto finish = base + it->second->bytes; if (finish <= cursor) continue; - if (base > cursor || !range->releasable) return false; + if (base > cursor || !it->second->releasable) return false; cursor = finish; if (cursor >= end) return true; } @@ -424,22 +428,31 @@ void GuestAllocationsRemove_nid_postfix(void* mutation, const void* pointer) { namespace { -std::map> replaceRange(const void* pointer, std::size_t bytes, bool remove, bool readable, bool writable, bool gpu) { +struct RangePatch { + std::vector erased; + std::vector>> inserted; +}; + +RangePatch replaceRange(const void* pointer, std::size_t bytes, bool remove, bool readable, bool writable, bool gpu) { const auto address = reinterpret_cast(pointer); require(bytes != 0 && bytes <= std::numeric_limits::max() - address, "invalid guest protection or unmap range"); require(!writable || readable, "writable guest allocation must be readable"); const auto end = address + bytes; - auto replacement = registry().ranges; + RangePatch patch; auto cursor = address; - for (const auto& [base, entry] : registry().ranges) { - const auto& range = *entry; + const auto& ranges = registry().ranges; + auto it = ranges.upper_bound(address); + if (it != ranges.begin()) --it; + for (; it != ranges.end(); ++it) { + const auto base = it->first; + const auto& range = *it->second; const auto finish = base + range.bytes; if (finish <= address) continue; if (base >= end) break; require(base <= cursor, "guest protection or unmap range has a hole"); - replacement.erase(base); + patch.erased.push_back(base); const auto insert = [&](std::uint64_t first, std::uint64_t last, bool canRead, bool canWrite, bool gpuMapped) { - if (first < last) replacement.emplace(first, std::make_shared(Range{first, static_cast(last - first), canRead, canWrite, range.allocationAddress, range.allocationBytes, range.releasable, gpuMapped})); + if (first < last) patch.inserted.emplace_back(first, std::make_shared(Range{first, static_cast(last - first), canRead, canWrite, range.allocationAddress, range.allocationBytes, range.releasable, gpuMapped})); }; insert(base, std::max(base, address), range.readable, range.writable, range.gpu); if (!remove) insert(std::max(base, address), std::min(finish, end), readable, writable, gpu); @@ -447,7 +460,26 @@ std::map> replaceRange(const void* p cursor = std::min(finish, end); } require(cursor == end, "guest protection or unmap range is not registered"); - return replacement; + return patch; +} + +void applyPatch(const RangePatch& patch) { + auto& ranges = registry().ranges; + for (const auto key : patch.erased) ranges.erase(key); + for (const auto& [key, range] : patch.inserted) ranges.emplace(key, range); +} + +bool allocationRemains(const Range& range, const RangePatch& patch) { + for (const auto& [key, entry] : patch.inserted) { + if (entry->allocationAddress == range.allocationAddress) return true; + } + const auto& ranges = registry().ranges; + const auto limit = range.allocationAddress + range.allocationBytes; + for (auto it = ranges.lower_bound(range.allocationAddress); it != ranges.end() && it->first < limit; ++it) { + if (it->second->allocationAddress != range.allocationAddress) continue; + if (std::find(patch.erased.begin(), patch.erased.end(), it->first) == patch.erased.end()) return true; + } + return false; } } @@ -455,9 +487,9 @@ std::map> replaceRange(const void* p void GuestAllocationsProtect_nid_postfix(void* mutation, const void* pointer, std::size_t bytes, bool readable, bool writable, bool gpu, const std::function& apply) { GuestAllocationsRequireUnpinned_nid_postfix(mutation, pointer, bytes); recordChange(mutation, pointer, bytes); - auto replacement = replaceRange(pointer, bytes, false, readable, writable, gpu); + const auto patch = replaceRange(pointer, bytes, false, readable, writable, gpu); apply(); - registry().ranges.swap(replacement); + applyPatch(patch); } void GuestAllocationsUnmap_nid_postfix(void* mutation, const void* pointer, std::size_t bytes, const std::function& apply) { @@ -481,13 +513,10 @@ void GuestAllocationsUnmap_nid_postfix(void* mutation, const void* pointer, std: const auto range = *containing; require(range.releasable, "guest image memory cannot be unmapped"); const auto pieceEnd = std::min(end, range.address + range.bytes); - auto replacement = replaceRange(reinterpret_cast(cursor), pieceEnd - cursor, true, false, false, false); - bool last = true; - for (const auto& [base, entry] : replacement) { - if (entry->allocationAddress == range.allocationAddress) last = false; - } + const auto patch = replaceRange(reinterpret_cast(cursor), pieceEnd - cursor, true, false, false, false); + const bool last = !allocationRemains(range, patch); apply(reinterpret_cast(cursor), pieceEnd - cursor, reinterpret_cast(range.allocationAddress), last); - registry().ranges.swap(replacement); + applyPatch(patch); any = true; cursor = pieceEnd; } @@ -505,6 +534,22 @@ Lease GuestAllocationsAcquire_nid_postfix() { return Lease(std::move(ranges)); } +std::shared_ptr GuestAllocationsLookup_nid_postfix(std::uintptr_t address, bool next) { + std::lock_guard lock(registry().mutex); + const auto& ranges = registry().ranges; + auto it = ranges.upper_bound(address); + if (it != ranges.begin()) { + const auto& previous = std::prev(it)->second; + if (previous->bytes != 0 && address < previous->address + previous->bytes) return previous; + } + if (next) { + for (; it != ranges.end(); ++it) { + if (it->second->bytes != 0) return it->second; + } + } + return nullptr; +} + Lease GuestAllocationsAcquireAll_nid_postfix() { std::lock_guard lock(registry().mutex); if (auto shared = registry().allLease.lock()) return Lease(std::move(shared)); diff --git a/core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp b/core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp index df35b1890..0f04386b3 100644 --- a/core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp +++ b/core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp @@ -347,8 +347,6 @@ struct DirectMapping { }; std::map g_directMappings; - -#if defined(__linux__) struct PhysicalSpan { std::uint64_t end; std::vector bases; @@ -364,12 +362,10 @@ void SplitPhysicalSpan(std::uint64_t at) { g_physicalSpans.emplace_hint(std::next(span), at, PhysicalSpan{span->second.end, span->second.bases}); span->second.end = at; } -#endif std::map::iterator InsertMapping(std::uintptr_t base, DirectMapping mapping) { const auto [inserted, added] = g_directMappings.emplace(base, std::move(mapping)); if (!added) throw std::runtime_error("direct mapping inserted over a live one"); -#if defined(__linux__) const auto first = inserted->second.phys; const auto last = first + (inserted->second.end - base); SplitPhysicalSpan(first); @@ -381,12 +377,10 @@ std::map::iterator InsertMapping(std::uintptr_t b bases.insert(std::upper_bound(bases.begin(), bases.end(), base), base); cursor = span->second.end; } -#endif return inserted; } std::map::iterator RemoveMapping(std::map::iterator mapping) { -#if defined(__linux__) const auto base = mapping->first; const auto first = mapping->second.phys; const auto last = first + (mapping->second.end - base); @@ -406,10 +400,18 @@ std::map::iterator RemoveMapping(std::map MappingsSharingPhys(std::uint64_t first, std::uint64_t last) { + std::vector bases; + for (auto span = g_physicalSpans.lower_bound(first); span != g_physicalSpans.end() && span->first < last; ++span) bases.insert(bases.end(), span->second.bases.begin(), span->second.bases.end()); + if (auto span = g_physicalSpans.lower_bound(first); span != g_physicalSpans.begin() && (--span)->second.end > first) bases.insert(bases.end(), span->second.bases.begin(), span->second.bases.end()); + std::sort(bases.begin(), bases.end()); + bases.erase(std::unique(bases.begin(), bases.end()), bases.end()); + return bases; +} + void EraseMappings(std::uintptr_t start, std::uintptr_t end) { auto it = g_directMappings.lower_bound(start); if (it != g_directMappings.begin() && std::prev(it)->second.end > start) --it; @@ -423,22 +425,20 @@ void EraseMappings(std::uintptr_t start, std::uintptr_t end) { } void ErasePhysMappings(std::uint64_t first, std::uint64_t last) { - for (auto it = g_directMappings.begin(); it != g_directMappings.end();) { - const auto base = it->first; + for (const auto base : MappingsSharingPhys(first, last)) { + const auto it = g_directMappings.find(base); + if (it == g_directMappings.end()) continue; const auto mapping = it->second; const auto mappingLast = mapping.phys + (mapping.end - base); - if (mapping.phys >= last || mappingLast <= first) { - ++it; - continue; - } - it = RemoveMapping(it); + if (mapping.phys >= last || mappingLast <= first) continue; + RemoveMapping(it); if (mapping.phys < first) { const auto keep = first - mapping.phys; InsertMapping(base, DirectMapping{base + keep, mapping.phys, mapping.memoryType, mapping.backing}); } if (mappingLast > last) { const auto skip = last - mapping.phys; - it = InsertMapping(base + skip, DirectMapping{mapping.end, last, mapping.memoryType, mapping.backing}); + InsertMapping(base + skip, DirectMapping{mapping.end, last, mapping.memoryType, mapping.backing}); } } } @@ -458,15 +458,10 @@ void ValidatePhysicalRange(std::uint64_t phys, std::size_t len) { void WatchMapping(std::uintptr_t address, std::size_t len) { GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(reinterpret_cast(address), len); const auto end = address + len; - std::vector sharing; for (auto added = g_directMappings.lower_bound(address); added != g_directMappings.end() && added->first < end; ++added) { const auto addedPhys = added->second.phys; const auto addedPhysEnd = addedPhys + (added->second.end - added->first); - sharing.clear(); - for (auto span = g_physicalSpans.lower_bound(addedPhys); span != g_physicalSpans.end() && span->first < addedPhysEnd; ++span) sharing.insert(sharing.end(), span->second.bases.begin(), span->second.bases.end()); - std::sort(sharing.begin(), sharing.end()); - sharing.erase(std::unique(sharing.begin(), sharing.end()), sharing.end()); - for (const auto base : sharing) { + for (const auto base : MappingsSharingPhys(addedPhys, addedPhysEnd)) { const auto& other = g_directMappings.at(base); if ((base >= address && base < end) || other.backing != added->second.backing) continue; const auto first = std::max(addedPhys, other.phys); @@ -1155,7 +1150,8 @@ std::vector DirectMemoryViews(int64_t start, size_t len) { const auto last = first + len; std::vector views; std::lock_guard lock(g_directLock); - for (const auto& [base, mapping] : g_directMappings) { + for (const auto base : MappingsSharingPhys(first, last)) { + const auto& mapping = g_directMappings.at(base); const auto mappingLast = mapping.phys + (mapping.end - base); if (mapping.phys >= last || mappingLast <= first) continue; const auto viewStart = base + (std::max(first, mapping.phys) - mapping.phys); diff --git a/core/libs/prx/libkernel/DirectMemory/Export.cpp b/core/libs/prx/libkernel/DirectMemory/Export.cpp index 0e97c3c67..ea7e00b05 100644 --- a/core/libs/prx/libkernel/DirectMemory/Export.cpp +++ b/core/libs/prx/libkernel/DirectMemory/Export.cpp @@ -259,15 +259,7 @@ int APS5_VABI sceKernelVirtualQuery(const void* addr, int flags, VirtualQueryInf info->end = reservedEnd; return 0; } - const auto lease = GuestAllocations::GuestAllocationsAcquireAll_nid_postfix(); - assert(std::is_sorted(lease.begin(), lease.end(), [](const auto& left, const auto& right) { return left->address < right->address; })); - const GuestAllocations::Range* best = nullptr; - for (const auto& range : lease) { - const auto begin = range->address; - const auto end = begin + range->bytes; - if (address >= begin && address < end) { best = range.get(); break; } - if ((flags & findNext) != 0 && begin > address && (best == nullptr || begin < best->address)) best = range.get(); - } + const auto best = GuestAllocations::GuestAllocationsLookup_nid_postfix(address, (flags & findNext) != 0); if (best != nullptr) { info->start = best->address; info->end = best->address + best->bytes; diff --git a/core/libs/tests/GuestMemory.cpp b/core/libs/tests/GuestMemory.cpp index 86a0d9e14..9a12142d8 100644 --- a/core/libs/tests/GuestMemory.cpp +++ b/core/libs/tests/GuestMemory.cpp @@ -272,6 +272,57 @@ static void CheckBatchMapStopsAtInvalidEntry() { Require(sceKernelMunmap(entries[0].start, page) == 0); } +static void CheckVirtualQueryFindsNeighbours() { + constexpr std::size_t page = 0x4000; + void* low = nullptr; + void* high = nullptr; + Require(sceKernelMapNamedFlexibleMemory(&low, page * 4, 3, 0, "neighbours") == 0 && low != nullptr); + Require(sceKernelMapFlexibleMemory(&high, page, 3, 0) == 0 && high != nullptr); + auto* lowBytes = static_cast(low); + VirtualQueryInfo info{}; + Require(sceKernelVirtualQuery(lowBytes + page, 0, &info, sizeof(info)) == 0); + Require(info.start == reinterpret_cast(low) && info.end == reinterpret_cast(low) + page * 4); + Require(sceKernelMprotect(lowBytes + page, page, 1) == 0); + Require(sceKernelVirtualQuery(lowBytes + page, 0, &info, sizeof(info)) == 0); + Require(info.start == reinterpret_cast(low) + page && info.end == reinterpret_cast(low) + page * 2 && info.protection == 1); + Require(sceKernelVirtualQuery(lowBytes + page * 2, 0, &info, sizeof(info)) == 0); + Require(info.start == reinterpret_cast(low) + page * 2 && info.end == reinterpret_cast(low) + page * 4 && info.protection == 3); + Require(sceKernelMunmap(lowBytes + page * 2, page) == 0); + Require(sceKernelVirtualQuery(lowBytes + page * 2, 0, &info, sizeof(info)) != 0); + Require(sceKernelVirtualQuery(lowBytes + page * 2, 1, &info, sizeof(info)) == 0); + Require(info.start == reinterpret_cast(low) + page * 3 && info.end == reinterpret_cast(low) + page * 4); + Require(sceKernelVirtualQuery(lowBytes + page * 3, 0, &info, sizeof(info)) == 0 && info.start == reinterpret_cast(low) + page * 3); + Require(sceKernelMunmap(low, page * 2) == 0); + Require(sceKernelMunmap(lowBytes + page * 3, page) == 0); + Require(sceKernelMunmap(high, page) == 0); +} + +static void CheckReleaseFindsEveryDirectMapping() { + constexpr std::size_t page = 0x4000; + constexpr std::size_t pieces = 48; + std::int64_t phys = 0; + Require(sceKernelAllocateDirectMemory(0, 0x7fffffffffll, page * pieces, 0, 0, &phys) == 0); + void* mapped[pieces] = {}; + for (std::size_t index = 0; index < pieces; ++index) { + const auto piece = pieces - 1 - index; + Require(sceKernelMapDirectMemory(&mapped[piece], page, 3, 0, phys + static_cast(piece * page), 0) == 0); + static_cast(mapped[piece])[0] = static_cast(piece); + } + void* alias = nullptr; + Require(sceKernelMapDirectMemory(&alias, page * 2, 3, 0, phys + static_cast(page * 10), 0) == 0); + Require(static_cast(alias)[0] == 10 && static_cast(alias)[page] == 11); + Require(sceKernelReleaseDirectMemory(phys + static_cast(page * 10), page * 3) == 0); + VirtualQueryInfo info{}; + for (std::size_t index = 0; index < pieces; ++index) { + const bool released = index >= 10 && index < 13; + Require((sceKernelVirtualQuery(mapped[index], 0, &info, sizeof(info)) == 0) != released); + } + Require(sceKernelVirtualQuery(alias, 0, &info, sizeof(info)) != 0); + Require(sceKernelReleaseDirectMemory(phys, page * 10) == 0); + Require(sceKernelReleaseDirectMemory(phys + static_cast(page * 13), page * (pieces - 13)) == 0); + for (std::size_t index = 0; index < pieces; ++index) Require(sceKernelVirtualQuery(mapped[index], 0, &info, sizeof(info)) != 0); +} + static void CheckCheckedReleaseDirectMemory() { constexpr std::size_t page = 0x4000; std::int64_t phys = 0; @@ -2149,6 +2200,8 @@ int main() { CheckMappingCostFollowsTheMappingCount(); CheckDirectMemoryBackingNeedsNoFilesystem(); CheckDirectMemorySharedBacking(); + CheckVirtualQueryFindsNeighbours(); + CheckReleaseFindsEveryDirectMapping(); #endif constexpr std::size_t page = 0x4000; const auto failed = reinterpret_cast(static_cast(-1));