diff --git a/core/libs/prx/libc/include/GuestAllocations.hpp b/core/libs/prx/libc/include/GuestAllocations.hpp index e016be850..231826bb9 100644 --- a/core/libs/prx/libc/include/GuestAllocations.hpp +++ b/core/libs/prx/libc/include/GuestAllocations.hpp @@ -69,6 +69,7 @@ void GuestAllocationsProtect_nid_postfix(void* mutation, const void* pointer, st void GuestAllocationsUnmap_nid_postfix(void* mutation, const void* pointer, std::size_t bytes, const std::function& apply); Lease GuestAllocationsAcquire_nid_postfix(); Lease GuestAllocationsAcquireAll_nid_postfix(); +std::shared_ptr GuestAllocationsLookup_nid_postfix(std::uintptr_t address, bool next); // Changes whenever a mutation ends, so callers can cache facts about guest mappings between changes. std::uint64_t GuestAllocationsGeneration_nid_postfix(); // Precise invalidation for such caches: the callback runs, after the generation changed, for every diff --git a/core/libs/prx/libc/src/GuestAllocations.cpp b/core/libs/prx/libc/src/GuestAllocations.cpp index 33791e180..1faac8313 100644 --- a/core/libs/prx/libc/src/GuestAllocations.cpp +++ b/core/libs/prx/libc/src/GuestAllocations.cpp @@ -382,10 +382,14 @@ bool GuestAllocationsCovers_nid_postfix(void*, const void* pointer, std::size_t require(address != 0 && bytes != 0 && bytes <= std::numeric_limits::max() - address, "invalid guest allocation range"); const auto end = address + bytes; auto cursor = address; - for (const auto& [base, range] : registry().ranges) { - const auto finish = base + range->bytes; + const auto& ranges = registry().ranges; + auto it = ranges.upper_bound(address); + if (it != ranges.begin()) --it; + for (; it != ranges.end(); ++it) { + const auto base = it->first; + const auto finish = base + it->second->bytes; if (finish <= cursor) continue; - if (base > cursor || !range->releasable) return false; + if (base > cursor || !it->second->releasable) return false; cursor = finish; if (cursor >= end) return true; } @@ -424,22 +428,31 @@ void GuestAllocationsRemove_nid_postfix(void* mutation, const void* pointer) { namespace { -std::map> replaceRange(const void* pointer, std::size_t bytes, bool remove, bool readable, bool writable, bool gpu) { +struct RangePatch { + std::vector erased; + std::vector>> inserted; +}; + +RangePatch replaceRange(const void* pointer, std::size_t bytes, bool remove, bool readable, bool writable, bool gpu) { const auto address = reinterpret_cast(pointer); require(bytes != 0 && bytes <= std::numeric_limits::max() - address, "invalid guest protection or unmap range"); require(!writable || readable, "writable guest allocation must be readable"); const auto end = address + bytes; - auto replacement = registry().ranges; + RangePatch patch; auto cursor = address; - for (const auto& [base, entry] : registry().ranges) { - const auto& range = *entry; + const auto& ranges = registry().ranges; + auto it = ranges.upper_bound(address); + if (it != ranges.begin()) --it; + for (; it != ranges.end(); ++it) { + const auto base = it->first; + const auto& range = *it->second; const auto finish = base + range.bytes; if (finish <= address) continue; if (base >= end) break; require(base <= cursor, "guest protection or unmap range has a hole"); - replacement.erase(base); + patch.erased.push_back(base); const auto insert = [&](std::uint64_t first, std::uint64_t last, bool canRead, bool canWrite, bool gpuMapped) { - if (first < last) replacement.emplace(first, std::make_shared(Range{first, static_cast(last - first), canRead, canWrite, range.allocationAddress, range.allocationBytes, range.releasable, gpuMapped})); + if (first < last) patch.inserted.emplace_back(first, std::make_shared(Range{first, static_cast(last - first), canRead, canWrite, range.allocationAddress, range.allocationBytes, range.releasable, gpuMapped})); }; insert(base, std::max(base, address), range.readable, range.writable, range.gpu); if (!remove) insert(std::max(base, address), std::min(finish, end), readable, writable, gpu); @@ -447,7 +460,26 @@ std::map> replaceRange(const void* p cursor = std::min(finish, end); } require(cursor == end, "guest protection or unmap range is not registered"); - return replacement; + return patch; +} + +void applyPatch(const RangePatch& patch) { + auto& ranges = registry().ranges; + for (const auto key : patch.erased) ranges.erase(key); + for (const auto& [key, range] : patch.inserted) ranges.emplace(key, range); +} + +bool allocationRemains(const Range& range, const RangePatch& patch) { + for (const auto& [key, entry] : patch.inserted) { + if (entry->allocationAddress == range.allocationAddress) return true; + } + const auto& ranges = registry().ranges; + const auto limit = range.allocationAddress + range.allocationBytes; + for (auto it = ranges.lower_bound(range.allocationAddress); it != ranges.end() && it->first < limit; ++it) { + if (it->second->allocationAddress != range.allocationAddress) continue; + if (std::find(patch.erased.begin(), patch.erased.end(), it->first) == patch.erased.end()) return true; + } + return false; } } @@ -455,9 +487,9 @@ std::map> replaceRange(const void* p void GuestAllocationsProtect_nid_postfix(void* mutation, const void* pointer, std::size_t bytes, bool readable, bool writable, bool gpu, const std::function& apply) { GuestAllocationsRequireUnpinned_nid_postfix(mutation, pointer, bytes); recordChange(mutation, pointer, bytes); - auto replacement = replaceRange(pointer, bytes, false, readable, writable, gpu); + const auto patch = replaceRange(pointer, bytes, false, readable, writable, gpu); apply(); - registry().ranges.swap(replacement); + applyPatch(patch); } void GuestAllocationsUnmap_nid_postfix(void* mutation, const void* pointer, std::size_t bytes, const std::function& apply) { @@ -481,13 +513,10 @@ void GuestAllocationsUnmap_nid_postfix(void* mutation, const void* pointer, std: const auto range = *containing; require(range.releasable, "guest image memory cannot be unmapped"); const auto pieceEnd = std::min(end, range.address + range.bytes); - auto replacement = replaceRange(reinterpret_cast(cursor), pieceEnd - cursor, true, false, false, false); - bool last = true; - for (const auto& [base, entry] : replacement) { - if (entry->allocationAddress == range.allocationAddress) last = false; - } + const auto patch = replaceRange(reinterpret_cast(cursor), pieceEnd - cursor, true, false, false, false); + const bool last = !allocationRemains(range, patch); apply(reinterpret_cast(cursor), pieceEnd - cursor, reinterpret_cast(range.allocationAddress), last); - registry().ranges.swap(replacement); + applyPatch(patch); any = true; cursor = pieceEnd; } @@ -505,6 +534,22 @@ Lease GuestAllocationsAcquire_nid_postfix() { return Lease(std::move(ranges)); } +std::shared_ptr GuestAllocationsLookup_nid_postfix(std::uintptr_t address, bool next) { + std::lock_guard lock(registry().mutex); + const auto& ranges = registry().ranges; + auto it = ranges.upper_bound(address); + if (it != ranges.begin()) { + const auto& previous = std::prev(it)->second; + if (previous->bytes != 0 && address < previous->address + previous->bytes) return previous; + } + if (next) { + for (; it != ranges.end(); ++it) { + if (it->second->bytes != 0) return it->second; + } + } + return nullptr; +} + Lease GuestAllocationsAcquireAll_nid_postfix() { std::lock_guard lock(registry().mutex); if (auto shared = registry().allLease.lock()) return Lease(std::move(shared)); diff --git a/core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp b/core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp index df35b1890..0f04386b3 100644 --- a/core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp +++ b/core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp @@ -347,8 +347,6 @@ struct DirectMapping { }; std::map g_directMappings; - -#if defined(__linux__) struct PhysicalSpan { std::uint64_t end; std::vector bases; @@ -364,12 +362,10 @@ void SplitPhysicalSpan(std::uint64_t at) { g_physicalSpans.emplace_hint(std::next(span), at, PhysicalSpan{span->second.end, span->second.bases}); span->second.end = at; } -#endif std::map::iterator InsertMapping(std::uintptr_t base, DirectMapping mapping) { const auto [inserted, added] = g_directMappings.emplace(base, std::move(mapping)); if (!added) throw std::runtime_error("direct mapping inserted over a live one"); -#if defined(__linux__) const auto first = inserted->second.phys; const auto last = first + (inserted->second.end - base); SplitPhysicalSpan(first); @@ -381,12 +377,10 @@ std::map::iterator InsertMapping(std::uintptr_t b bases.insert(std::upper_bound(bases.begin(), bases.end(), base), base); cursor = span->second.end; } -#endif return inserted; } std::map::iterator RemoveMapping(std::map::iterator mapping) { -#if defined(__linux__) const auto base = mapping->first; const auto first = mapping->second.phys; const auto last = first + (mapping->second.end - base); @@ -406,10 +400,18 @@ std::map::iterator RemoveMapping(std::map MappingsSharingPhys(std::uint64_t first, std::uint64_t last) { + std::vector bases; + for (auto span = g_physicalSpans.lower_bound(first); span != g_physicalSpans.end() && span->first < last; ++span) bases.insert(bases.end(), span->second.bases.begin(), span->second.bases.end()); + if (auto span = g_physicalSpans.lower_bound(first); span != g_physicalSpans.begin() && (--span)->second.end > first) bases.insert(bases.end(), span->second.bases.begin(), span->second.bases.end()); + std::sort(bases.begin(), bases.end()); + bases.erase(std::unique(bases.begin(), bases.end()), bases.end()); + return bases; +} + void EraseMappings(std::uintptr_t start, std::uintptr_t end) { auto it = g_directMappings.lower_bound(start); if (it != g_directMappings.begin() && std::prev(it)->second.end > start) --it; @@ -423,22 +425,20 @@ void EraseMappings(std::uintptr_t start, std::uintptr_t end) { } void ErasePhysMappings(std::uint64_t first, std::uint64_t last) { - for (auto it = g_directMappings.begin(); it != g_directMappings.end();) { - const auto base = it->first; + for (const auto base : MappingsSharingPhys(first, last)) { + const auto it = g_directMappings.find(base); + if (it == g_directMappings.end()) continue; const auto mapping = it->second; const auto mappingLast = mapping.phys + (mapping.end - base); - if (mapping.phys >= last || mappingLast <= first) { - ++it; - continue; - } - it = RemoveMapping(it); + if (mapping.phys >= last || mappingLast <= first) continue; + RemoveMapping(it); if (mapping.phys < first) { const auto keep = first - mapping.phys; InsertMapping(base, DirectMapping{base + keep, mapping.phys, mapping.memoryType, mapping.backing}); } if (mappingLast > last) { const auto skip = last - mapping.phys; - it = InsertMapping(base + skip, DirectMapping{mapping.end, last, mapping.memoryType, mapping.backing}); + InsertMapping(base + skip, DirectMapping{mapping.end, last, mapping.memoryType, mapping.backing}); } } } @@ -458,15 +458,10 @@ void ValidatePhysicalRange(std::uint64_t phys, std::size_t len) { void WatchMapping(std::uintptr_t address, std::size_t len) { GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(reinterpret_cast(address), len); const auto end = address + len; - std::vector sharing; for (auto added = g_directMappings.lower_bound(address); added != g_directMappings.end() && added->first < end; ++added) { const auto addedPhys = added->second.phys; const auto addedPhysEnd = addedPhys + (added->second.end - added->first); - sharing.clear(); - for (auto span = g_physicalSpans.lower_bound(addedPhys); span != g_physicalSpans.end() && span->first < addedPhysEnd; ++span) sharing.insert(sharing.end(), span->second.bases.begin(), span->second.bases.end()); - std::sort(sharing.begin(), sharing.end()); - sharing.erase(std::unique(sharing.begin(), sharing.end()), sharing.end()); - for (const auto base : sharing) { + for (const auto base : MappingsSharingPhys(addedPhys, addedPhysEnd)) { const auto& other = g_directMappings.at(base); if ((base >= address && base < end) || other.backing != added->second.backing) continue; const auto first = std::max(addedPhys, other.phys); @@ -1155,7 +1150,8 @@ std::vector DirectMemoryViews(int64_t start, size_t len) { const auto last = first + len; std::vector views; std::lock_guard lock(g_directLock); - for (const auto& [base, mapping] : g_directMappings) { + for (const auto base : MappingsSharingPhys(first, last)) { + const auto& mapping = g_directMappings.at(base); const auto mappingLast = mapping.phys + (mapping.end - base); if (mapping.phys >= last || mappingLast <= first) continue; const auto viewStart = base + (std::max(first, mapping.phys) - mapping.phys); diff --git a/core/libs/prx/libkernel/DirectMemory/Export.cpp b/core/libs/prx/libkernel/DirectMemory/Export.cpp index 0e97c3c67..ea7e00b05 100644 --- a/core/libs/prx/libkernel/DirectMemory/Export.cpp +++ b/core/libs/prx/libkernel/DirectMemory/Export.cpp @@ -259,15 +259,7 @@ int APS5_VABI sceKernelVirtualQuery(const void* addr, int flags, VirtualQueryInf info->end = reservedEnd; return 0; } - const auto lease = GuestAllocations::GuestAllocationsAcquireAll_nid_postfix(); - assert(std::is_sorted(lease.begin(), lease.end(), [](const auto& left, const auto& right) { return left->address < right->address; })); - const GuestAllocations::Range* best = nullptr; - for (const auto& range : lease) { - const auto begin = range->address; - const auto end = begin + range->bytes; - if (address >= begin && address < end) { best = range.get(); break; } - if ((flags & findNext) != 0 && begin > address && (best == nullptr || begin < best->address)) best = range.get(); - } + const auto best = GuestAllocations::GuestAllocationsLookup_nid_postfix(address, (flags & findNext) != 0); if (best != nullptr) { info->start = best->address; info->end = best->address + best->bytes; diff --git a/core/libs/tests/GuestMemory.cpp b/core/libs/tests/GuestMemory.cpp index 86a0d9e14..9a12142d8 100644 --- a/core/libs/tests/GuestMemory.cpp +++ b/core/libs/tests/GuestMemory.cpp @@ -272,6 +272,57 @@ static void CheckBatchMapStopsAtInvalidEntry() { Require(sceKernelMunmap(entries[0].start, page) == 0); } +static void CheckVirtualQueryFindsNeighbours() { + constexpr std::size_t page = 0x4000; + void* low = nullptr; + void* high = nullptr; + Require(sceKernelMapNamedFlexibleMemory(&low, page * 4, 3, 0, "neighbours") == 0 && low != nullptr); + Require(sceKernelMapFlexibleMemory(&high, page, 3, 0) == 0 && high != nullptr); + auto* lowBytes = static_cast(low); + VirtualQueryInfo info{}; + Require(sceKernelVirtualQuery(lowBytes + page, 0, &info, sizeof(info)) == 0); + Require(info.start == reinterpret_cast(low) && info.end == reinterpret_cast(low) + page * 4); + Require(sceKernelMprotect(lowBytes + page, page, 1) == 0); + Require(sceKernelVirtualQuery(lowBytes + page, 0, &info, sizeof(info)) == 0); + Require(info.start == reinterpret_cast(low) + page && info.end == reinterpret_cast(low) + page * 2 && info.protection == 1); + Require(sceKernelVirtualQuery(lowBytes + page * 2, 0, &info, sizeof(info)) == 0); + Require(info.start == reinterpret_cast(low) + page * 2 && info.end == reinterpret_cast(low) + page * 4 && info.protection == 3); + Require(sceKernelMunmap(lowBytes + page * 2, page) == 0); + Require(sceKernelVirtualQuery(lowBytes + page * 2, 0, &info, sizeof(info)) != 0); + Require(sceKernelVirtualQuery(lowBytes + page * 2, 1, &info, sizeof(info)) == 0); + Require(info.start == reinterpret_cast(low) + page * 3 && info.end == reinterpret_cast(low) + page * 4); + Require(sceKernelVirtualQuery(lowBytes + page * 3, 0, &info, sizeof(info)) == 0 && info.start == reinterpret_cast(low) + page * 3); + Require(sceKernelMunmap(low, page * 2) == 0); + Require(sceKernelMunmap(lowBytes + page * 3, page) == 0); + Require(sceKernelMunmap(high, page) == 0); +} + +static void CheckReleaseFindsEveryDirectMapping() { + constexpr std::size_t page = 0x4000; + constexpr std::size_t pieces = 48; + std::int64_t phys = 0; + Require(sceKernelAllocateDirectMemory(0, 0x7fffffffffll, page * pieces, 0, 0, &phys) == 0); + void* mapped[pieces] = {}; + for (std::size_t index = 0; index < pieces; ++index) { + const auto piece = pieces - 1 - index; + Require(sceKernelMapDirectMemory(&mapped[piece], page, 3, 0, phys + static_cast(piece * page), 0) == 0); + static_cast(mapped[piece])[0] = static_cast(piece); + } + void* alias = nullptr; + Require(sceKernelMapDirectMemory(&alias, page * 2, 3, 0, phys + static_cast(page * 10), 0) == 0); + Require(static_cast(alias)[0] == 10 && static_cast(alias)[page] == 11); + Require(sceKernelReleaseDirectMemory(phys + static_cast(page * 10), page * 3) == 0); + VirtualQueryInfo info{}; + for (std::size_t index = 0; index < pieces; ++index) { + const bool released = index >= 10 && index < 13; + Require((sceKernelVirtualQuery(mapped[index], 0, &info, sizeof(info)) == 0) != released); + } + Require(sceKernelVirtualQuery(alias, 0, &info, sizeof(info)) != 0); + Require(sceKernelReleaseDirectMemory(phys, page * 10) == 0); + Require(sceKernelReleaseDirectMemory(phys + static_cast(page * 13), page * (pieces - 13)) == 0); + for (std::size_t index = 0; index < pieces; ++index) Require(sceKernelVirtualQuery(mapped[index], 0, &info, sizeof(info)) != 0); +} + static void CheckCheckedReleaseDirectMemory() { constexpr std::size_t page = 0x4000; std::int64_t phys = 0; @@ -2149,6 +2200,8 @@ int main() { CheckMappingCostFollowsTheMappingCount(); CheckDirectMemoryBackingNeedsNoFilesystem(); CheckDirectMemorySharedBacking(); + CheckVirtualQueryFindsNeighbours(); + CheckReleaseFindsEveryDirectMapping(); #endif constexpr std::size_t page = 0x4000; const auto failed = reinterpret_cast(static_cast(-1));