perf(libkernel): find guest allocations and direct mappings by address
sceKernelVirtualQuery, sceKernelMunmap, sceKernelReleaseDirectMemory and the reservation paths walked a copy of the whole allocation registry or of every direct mapping under one lock on each call. The registry now answers lookups from its ordered map, applies range replacements as patches instead of rebuilding the map, and direct mappings carry an index by physical offset so a release finds only the mappings it covers. Minecraft (PPSA17221) holds about 7,000 direct mappings and spent its world generation in these walks. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Mouad-Scriptz-0
Dean Galvin
parent
77b9f7270c
commit
df3d027ebe
|
|
@ -69,6 +69,7 @@ void GuestAllocationsProtect_nid_postfix(void* mutation, const void* pointer, st
|
|||
void GuestAllocationsUnmap_nid_postfix(void* mutation, const void* pointer, std::size_t bytes, const std::function<void(const void*, std::size_t, const void*, bool)>& apply);
|
||||
Lease GuestAllocationsAcquire_nid_postfix();
|
||||
Lease GuestAllocationsAcquireAll_nid_postfix();
|
||||
std::shared_ptr<const Range> GuestAllocationsLookup_nid_postfix(std::uintptr_t address, bool next);
|
||||
// Changes whenever a mutation ends, so callers can cache facts about guest mappings between changes.
|
||||
std::uint64_t GuestAllocationsGeneration_nid_postfix();
|
||||
// Precise invalidation for such caches: the callback runs, after the generation changed, for every
|
||||
|
|
|
|||
|
|
@ -382,10 +382,14 @@ bool GuestAllocationsCovers_nid_postfix(void*, const void* pointer, std::size_t
|
|||
require(address != 0 && bytes != 0 && bytes <= std::numeric_limits<std::uint64_t>::max() - address, "invalid guest allocation range");
|
||||
const auto end = address + bytes;
|
||||
auto cursor = address;
|
||||
for (const auto& [base, range] : registry().ranges) {
|
||||
const auto finish = base + range->bytes;
|
||||
const auto& ranges = registry().ranges;
|
||||
auto it = ranges.upper_bound(address);
|
||||
if (it != ranges.begin()) --it;
|
||||
for (; it != ranges.end(); ++it) {
|
||||
const auto base = it->first;
|
||||
const auto finish = base + it->second->bytes;
|
||||
if (finish <= cursor) continue;
|
||||
if (base > cursor || !range->releasable) return false;
|
||||
if (base > cursor || !it->second->releasable) return false;
|
||||
cursor = finish;
|
||||
if (cursor >= end) return true;
|
||||
}
|
||||
|
|
@ -424,22 +428,31 @@ void GuestAllocationsRemove_nid_postfix(void* mutation, const void* pointer) {
|
|||
|
||||
namespace {
|
||||
|
||||
std::map<std::uint64_t, std::shared_ptr<const Range>> replaceRange(const void* pointer, std::size_t bytes, bool remove, bool readable, bool writable, bool gpu) {
|
||||
struct RangePatch {
|
||||
std::vector<std::uint64_t> erased;
|
||||
std::vector<std::pair<std::uint64_t, std::shared_ptr<const Range>>> inserted;
|
||||
};
|
||||
|
||||
RangePatch replaceRange(const void* pointer, std::size_t bytes, bool remove, bool readable, bool writable, bool gpu) {
|
||||
const auto address = reinterpret_cast<std::uintptr_t>(pointer);
|
||||
require(bytes != 0 && bytes <= std::numeric_limits<std::uint64_t>::max() - address, "invalid guest protection or unmap range");
|
||||
require(!writable || readable, "writable guest allocation must be readable");
|
||||
const auto end = address + bytes;
|
||||
auto replacement = registry().ranges;
|
||||
RangePatch patch;
|
||||
auto cursor = address;
|
||||
for (const auto& [base, entry] : registry().ranges) {
|
||||
const auto& range = *entry;
|
||||
const auto& ranges = registry().ranges;
|
||||
auto it = ranges.upper_bound(address);
|
||||
if (it != ranges.begin()) --it;
|
||||
for (; it != ranges.end(); ++it) {
|
||||
const auto base = it->first;
|
||||
const auto& range = *it->second;
|
||||
const auto finish = base + range.bytes;
|
||||
if (finish <= address) continue;
|
||||
if (base >= end) break;
|
||||
require(base <= cursor, "guest protection or unmap range has a hole");
|
||||
replacement.erase(base);
|
||||
patch.erased.push_back(base);
|
||||
const auto insert = [&](std::uint64_t first, std::uint64_t last, bool canRead, bool canWrite, bool gpuMapped) {
|
||||
if (first < last) replacement.emplace(first, std::make_shared<const Range>(Range{first, static_cast<std::size_t>(last - first), canRead, canWrite, range.allocationAddress, range.allocationBytes, range.releasable, gpuMapped}));
|
||||
if (first < last) patch.inserted.emplace_back(first, std::make_shared<const Range>(Range{first, static_cast<std::size_t>(last - first), canRead, canWrite, range.allocationAddress, range.allocationBytes, range.releasable, gpuMapped}));
|
||||
};
|
||||
insert(base, std::max(base, address), range.readable, range.writable, range.gpu);
|
||||
if (!remove) insert(std::max(base, address), std::min(finish, end), readable, writable, gpu);
|
||||
|
|
@ -447,7 +460,26 @@ std::map<std::uint64_t, std::shared_ptr<const Range>> replaceRange(const void* p
|
|||
cursor = std::min(finish, end);
|
||||
}
|
||||
require(cursor == end, "guest protection or unmap range is not registered");
|
||||
return replacement;
|
||||
return patch;
|
||||
}
|
||||
|
||||
void applyPatch(const RangePatch& patch) {
|
||||
auto& ranges = registry().ranges;
|
||||
for (const auto key : patch.erased) ranges.erase(key);
|
||||
for (const auto& [key, range] : patch.inserted) ranges.emplace(key, range);
|
||||
}
|
||||
|
||||
bool allocationRemains(const Range& range, const RangePatch& patch) {
|
||||
for (const auto& [key, entry] : patch.inserted) {
|
||||
if (entry->allocationAddress == range.allocationAddress) return true;
|
||||
}
|
||||
const auto& ranges = registry().ranges;
|
||||
const auto limit = range.allocationAddress + range.allocationBytes;
|
||||
for (auto it = ranges.lower_bound(range.allocationAddress); it != ranges.end() && it->first < limit; ++it) {
|
||||
if (it->second->allocationAddress != range.allocationAddress) continue;
|
||||
if (std::find(patch.erased.begin(), patch.erased.end(), it->first) == patch.erased.end()) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -455,9 +487,9 @@ std::map<std::uint64_t, std::shared_ptr<const Range>> replaceRange(const void* p
|
|||
void GuestAllocationsProtect_nid_postfix(void* mutation, const void* pointer, std::size_t bytes, bool readable, bool writable, bool gpu, const std::function<void()>& apply) {
|
||||
GuestAllocationsRequireUnpinned_nid_postfix(mutation, pointer, bytes);
|
||||
recordChange(mutation, pointer, bytes);
|
||||
auto replacement = replaceRange(pointer, bytes, false, readable, writable, gpu);
|
||||
const auto patch = replaceRange(pointer, bytes, false, readable, writable, gpu);
|
||||
apply();
|
||||
registry().ranges.swap(replacement);
|
||||
applyPatch(patch);
|
||||
}
|
||||
|
||||
void GuestAllocationsUnmap_nid_postfix(void* mutation, const void* pointer, std::size_t bytes, const std::function<void(const void*, std::size_t, const void*, bool)>& apply) {
|
||||
|
|
@ -481,13 +513,10 @@ void GuestAllocationsUnmap_nid_postfix(void* mutation, const void* pointer, std:
|
|||
const auto range = *containing;
|
||||
require(range.releasable, "guest image memory cannot be unmapped");
|
||||
const auto pieceEnd = std::min<std::uint64_t>(end, range.address + range.bytes);
|
||||
auto replacement = replaceRange(reinterpret_cast<const void*>(cursor), pieceEnd - cursor, true, false, false, false);
|
||||
bool last = true;
|
||||
for (const auto& [base, entry] : replacement) {
|
||||
if (entry->allocationAddress == range.allocationAddress) last = false;
|
||||
}
|
||||
const auto patch = replaceRange(reinterpret_cast<const void*>(cursor), pieceEnd - cursor, true, false, false, false);
|
||||
const bool last = !allocationRemains(range, patch);
|
||||
apply(reinterpret_cast<const void*>(cursor), pieceEnd - cursor, reinterpret_cast<const void*>(range.allocationAddress), last);
|
||||
registry().ranges.swap(replacement);
|
||||
applyPatch(patch);
|
||||
any = true;
|
||||
cursor = pieceEnd;
|
||||
}
|
||||
|
|
@ -505,6 +534,22 @@ Lease GuestAllocationsAcquire_nid_postfix() {
|
|||
return Lease(std::move(ranges));
|
||||
}
|
||||
|
||||
std::shared_ptr<const Range> GuestAllocationsLookup_nid_postfix(std::uintptr_t address, bool next) {
|
||||
std::lock_guard lock(registry().mutex);
|
||||
const auto& ranges = registry().ranges;
|
||||
auto it = ranges.upper_bound(address);
|
||||
if (it != ranges.begin()) {
|
||||
const auto& previous = std::prev(it)->second;
|
||||
if (previous->bytes != 0 && address < previous->address + previous->bytes) return previous;
|
||||
}
|
||||
if (next) {
|
||||
for (; it != ranges.end(); ++it) {
|
||||
if (it->second->bytes != 0) return it->second;
|
||||
}
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
Lease GuestAllocationsAcquireAll_nid_postfix() {
|
||||
std::lock_guard lock(registry().mutex);
|
||||
if (auto shared = registry().allLease.lock()) return Lease(std::move(shared));
|
||||
|
|
|
|||
|
|
@ -347,8 +347,6 @@ struct DirectMapping {
|
|||
};
|
||||
|
||||
std::map<std::uintptr_t, DirectMapping> g_directMappings;
|
||||
|
||||
#if defined(__linux__)
|
||||
struct PhysicalSpan {
|
||||
std::uint64_t end;
|
||||
std::vector<std::uintptr_t> bases;
|
||||
|
|
@ -364,12 +362,10 @@ void SplitPhysicalSpan(std::uint64_t at) {
|
|||
g_physicalSpans.emplace_hint(std::next(span), at, PhysicalSpan{span->second.end, span->second.bases});
|
||||
span->second.end = at;
|
||||
}
|
||||
#endif
|
||||
|
||||
std::map<std::uintptr_t, DirectMapping>::iterator InsertMapping(std::uintptr_t base, DirectMapping mapping) {
|
||||
const auto [inserted, added] = g_directMappings.emplace(base, std::move(mapping));
|
||||
if (!added) throw std::runtime_error("direct mapping inserted over a live one");
|
||||
#if defined(__linux__)
|
||||
const auto first = inserted->second.phys;
|
||||
const auto last = first + (inserted->second.end - base);
|
||||
SplitPhysicalSpan(first);
|
||||
|
|
@ -381,12 +377,10 @@ std::map<std::uintptr_t, DirectMapping>::iterator InsertMapping(std::uintptr_t b
|
|||
bases.insert(std::upper_bound(bases.begin(), bases.end(), base), base);
|
||||
cursor = span->second.end;
|
||||
}
|
||||
#endif
|
||||
return inserted;
|
||||
}
|
||||
|
||||
std::map<std::uintptr_t, DirectMapping>::iterator RemoveMapping(std::map<std::uintptr_t, DirectMapping>::iterator mapping) {
|
||||
#if defined(__linux__)
|
||||
const auto base = mapping->first;
|
||||
const auto first = mapping->second.phys;
|
||||
const auto last = first + (mapping->second.end - base);
|
||||
|
|
@ -406,10 +400,18 @@ std::map<std::uintptr_t, DirectMapping>::iterator RemoveMapping(std::map<std::ui
|
|||
}
|
||||
++span;
|
||||
}
|
||||
#endif
|
||||
return g_directMappings.erase(mapping);
|
||||
}
|
||||
|
||||
std::vector<std::uintptr_t> MappingsSharingPhys(std::uint64_t first, std::uint64_t last) {
|
||||
std::vector<std::uintptr_t> bases;
|
||||
for (auto span = g_physicalSpans.lower_bound(first); span != g_physicalSpans.end() && span->first < last; ++span) bases.insert(bases.end(), span->second.bases.begin(), span->second.bases.end());
|
||||
if (auto span = g_physicalSpans.lower_bound(first); span != g_physicalSpans.begin() && (--span)->second.end > first) bases.insert(bases.end(), span->second.bases.begin(), span->second.bases.end());
|
||||
std::sort(bases.begin(), bases.end());
|
||||
bases.erase(std::unique(bases.begin(), bases.end()), bases.end());
|
||||
return bases;
|
||||
}
|
||||
|
||||
void EraseMappings(std::uintptr_t start, std::uintptr_t end) {
|
||||
auto it = g_directMappings.lower_bound(start);
|
||||
if (it != g_directMappings.begin() && std::prev(it)->second.end > start) --it;
|
||||
|
|
@ -423,22 +425,20 @@ void EraseMappings(std::uintptr_t start, std::uintptr_t end) {
|
|||
}
|
||||
|
||||
void ErasePhysMappings(std::uint64_t first, std::uint64_t last) {
|
||||
for (auto it = g_directMappings.begin(); it != g_directMappings.end();) {
|
||||
const auto base = it->first;
|
||||
for (const auto base : MappingsSharingPhys(first, last)) {
|
||||
const auto it = g_directMappings.find(base);
|
||||
if (it == g_directMappings.end()) continue;
|
||||
const auto mapping = it->second;
|
||||
const auto mappingLast = mapping.phys + (mapping.end - base);
|
||||
if (mapping.phys >= last || mappingLast <= first) {
|
||||
++it;
|
||||
continue;
|
||||
}
|
||||
it = RemoveMapping(it);
|
||||
if (mapping.phys >= last || mappingLast <= first) continue;
|
||||
RemoveMapping(it);
|
||||
if (mapping.phys < first) {
|
||||
const auto keep = first - mapping.phys;
|
||||
InsertMapping(base, DirectMapping{base + keep, mapping.phys, mapping.memoryType, mapping.backing});
|
||||
}
|
||||
if (mappingLast > last) {
|
||||
const auto skip = last - mapping.phys;
|
||||
it = InsertMapping(base + skip, DirectMapping{mapping.end, last, mapping.memoryType, mapping.backing});
|
||||
InsertMapping(base + skip, DirectMapping{mapping.end, last, mapping.memoryType, mapping.backing});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -458,15 +458,10 @@ void ValidatePhysicalRange(std::uint64_t phys, std::size_t len) {
|
|||
void WatchMapping(std::uintptr_t address, std::size_t len) {
|
||||
GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(reinterpret_cast<const void*>(address), len);
|
||||
const auto end = address + len;
|
||||
std::vector<std::uintptr_t> sharing;
|
||||
for (auto added = g_directMappings.lower_bound(address); added != g_directMappings.end() && added->first < end; ++added) {
|
||||
const auto addedPhys = added->second.phys;
|
||||
const auto addedPhysEnd = addedPhys + (added->second.end - added->first);
|
||||
sharing.clear();
|
||||
for (auto span = g_physicalSpans.lower_bound(addedPhys); span != g_physicalSpans.end() && span->first < addedPhysEnd; ++span) sharing.insert(sharing.end(), span->second.bases.begin(), span->second.bases.end());
|
||||
std::sort(sharing.begin(), sharing.end());
|
||||
sharing.erase(std::unique(sharing.begin(), sharing.end()), sharing.end());
|
||||
for (const auto base : sharing) {
|
||||
for (const auto base : MappingsSharingPhys(addedPhys, addedPhysEnd)) {
|
||||
const auto& other = g_directMappings.at(base);
|
||||
if ((base >= address && base < end) || other.backing != added->second.backing) continue;
|
||||
const auto first = std::max(addedPhys, other.phys);
|
||||
|
|
@ -1155,7 +1150,8 @@ std::vector<DirectMemoryView> DirectMemoryViews(int64_t start, size_t len) {
|
|||
const auto last = first + len;
|
||||
std::vector<DirectMemoryView> views;
|
||||
std::lock_guard lock(g_directLock);
|
||||
for (const auto& [base, mapping] : g_directMappings) {
|
||||
for (const auto base : MappingsSharingPhys(first, last)) {
|
||||
const auto& mapping = g_directMappings.at(base);
|
||||
const auto mappingLast = mapping.phys + (mapping.end - base);
|
||||
if (mapping.phys >= last || mappingLast <= first) continue;
|
||||
const auto viewStart = base + (std::max(first, mapping.phys) - mapping.phys);
|
||||
|
|
|
|||
|
|
@ -259,15 +259,7 @@ int APS5_VABI sceKernelVirtualQuery(const void* addr, int flags, VirtualQueryInf
|
|||
info->end = reservedEnd;
|
||||
return 0;
|
||||
}
|
||||
const auto lease = GuestAllocations::GuestAllocationsAcquireAll_nid_postfix();
|
||||
assert(std::is_sorted(lease.begin(), lease.end(), [](const auto& left, const auto& right) { return left->address < right->address; }));
|
||||
const GuestAllocations::Range* best = nullptr;
|
||||
for (const auto& range : lease) {
|
||||
const auto begin = range->address;
|
||||
const auto end = begin + range->bytes;
|
||||
if (address >= begin && address < end) { best = range.get(); break; }
|
||||
if ((flags & findNext) != 0 && begin > address && (best == nullptr || begin < best->address)) best = range.get();
|
||||
}
|
||||
const auto best = GuestAllocations::GuestAllocationsLookup_nid_postfix(address, (flags & findNext) != 0);
|
||||
if (best != nullptr) {
|
||||
info->start = best->address;
|
||||
info->end = best->address + best->bytes;
|
||||
|
|
|
|||
|
|
@ -272,6 +272,57 @@ static void CheckBatchMapStopsAtInvalidEntry() {
|
|||
Require(sceKernelMunmap(entries[0].start, page) == 0);
|
||||
}
|
||||
|
||||
static void CheckVirtualQueryFindsNeighbours() {
|
||||
constexpr std::size_t page = 0x4000;
|
||||
void* low = nullptr;
|
||||
void* high = nullptr;
|
||||
Require(sceKernelMapNamedFlexibleMemory(&low, page * 4, 3, 0, "neighbours") == 0 && low != nullptr);
|
||||
Require(sceKernelMapFlexibleMemory(&high, page, 3, 0) == 0 && high != nullptr);
|
||||
auto* lowBytes = static_cast<unsigned char*>(low);
|
||||
VirtualQueryInfo info{};
|
||||
Require(sceKernelVirtualQuery(lowBytes + page, 0, &info, sizeof(info)) == 0);
|
||||
Require(info.start == reinterpret_cast<std::uintptr_t>(low) && info.end == reinterpret_cast<std::uintptr_t>(low) + page * 4);
|
||||
Require(sceKernelMprotect(lowBytes + page, page, 1) == 0);
|
||||
Require(sceKernelVirtualQuery(lowBytes + page, 0, &info, sizeof(info)) == 0);
|
||||
Require(info.start == reinterpret_cast<std::uintptr_t>(low) + page && info.end == reinterpret_cast<std::uintptr_t>(low) + page * 2 && info.protection == 1);
|
||||
Require(sceKernelVirtualQuery(lowBytes + page * 2, 0, &info, sizeof(info)) == 0);
|
||||
Require(info.start == reinterpret_cast<std::uintptr_t>(low) + page * 2 && info.end == reinterpret_cast<std::uintptr_t>(low) + page * 4 && info.protection == 3);
|
||||
Require(sceKernelMunmap(lowBytes + page * 2, page) == 0);
|
||||
Require(sceKernelVirtualQuery(lowBytes + page * 2, 0, &info, sizeof(info)) != 0);
|
||||
Require(sceKernelVirtualQuery(lowBytes + page * 2, 1, &info, sizeof(info)) == 0);
|
||||
Require(info.start == reinterpret_cast<std::uintptr_t>(low) + page * 3 && info.end == reinterpret_cast<std::uintptr_t>(low) + page * 4);
|
||||
Require(sceKernelVirtualQuery(lowBytes + page * 3, 0, &info, sizeof(info)) == 0 && info.start == reinterpret_cast<std::uintptr_t>(low) + page * 3);
|
||||
Require(sceKernelMunmap(low, page * 2) == 0);
|
||||
Require(sceKernelMunmap(lowBytes + page * 3, page) == 0);
|
||||
Require(sceKernelMunmap(high, page) == 0);
|
||||
}
|
||||
|
||||
static void CheckReleaseFindsEveryDirectMapping() {
|
||||
constexpr std::size_t page = 0x4000;
|
||||
constexpr std::size_t pieces = 48;
|
||||
std::int64_t phys = 0;
|
||||
Require(sceKernelAllocateDirectMemory(0, 0x7fffffffffll, page * pieces, 0, 0, &phys) == 0);
|
||||
void* mapped[pieces] = {};
|
||||
for (std::size_t index = 0; index < pieces; ++index) {
|
||||
const auto piece = pieces - 1 - index;
|
||||
Require(sceKernelMapDirectMemory(&mapped[piece], page, 3, 0, phys + static_cast<std::int64_t>(piece * page), 0) == 0);
|
||||
static_cast<unsigned char*>(mapped[piece])[0] = static_cast<unsigned char>(piece);
|
||||
}
|
||||
void* alias = nullptr;
|
||||
Require(sceKernelMapDirectMemory(&alias, page * 2, 3, 0, phys + static_cast<std::int64_t>(page * 10), 0) == 0);
|
||||
Require(static_cast<unsigned char*>(alias)[0] == 10 && static_cast<unsigned char*>(alias)[page] == 11);
|
||||
Require(sceKernelReleaseDirectMemory(phys + static_cast<std::int64_t>(page * 10), page * 3) == 0);
|
||||
VirtualQueryInfo info{};
|
||||
for (std::size_t index = 0; index < pieces; ++index) {
|
||||
const bool released = index >= 10 && index < 13;
|
||||
Require((sceKernelVirtualQuery(mapped[index], 0, &info, sizeof(info)) == 0) != released);
|
||||
}
|
||||
Require(sceKernelVirtualQuery(alias, 0, &info, sizeof(info)) != 0);
|
||||
Require(sceKernelReleaseDirectMemory(phys, page * 10) == 0);
|
||||
Require(sceKernelReleaseDirectMemory(phys + static_cast<std::int64_t>(page * 13), page * (pieces - 13)) == 0);
|
||||
for (std::size_t index = 0; index < pieces; ++index) Require(sceKernelVirtualQuery(mapped[index], 0, &info, sizeof(info)) != 0);
|
||||
}
|
||||
|
||||
static void CheckCheckedReleaseDirectMemory() {
|
||||
constexpr std::size_t page = 0x4000;
|
||||
std::int64_t phys = 0;
|
||||
|
|
@ -2149,6 +2200,8 @@ int main() {
|
|||
CheckMappingCostFollowsTheMappingCount();
|
||||
CheckDirectMemoryBackingNeedsNoFilesystem();
|
||||
CheckDirectMemorySharedBacking();
|
||||
CheckVirtualQueryFindsNeighbours();
|
||||
CheckReleaseFindsEveryDirectMapping();
|
||||
#endif
|
||||
constexpr std::size_t page = 0x4000;
|
||||
const auto failed = reinterpret_cast<void*>(static_cast<std::uintptr_t>(-1));
|
||||
|
|
|
|||
Loading…
Reference in New Issue