From 119560206d6c874d65dfdfd0f46bcc537aca3ebb Mon Sep 17 00:00:00 2001 From: STL1te Date: Tue, 29 Sep 2026 20:12:28 +0300 Subject: [PATCH] feat(agc-driver): heap ranges the host imports did not take are mirrored An address-based build maps every readable registered range. A heap range past APS5_HOST_IMPORT_MIB (or refused by the driver) was copied by every such build: about 3200 ranges, 670 MiB and 500 ms per build on the story-mode load, which then stalled. Such ranges are now served by persistent host-cached mirrors, as the main image is, and the load reaches its next failure. - a heap mirror keeps no shadow; it keeps a write-tracking generation per 64 KiB block and a build copies only the blocks stamped since. The generation is stored by the copy, so a build that throws before it leaves no block marked current with stale bytes - the build gathers its heap mirrors, sorts them and collects each run of adjacent ranges once (a collect costs about 4 us whatever its size; ~3000 mirrors took 16 ms per build, now about 3 ms) - writable ranges are mirrored too: each build keeps the bytes its merged written ranges held at upload and write-back stores only what the GPU changed from them, so a CPU store made meanwhile stays; recorded work writing a range completes before a refresh reads it - the mirrors are limited by APS5_HEAP_MIRROR_MIB (24 GiB by default); a mirror past it, or one whose buffer the driver refuses, stops the process. Heap mirrors are not given to descriptor sub-ranges, image mirror serials or reusable direct regions - past APS5_ADDRESS_COPY_MAX_MIB (64 MiB by default) of committed bytes still copied, a build stops the process, naming the largest ranges and why each is copied (unreadable pages, an image mirror the driver refused, a range outside the write-watched arena, mirrors disabled) - a direct memory page mapped at several addresses is read-only in every view while its range stays writable: copies, descriptor regions and mirrors take every readable page, write-back stores the writable ones and fails on a GPU change inside a read-only page instead of dropping it - a store through the BDA table into a mirrored or copied range faults as before (only writable ranges imported in place are writable there); the fault now names the table range and that rule - the BDA resource tests run with a storage range limit that holds the fault buffer and cover a mirror's fill, reuse, per-block refill, an interrupted refresh, a descriptor store beside a concurrent CPU store, and the copy limit Co-Authored-By: Claude Opus 5.5 --- .../Graphics/include/GuestBufferMemory.hpp | 29 ++ .../Graphics/src/BdaResources.cpp | 15 + .../Graphics/src/GuestBufferMemory.cpp | 297 ++++++++++++++++-- .../libSceAgcDriver/tests/BdaResources.cpp | 218 +++++++++++++ .../prx/libSceAgcDriver/tests/BdaTests.hpp | 2 + .../prx/libSceAgcDriver/tests/Graphics.cpp | 8 + 6 files changed, 537 insertions(+), 32 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp index 5677b2a1b..00cbae6da 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp @@ -6,6 +6,7 @@ #include "prx/libc/include/GuestAllocations.hpp" #include #include +#include #include #include @@ -123,6 +124,29 @@ struct AddressSpaceStats { }; AddressSpaceStats AddressSpaceCounters(); +// The image and heap mirrors (see GuestBufferMemory.cpp): heap mirrors held and their bytes, then +// cumulative: mirrors made, 64 KiB blocks copied into them after their fill, heap mirrors read again. +struct MirrorStats { + std::uint64_t heapMirrors = 0; + std::uint64_t heapBytes = 0; + std::uint64_t rebuilds = 0; + std::uint64_t blocksCopied = 0; + std::uint64_t heapRefills = 0; +}; +MirrorStats MirrorCounters(); + +// A registered range an address-based build copies instead of serving in place: its committed bytes +// and why no import or mirror serves it. +struct AddressCopy { + std::uint64_t begin; + std::uint64_t end; + std::uint64_t committed; + const char* reason; +}; +// The FATAL message for copies whose committed bytes exceed `limit` (APS5_ADDRESS_COPY_MAX_MIB), +// naming the largest ones; empty within the limit. +std::string AddressCopyOverflow(std::vector copies, std::uint64_t limit); + class GuestBufferMemory { public: explicit GuestBufferMemory(const Context& context); @@ -301,6 +325,9 @@ private: // Records the import-to-buffer copies of the given gpuCopy regions into the open batch, with // the barriers that order them after earlier recorded writes and before the shaders reading them. void recordGpuCopies(std::span copies, bool addressable); + // Keeps the bytes of every merged written range a writable heap mirror serves, as uploaded: the + // mirror keeps no shadow, and write-back stores only what the GPU changed from them. + void takeHeapReferences(); Context context; bool stagingAllowed = false; GuestAllocations::Lease lease; @@ -315,6 +342,8 @@ private: // regions follow the registry's order), so AddSnapshot can search instead of scanning. bool regionsSorted = false; std::vector> writes; + // By the begin of the merged written range (see takeHeapReferences). + std::vector>> heapReferences; // UploadPrepare ran (regions are frozen); `uploaded` once UploadFinish ran. bool prepared = false; bool uploaded = false; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp index 954db6398..7268aea59 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp @@ -254,6 +254,21 @@ void BdaResources::CheckFault() const { } std::ostringstream message; message << "BDA access failed: address=0x" << std::hex << report.address << " instruction=0x" << report.instruction << std::dec << " bytes=" << report.bytes << " stage=" << report.stage << " reason=" << static_cast(report.reason); + if (report.reason == ShaderRecompiler::BdaAbi::FaultReason::Permission && table != nullptr) { + // Only a writable range imported in place is writable through the table (see + // GuestBufferMemory::addressRange): a store into a range served by a mirror or a copy would + // not reach guest memory, so it faults. Name the range and how it is served. + const auto bytes = table->Bytes(); + ShaderRecompiler::BdaAbi::Header header{}; + std::memcpy(&header, bytes.data(), sizeof(header)); + for (std::uint32_t index = 0; index < header.count; ++index) { + ShaderRecompiler::BdaAbi::Range range{}; + std::memcpy(&range, bytes.data() + sizeof(header) + index * sizeof(range), sizeof(range)); + if (report.address < range.begin || report.address >= range.end) continue; + message << std::hex << "; the store hit 0x" << range.begin << "+0x" << range.end - range.begin << ", read-only in the BDA table: stores through GPU-selected descriptors reach only writable ranges imported in place, not ones served by a mirror or a copy (past APS5_HOST_IMPORT_MIB, or refused by the driver)"; + break; + } + } throw std::runtime_error(message.str()); } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp index ca290d06f..e7f6beb9a 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp @@ -36,10 +36,20 @@ namespace AgcDriver::Graphics { // mprotect, which replaces the registered Range object, and a mirror whose Range is gone is rebuilt. // Writable ranges keep a shadow of the guest bytes and copy the 64 KiB blocks that differ on each use. // A mirror never holds the Range shared_ptr: a pinned range makes the guest's mprotect spin and fail. +// Heap mirrors serve the readable registered heap ranges the host imports did not take (past +// APS5_HOST_IMPORT_MIB, or refused by the driver), which each address-based build copied instead. +// They keep no shadow: the arena's write tracking says which 64 KiB blocks changed since the mirror +// took them, and those blocks are read again. A writable one is written back per build from the +// bytes its written ranges held at upload (takeHeapReferences); stores through the BDA table fault +// there, as in a copy. struct ImageMirror { std::uint64_t base = 0; std::uint64_t bytes = 0; bool writable = false; + bool heap = false; + // Heap mirrors only: per tracker block of the range (block 0 holds `base`), the generation + // collected before the block was last read into the buffer. + std::vector generations; std::weak_ptr range; std::shared_ptr buffer; // Writable ranges only: the guest bytes the mirror holds, compared on refresh and at write-back. @@ -306,8 +316,14 @@ struct ImageMirrors { VkDevice device = VK_NULL_HANDLE; std::map> entries; std::set failed; + // Bytes of the heap mirrors in `entries` (against APS5_HEAP_MIRROR_MIB). + std::uint64_t heapBytes = 0; // APS5_PROFILE_DRAW counters, reported every 10 s as [buffers] image mirrors. std::uint64_t builds = 0; + // Heap mirrors read again because their range changed, and heap ranges left to copies because + // the arena does not track their writes. + std::uint64_t heapRefills = 0; + std::uint64_t heapUnwatched = 0; std::uint64_t subranges = 0; std::uint64_t rebuilds = 0; std::uint64_t refreshes = 0; @@ -361,6 +377,21 @@ bool mirrorsEnabled() { return !disabled; } +// The heap mirrors' limit: past it the process stops rather than copying gigabytes per dispatch. +std::uint64_t heapMirrorBudget() { + static const std::uint64_t bytes = [] { + const char* value = std::getenv("APS5_HEAP_MIRROR_MIB"); + return (value ? std::strtoull(value, nullptr, 10) : 24576ull) << 20u; + }(); + return bytes; +} + +[[noreturn]] void heapMirrorFatal(const GuestAllocations::Range& range, std::uint64_t held, const char* reason) { + std::fprintf(stderr, "FATAL: heap mirror of 0x%llx+0x%llx: %s; heap mirrors hold %llu MiB of %llu MiB (APS5_HEAP_MIRROR_MIB)\n", static_cast(range.address), static_cast(range.bytes), reason, static_cast(held >> 20u), static_cast(heapMirrorBudget() >> 20u)); + std::fflush(stderr); + std::abort(); +} + bool sameRange(const std::weak_ptr& mirrored, const std::shared_ptr& range) { return !mirrored.owner_before(range) && !range.owner_before(mirrored); } @@ -370,13 +401,22 @@ struct RefreshBlock { ImageMirror* mirror; std::uint64_t address; std::size_t length; + // Heap mirrors: the generation the block's copy makes current, stored only once the copy ran, + // so a build that throws before its copies leaves the block stale in the tracker's eyes too. + std::uint64_t generation = 0; }; // Compares a block's guest bytes with the mirror's shadow and copies a changed block into the device -// buffer and the shadow. Blocks are distinct, so blocks of one refresh can be compared in parallel. +// buffer and the shadow (a heap mirror's block is known to have changed and is copied). Blocks are +// distinct, so blocks of one refresh can be compared in parallel. bool compareBlock(const RefreshBlock& block) { const auto offset = static_cast(block.address - block.mirror->base); const auto* guest = reinterpret_cast(block.address); + if (block.mirror->heap) { + std::memcpy(block.mirror->buffer->Bytes().data() + offset, guest, block.length); + block.mirror->generations[static_cast(block.address / 65536 - block.mirror->base / 65536)] = block.generation; + return true; + } if (std::memcmp(block.mirror->shadow.data() + offset, guest, block.length) == 0) return false; std::memcpy(block.mirror->buffer->Bytes().data() + offset, guest, block.length); std::memcpy(block.mirror->shadow.data() + offset, guest, block.length); @@ -480,9 +520,10 @@ private: // completes first, so its results reach the guest and the shadow before the compare and no batch // writes a block being replaced; GPU reads of the mirror by earlier recorded work (an address-based // dispatch whose lease is released at completion) see the newer bytes, as they would on hardware. -// Returns whether there is anything to compare. -bool prepareRefresh(ImageMirror& mirror, std::uint64_t address, std::uint64_t bytes) { - if (!mirror.writable || bytes == 0) return false; +// prepareRange does it for [address, address + bytes), prepareRefresh for a writable image mirror and +// returns whether there is anything to compare. Heap mirrors are prepared a run of adjacent ranges at +// a time, before their write tracking is asked what changed (refreshHeapMirrors). +void prepareRange(std::uint64_t address, std::uint64_t bytes) { // The compare reads the pages directly; a mirror is only reused while its Range exists, but a // lease of a build in flight can keep a replaced Range alive past a protection change. Require(GuestMemory::Accessible(reinterpret_cast(address), static_cast(bytes)), "image mirror range became inaccessible"); @@ -498,6 +539,11 @@ bool prepareRefresh(ImageMirror& mirror, std::uint64_t address, std::uint64_t by recorder->Sync(); } ++state.refreshes; +} + +bool prepareRefresh(ImageMirror& mirror, std::uint64_t address, std::uint64_t bytes) { + if ((!mirror.writable && !mirror.heap) || bytes == 0) return false; + prepareRange(address, bytes); return true; } @@ -528,11 +574,49 @@ void refreshMirror(ImageMirror& mirror, std::uint64_t address, std::uint64_t byt compareBlocks(blocks); } +// Appends to `blocks` the blocks of one build's heap mirrors that a CPU or driver store changed +// since the mirrors took them. The mirrors of adjacent ranges are prepared and collected as one run: a +// collect walks the run's page tables at a fixed cost of microseconds, which paid per mirror (thousands +// on a story-mode load) was most of an address-based build. The copies follow and store the run's +// collected generation (see RefreshBlock). +void refreshHeapMirrors(std::vector& mirrors, std::vector& blocks) { + constexpr std::uint64_t block = 65536; + std::sort(mirrors.begin(), mirrors.end(), [](const ImageMirror* left, const ImageMirror* right) { return left->base < right->base; }); + thread_local std::vector changed; + for (std::size_t first = 0; first < mirrors.size();) { + auto last = first + 1; + while (last < mirrors.size() && mirrors[last]->base == mirrors[last - 1]->base + mirrors[last - 1]->bytes) ++last; + const auto begin = mirrors[first]->base; + const auto bytes = mirrors[last - 1]->base + mirrors[last - 1]->bytes - begin; + prepareRange(begin, bytes); + const auto generation = GuestMemory::CollectWrites(begin, static_cast(bytes)); + Require(generation != 0, "a heap mirror's range is no longer write-watched"); + for (auto index = first; index < last; ++index) { + auto& mirror = *mirrors[index]; + changed.assign(mirror.generations.size(), 0); + GuestMemory::ChangedBlocks(mirror.base, static_cast(mirror.bytes), mirror.generations, changed); + const auto aligned = mirror.base / block * block; + const auto end = mirror.base + mirror.bytes; + bool refilled = false; + for (std::size_t at = 0; at < changed.size(); ++at) { + if (changed[at] == 0) continue; + const auto from = std::max(mirror.base, aligned + at * block); + blocks.push_back({&mirror, from, static_cast(std::min(end, aligned + (at + 1) * block) - from), generation}); + refilled = true; + } + if (refilled) ++Mirrors().heapRefills; + } + first = last; + } +} + // The mirror for a leased image range: the existing one, or a new one when none exists or the // registered Range object changed (a protection change). Null when the range cannot be mirrored. An // existing writable mirror's blocks are appended to `blocks` for the caller's one compare of every -// mirror of the build (prepareRefresh ran); a new mirror is filled here. -std::shared_ptr acquireMirror(const Context& context, const std::shared_ptr& range, std::vector& blocks) { +// mirror of the build (prepareRefresh ran); the caller refreshes an existing heap mirror with the +// build's others (refreshHeapMirrors); a new mirror is filled here. A heap mirror past APS5_HEAP_MIRROR_MIB, or whose buffer the driver +// refuses, stops the process. +std::shared_ptr acquireMirror(const Context& context, const std::shared_ptr& range, std::vector& blocks, bool heap) { auto& state = Mirrors(); std::shared_ptr mirror; { @@ -540,26 +624,47 @@ std::shared_ptr acquireMirror(const Context& context, const std::sh if (state.device != context.device) { state.entries.clear(); state.failed.clear(); + state.heapBytes = 0; state.device = context.device; } if (state.failed.contains(range->address)) return nullptr; const auto found = state.entries.find(range->address); - if (found != state.entries.end() && sameRange(found->second->range, range) && found->second->bytes == range->bytes && found->second->writable == range->writable) mirror = found->second; + if (found != state.entries.end() && sameRange(found->second->range, range) && found->second->bytes == range->bytes && found->second->writable == range->writable && found->second->heap == heap) mirror = found->second; } if (mirror != nullptr) { - if (prepareRefresh(*mirror, range->address, range->bytes)) appendBlocks(blocks, *mirror, range->address, range->bytes); + if (!heap && prepareRefresh(*mirror, range->address, range->bytes)) appendBlocks(blocks, *mirror, range->address, range->bytes); return mirror; } - // The registered flags were taken from the pages; a range that disagrees is copied by this build. - if (!GuestMemory::Accessible(reinterpret_cast(range->address), range->bytes, range->writable)) return nullptr; + // The registered flags were taken from the pages; an image range that disagrees is copied by this + // build. A heap mirror needs readable pages only: a writable range may hold read-only pages (a + // direct memory page mapped at several addresses is read-only in every view), whose bytes the GPU + // reads like the others and whose write-back storeChanged refuses. + if (!GuestMemory::Accessible(reinterpret_cast(range->address), range->bytes, range->writable && !heap)) return nullptr; mirror = std::make_shared(); mirror->base = range->address; mirror->bytes = range->bytes; mirror->writable = range->writable; + mirror->heap = heap; mirror->range = range; + if (heap) { + // Pending GPU results reach the range before the collect, so the fill below is current. + prepareRefresh(*mirror, range->address, range->bytes); + const auto generation = GuestMemory::CollectWrites(range->address, range->bytes); + constexpr std::uint64_t block = 65536; + mirror->generations.assign(static_cast(((range->address + range->bytes + block - 1) / block) - range->address / block), generation); + std::lock_guard lock(state.mutex); + if (generation == 0) { + ++state.heapUnwatched; + return nullptr; + } + std::uint64_t held = state.heapBytes; + if (const auto found = state.entries.find(range->address); found != state.entries.end() && found->second->heap) held -= found->second->bytes; + if (held + range->bytes > heapMirrorBudget()) heapMirrorFatal(*range, held, "past the budget"); + } try { mirror->buffer = std::make_shared(context, range->bytes, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT); } catch (const std::runtime_error& error) { + if (heap) heapMirrorFatal(*range, state.heapBytes, error.what()); std::fprintf(stderr, "[gpu] image mirror of 0x%llx+0x%llx failed: %s; falling back to copies\n", static_cast(range->address), static_cast(range->bytes), error.what()); std::lock_guard lock(state.mutex); state.failed.insert(range->address); @@ -571,17 +676,20 @@ std::shared_ptr acquireMirror(const Context& context, const std::sh const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::MirrorRefresh); GuestMemory::Read(range->address, mirror->buffer->Bytes()); } - if (range->writable) mirror->shadow.assign(mirror->buffer->Bytes().begin(), mirror->buffer->Bytes().end()); + if (range->writable && !heap) mirror->shadow.assign(mirror->buffer->Bytes().begin(), mirror->buffer->Bytes().end()); std::lock_guard lock(state.mutex); ++state.rebuilds; mirror->serial = (1ull << 63u) | ++state.serials; // A replaced mirror lives on in the regions of recorded work that bound it. + if (const auto found = state.entries.find(range->address); found != state.entries.end() && found->second->heap) state.heapBytes -= found->second->bytes; + if (heap) state.heapBytes += mirror->bytes; state.entries[range->address] = mirror; return mirror; } -// The mirror containing [begin, end) whose registered range still exists, or null. A mirror whose -// Range was replaced is only rebuilt by the next address-based build; until then the range is copied. +// The image mirror containing [begin, end) whose registered range still exists, or null. A mirror +// whose Range was replaced is only rebuilt by the next address-based build; until then the range is +// copied. Heap mirrors are not given out: only an address-based build refreshes them. std::shared_ptr findMirror(const Context& context, std::uint64_t begin, std::uint64_t end) { auto& state = Mirrors(); std::lock_guard lock(state.mutex); @@ -590,7 +698,7 @@ std::shared_ptr findMirror(const Context& context, std::uint64_t be if (found == state.entries.begin()) return nullptr; --found; const auto& mirror = found->second; - if (begin < mirror->base || end > mirror->base + mirror->bytes || mirror->range.expired()) return nullptr; + if (mirror->heap || begin < mirror->base || end > mirror->base + mirror->bytes || mirror->range.expired()) return nullptr; return mirror; } @@ -598,7 +706,14 @@ std::shared_ptr findMirror(const Context& context, std::uint64_t be void sweepMirrors() { auto& state = Mirrors(); std::lock_guard lock(state.mutex); - for (auto it = state.entries.begin(); it != state.entries.end();) it = it->second->range.expired() ? state.entries.erase(it) : std::next(it); + for (auto it = state.entries.begin(); it != state.entries.end();) { + if (!it->second->range.expired()) { + ++it; + continue; + } + if (it->second->heap) state.heapBytes -= it->second->bytes; + it = state.entries.erase(it); + } } void reportMirrors() { @@ -610,16 +725,20 @@ void reportMirrors() { state.lastReport = now; std::size_t count = 0; std::size_t writable = 0; + std::size_t heaps = 0; std::uint64_t bytes = 0; + std::uint64_t heapBytes = 0; { std::lock_guard lock(state.mutex); for (const auto& [base, mirror] : state.entries) { ++count; bytes += mirror->bytes; if (mirror->writable) ++writable; + if (mirror->heap) ++heaps; } + heapBytes = state.heapBytes; } - std::fprintf(stderr, "[buffers] image mirrors: %zu ranges (%.1f MiB, %zu writable), %llu address-based builds served, %llu descriptor sub-ranges bound, %llu rebuilds, %llu refreshes: %llu blocks compared, %llu copied, %llu refresh syncs\n", count, bytes / 1048576.0, writable, static_cast(state.builds), static_cast(state.subranges), static_cast(state.rebuilds), static_cast(state.refreshes), static_cast(state.blocksCompared), static_cast(state.blocksCopied), static_cast(state.syncs)); + std::fprintf(stderr, "[buffers] image mirrors: %zu ranges (%.1f MiB, %zu writable, %zu heap %.1f MiB), %llu address-based builds served, %llu descriptor sub-ranges bound, %llu rebuilds, %llu refreshes: %llu blocks compared, %llu copied, %llu refresh syncs; heap refills %llu, unwatched heaps %llu\n", count, bytes / 1048576.0, writable, heaps, heapBytes / 1048576.0, static_cast(state.builds), static_cast(state.subranges), static_cast(state.rebuilds), static_cast(state.refreshes), static_cast(state.blocksCompared), static_cast(state.blocksCopied), static_cast(state.syncs), static_cast(state.heapRefills), static_cast(state.heapUnwatched)); } // Deferred lease release. The lease an address-based build takes (AcquireRegistered) is dropped by its @@ -778,6 +897,30 @@ AddressSpaceStats AddressSpaceCounters() { return {addressSpaceCacheEnabled(), load(cache.hits), load(cache.rebuiltFirst), load(cache.rebuiltGeneration), load(cache.rebuiltEpoch), load(cache.rebuiltDevice), load(cache.rebuiltWaiterDrop), load(cache.unpublished), load(cache.dissolvedOverlap), load(cache.dissolvedImports), load(cache.waiterDrops)}; } +std::string AddressCopyOverflow(std::vector copies, std::uint64_t limit) { + std::uint64_t total = 0; + for (const auto& copy : copies) total += copy.committed; + if (total <= limit) return {}; + std::sort(copies.begin(), copies.end(), [](const AddressCopy& left, const AddressCopy& right) { return left.committed > right.committed; }); + char text[256]; + std::snprintf(text, sizeof(text), "an address-based build copies %llu MiB of %zu registered ranges, past %llu MiB (APS5_ADDRESS_COPY_MAX_MIB); the largest:", static_cast(total >> 20u), copies.size(), static_cast(limit >> 20u)); + std::string message = text; + for (std::size_t index = 0; index < copies.size() && index < 8; ++index) { + const auto& copy = copies[index]; + std::snprintf(text, sizeof(text), " 0x%llx+0x%llx (%.1f MiB committed, %s)", static_cast(copy.begin), static_cast(copy.end - copy.begin), copy.committed / 1048576.0, copy.reason); + message += text; + } + return message; +} + +MirrorStats MirrorCounters() { + auto& state = Mirrors(); + std::lock_guard lock(state.mutex); + MirrorStats stats{0, state.heapBytes, state.rebuilds, state.blocksCopied, state.heapRefills}; + for (const auto& [base, mirror] : state.entries) stats.heapMirrors += mirror->heap ? 1 : 0; + return stats; +} + const HostImport* HostImportFor(const Context& context, std::uint64_t address, std::size_t bytes) { if (context.hostImportAlignment == 0 || bytes == 0 || bytes > std::numeric_limits::max() - address) return nullptr; auto& state = Imports(); @@ -859,6 +1002,8 @@ void GuestBufferMemory::AcquireRegistered() { // APS5_NO_SORTED_SNAPSHOT_LOOKUP=1: AddSnapshot scans the regions as before instead of searching. static const bool sortedLookup = std::getenv("APS5_NO_SORTED_SNAPSHOT_LOOKUP") == nullptr; std::vector blocks; + // The build's heap mirrors, refreshed together before the blocks are copied. + std::vector heaps; bool mirrored = false; auto& spaces = Spaces(); // Read before the lease below is acquired: a mutation ending in between leaves the space @@ -890,7 +1035,8 @@ void GuestBufferMemory::AcquireRegistered() { for (const auto& region : space->base) { if (region.mirror == nullptr) continue; mirrored = true; - if (region.mirror->writable && prepareRefresh(*region.mirror, region.begin, region.end - region.begin)) appendBlocks(blocks, *region.mirror, region.begin, region.end - region.begin); + if (region.mirror->heap) heaps.push_back(region.mirror.get()); + else if (region.mirror->writable && prepareRefresh(*region.mirror, region.begin, region.end - region.begin)) appendBlocks(blocks, *region.mirror, region.begin, region.end - region.begin); } lap(timing.mirrorsUs, at); } @@ -914,6 +1060,8 @@ void GuestBufferMemory::AcquireRegistered() { // The import lock is taken per range: a mirror's preparation may wait for recorded work, which // must not happen under it. regions.reserve(lease.size()); + // The ranges left to copies, with why no import or mirror serves them (the copy limit below). + std::vector copies; for (const auto& range : lease) { if (!range->readable) continue; validate(range->address, range->bytes); @@ -932,20 +1080,41 @@ void GuestBufferMemory::AcquireRegistered() { regions.push_back(std::move(region)); continue; } - if (!range->releasable && mirrorsEnabled()) { - // Main image: served by a persistent mirror (see ImageMirror); the writable ones are - // compared with guest memory below, all at once. - region.mirror = acquireMirror(context, range, blocks); + if (mirrorsEnabled()) { + // Main image and heaps: served by a persistent mirror (see ImageMirror); the writable + // image ones are compared with guest memory below, all at once. + region.mirror = acquireMirror(context, range, blocks, range->releasable); lap(timing.mirrorsUs, at); if (region.mirror != nullptr) { mirrored = true; + if (region.mirror->heap) heaps.push_back(region.mirror.get()); regions.push_back(std::move(region)); continue; } } addCopiedRange({range->address, range->address + range->bytes, range->writable}); + const auto& copied = regions.back(); + std::uint64_t committed = copied.end - copied.begin; + if (copied.sparse) { + committed = 0; + for (const auto& [first, last] : copied.backed) committed += last - first; + } + // A heap range with readable pages is mirrored unless the arena does not track its writes. + const char* reason = !mirrorsEnabled() ? "mirrors disabled by APS5_NO_LEASE_MIRROR" : copied.sparse ? "unreadable pages" : !range->releasable ? "image mirror refused" : "outside the write-watched arena"; + copies.push_back({copied.begin, copied.end, committed, reason}); } regionsSorted = sortedLookup; + // The ranges left to copies are copied by every build of this space; past the limit the + // process stops rather than copying them per dispatch. + static const std::uint64_t copyLimit = [] { + const char* value = std::getenv("APS5_ADDRESS_COPY_MAX_MIB"); + return (value ? std::strtoull(value, nullptr, 10) : 64ull) << 20u; + }(); + if (const auto overflow = AddressCopyOverflow(std::move(copies), copyLimit); !overflow.empty()) { + std::fprintf(stderr, "FATAL: %s\n", overflow.c_str()); + std::fflush(stderr); + std::abort(); + } lap(timing.importsUs, at); if (addressSpaceCacheEnabled()) { // Published only while the imports the regions point at stand: the table entries below @@ -991,6 +1160,8 @@ void GuestBufferMemory::AcquireRegistered() { } } } + refreshHeapMirrors(heaps, blocks); + lap(timing.mirrorsUs, at); const auto copiedBefore = Mirrors().blocksCopied; compareBlocks(blocks); lap(timing.compareUs, at); @@ -1018,7 +1189,8 @@ void GuestBufferMemory::addCopiedRange(const CopiedRange& range) { // cannot write them, so that equals a snapshot taken now without the extra copy. Reserved heaps // are registered whole while the guest commits pages on demand: committed pages only. region.hostBacked = !range.writable; - auto committed = GuestMemory::DescribeCommitted(range.begin, range.end - range.begin, range.writable); + // Readable pages are copied in, a writable range's read-only ones included (see storeChanged). + auto committed = GuestMemory::DescribeCommitted(range.begin, range.end - range.begin); if (!committed.whole) { region.sparse = true; region.backed = std::move(committed.ranges); @@ -1094,11 +1266,11 @@ void GuestBufferMemory::addDescriptorRegion(std::uint64_t address, std::size_t b // the shader stores to them; what is written back is decided by Writes() alone. Region region{address, address + bytes, true, {}, nullptr}; region.atomic = atomic; - auto committed = GuestMemory::DescribeCommitted(address, bytes, true); + auto committed = GuestMemory::DescribeCommitted(address, bytes); if (!committed.whole) { // A GPU heap bound whole while the guest commits its pages on demand, or a descriptor left - // pointing at unmapped memory: only committed pages are copied in and stored back. Shaders - // must not touch the rest, which reads as zeros. + // pointing at unmapped memory: only readable pages are copied in and stored back (writable + // ones only, see storeChanged). Shaders must not touch the rest, which reads as zeros. region.sparse = true; region.backed = std::move(committed.ranges); } @@ -1670,6 +1842,7 @@ void GuestBufferMemory::UploadFinish(bool addressable) { if (profile) copyUs += microsecondsSince(copyStart); } if (!gpuCopies.empty()) recordGpuCopies(gpuCopies, addressable); + takeHeapReferences(); if (!profile) return; const auto loopUs = microsecondsSince(loopStart); readUs.fetch_add(refreshUs, std::memory_order_relaxed); @@ -2003,14 +2176,64 @@ void GuestBufferMemory::MarkDirectWrites() const { } } -void GuestBufferMemory::WriteBack() { - Require(uploaded && !committed, "guest memory cannot be committed twice or before upload"); +namespace { + +// Stores what the GPU changed in [address, address + current.size()) against `original`. Only the +// writable pages are stored: a read-only page of a writable range (a direct memory page mapped at +// several addresses is read-only in every view, aliased writes are not implemented) must come back +// unchanged, else the write-back fails rather than dropping the GPU's store. +void storeChanged(std::uint64_t address, std::span current, std::span original) { + const auto writable = GuestMemory::DescribeCommitted(address, current.size(), true); + if (writable.whole) { + GuestMemory::WriteChanged(address, current, original); + return; + } + const auto unchanged = [&](std::uint64_t from, std::uint64_t to) { + const auto at = static_cast(from - address); + if (std::memcmp(current.data() + at, original.data() + at, static_cast(to - from)) == 0) return; + char message[192]; + std::snprintf(message, sizeof(message), "the GPU changed guest memory 0x%llx+0x%llx that is not writable (a direct memory page shared by several views; aliased writes are not implemented)", static_cast(from), static_cast(to - from)); + throw std::runtime_error(message); + }; + auto cursor = address; + for (const auto& [first, last] : writable.ranges) { + if (cursor < first) unchanged(cursor, first); + const auto at = static_cast(first - address); + const auto length = static_cast(last - first); + GuestMemory::WriteChanged(first, current.subspan(at, length), original.subspan(at, length)); + cursor = last; + } + if (cursor < address + current.size()) unchanged(cursor, address + current.size()); +} + +// The written ranges sorted, overlapping ones merged. +std::vector> mergeWrites(std::vector> writes) { std::sort(writes.begin(), writes.end()); std::vector> merged; for (const auto& range : writes) { if (!merged.empty() && range.first < merged.back().second) merged.back().second = std::max(merged.back().second, range.second); else merged.push_back(range); } + return merged; +} + +} + +void GuestBufferMemory::takeHeapReferences() { + heapReferences.clear(); + for (const auto& [begin, end] : mergeWrites(writes)) { + const auto* found = owner(begin); + if (found == nullptr || found->mirror == nullptr || !found->mirror->heap || !found->mirror->writable) continue; + Require(end <= found->end, "written range exceeds its heap mirror"); + const auto& mirror = *found->mirror; + const auto bytes = mirror.buffer->Bytes().subspan(static_cast(begin - mirror.base), static_cast(end - begin)); + heapReferences.emplace_back(begin, std::vector(bytes.begin(), bytes.end())); + } +} + +void GuestBufferMemory::WriteBack() { + Require(uploaded && !committed, "guest memory cannot be committed twice or before upload"); + const auto merged = mergeWrites(writes); // The CPU keeps running while the GPU works, so storing whole ranges would roll back its writes to // bytes the shader never touched. Only runs that differ from the uploaded copy are stored. for (const auto& [begin, end] : merged) { @@ -2033,6 +2256,14 @@ void GuestBufferMemory::WriteBack() { const auto offset = static_cast(begin - mirror.base); const auto length = static_cast(end - begin); const auto current = mirror.buffer->Bytes().subspan(offset, length); + if (mirror.heap) { + // The build's reference instead of a shadow; the stores stamp the blocks, and the + // next refresh takes them back from guest memory. + const auto reference = std::find_if(heapReferences.begin(), heapReferences.end(), [&](const auto& entry) { return entry.first == begin; }); + Require(reference != heapReferences.end() && reference->second.size() == length, "heap mirror write-back has no reference"); + storeChanged(begin, current, reference->second); + continue; + } GuestMemory::WriteChanged(begin, current, std::span(mirror.shadow).subspan(offset, length)); std::memcpy(mirror.shadow.data() + offset, current.data(), length); continue; @@ -2065,7 +2296,7 @@ void GuestBufferMemory::WriteBack() { if (!region.sparse) { const auto first = static_cast(begin - region.begin); const auto length = static_cast(end - begin); - GuestMemory::WriteChanged(begin, region.buffer->Bytes().subspan(first, length), std::span(region.uploaded).subspan(first, length)); + storeChanged(begin, region.buffer->Bytes().subspan(first, length), std::span(region.uploaded).subspan(first, length)); continue; } std::size_t packed = 0; @@ -2074,7 +2305,7 @@ void GuestBufferMemory::WriteBack() { const auto to = std::min(last, end); if (from < to) { const auto length = static_cast(to - from); - GuestMemory::WriteChanged(from, region.buffer->Bytes().subspan(static_cast(from - region.begin), length), std::span(region.uploaded).subspan(packed + static_cast(from - first), length)); + storeChanged(from, region.buffer->Bytes().subspan(static_cast(from - region.begin), length), std::span(region.uploaded).subspan(packed + static_cast(from - first), length)); } packed += static_cast(last - first); } @@ -2082,6 +2313,7 @@ void GuestBufferMemory::WriteBack() { committed = true; lease.clear(); space.reset(); + heapReferences.clear(); } std::optional>> GuestBufferMemory::DirectRegions() const { @@ -2089,16 +2321,17 @@ std::optional>> GuestBufferM std::vector> result; if (space != nullptr) { for (const auto& region : space->base) { - if (region.direct == nullptr && (region.mirror == nullptr || region.mirror->writable)) return std::nullopt; + if (region.direct == nullptr && (region.mirror == nullptr || region.mirror->writable || region.mirror->heap)) return std::nullopt; result.emplace_back(region.begin, region.end); } } for (const auto& region : regions) { // A read-only mirror is as fixed as an import: its bytes and VkBuffer never change while its - // Range exists (see ImageMirrorSerial). A writable one is refreshed and written back per build. + // Range exists (see ImageMirrorSerial). A writable one is refreshed and written back per build, + // a heap one refreshed per build. // A staged region is keyed by its import like one bound in place: the import is the source // of its copy-in and the destination of its copy-back, both recorded per use. - const bool fixedMirror = region.mirror != nullptr && !region.mirror->writable; + const bool fixedMirror = region.mirror != nullptr && !region.mirror->writable && !region.mirror->heap; const bool staged = region.gpuCopy && region.deviceLocal; if (region.direct == nullptr && !fixedMirror && !staged) return std::nullopt; if (staged) { diff --git a/core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp b/core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp index c1b432a7f..5b4028eeb 100644 --- a/core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp @@ -2,10 +2,19 @@ #include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp" #include "prx/libc/include/GuestAllocations.hpp" #include "prx/libc/include/GuestHeap.hpp" +#include "prx/libc/include/GuestArena.hpp" +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#endif #include #include #include +#include #include +#include namespace { @@ -26,6 +35,205 @@ ShaderRecompiler::DescriptorBinding binding(Role role, std::uint32_t slot) { return {ShaderRecompiler::DescriptorKind::StorageBuffer, role, 0, slot, 1, {}, false}; } +// Heap ranges that no import serves are mirrored. A read-only one: the build binds the mirror, the +// next build with the range unchanged copies nothing into it, and the blocks a CPU write changed are +// read again. A writable one: a descriptor's store reaches guest memory at write-back, and only the +// bytes the GPU changed (a CPU store made while the GPU works stays). +void heapMirrorTests(const Context& context, const BdaTestAccess& access) { + // Heap mirrors refresh from the arena's write tracking; without it they are never made. + Require(GuestArena::GuestArenaAvailable_nid_postfix() && GuestArena::GuestArenaWriteWatched_nid_postfix(), "the heap mirror tests need the write-watched guest arena"); + constexpr std::size_t bytes = 2 * 65536; + void* block = GuestArena::GuestArenaAllocate_nid_postfix(bytes, 65536); +#ifdef _WIN32 + Require(VirtualAlloc(block, bytes, MEM_COMMIT, PAGE_READWRITE) != nullptr, "cannot commit the heap mirror block"); +#endif + auto* guest = static_cast(block); + std::memset(block, 0x11, bytes); + const auto address = reinterpret_cast(block); + const auto registry = [&](bool add, bool writable) { + auto* mutation = GuestAllocations::GuestAllocationsBegin_nid_postfix(); + if (add) GuestAllocations::GuestAllocationsAdd_nid_postfix(mutation, block, bytes, true, writable); + else GuestAllocations::GuestAllocationsRemove_nid_postfix(mutation, block); + GuestAllocations::GuestAllocationsEnd_nid_postfix(mutation); + }; + // An address-based build that writes 32 bytes at `written` (nothing for 0), with `gpu` run + // between its upload and its write-back; returns the device address serving the range. + const auto build = [&](std::uint64_t written, const std::function& gpu) { + GuestBufferMemory leased(context); + leased.AcquireRegistered(); + if (written != 0) leased.AddWritable(written, 32); + leased.Upload(true); + const auto ranges = leased.AddressRanges(); + const auto found = std::find_if(ranges.begin(), ranges.end(), [&](const auto& range) { return range.begin == address && range.end == address + bytes; }); + Require(found != ranges.end() && found->permissions == ShaderRecompiler::BdaAbi::Read, "the heap range is missing from the BDA table"); + const auto device = found->deviceAddress; + if (gpu) gpu(leased); + leased.WriteBack(); + return device; + }; + // An address-based build sweeps the mirrors whose ranges are gone (an earlier test's too). + const auto sweep = [&] { + GuestBufferMemory leased(context); + leased.AcquireRegistered(); + leased.Upload(true); + leased.WriteBack(); + }; + sweep(); + registry(true, false); + const auto before = MirrorCounters(); + const auto first = build(0, {}); + const auto made = MirrorCounters(); + Require(made.heapMirrors == before.heapMirrors + 1 && made.heapBytes == before.heapBytes + bytes, "a read-only heap range was not mirrored"); + Require(access.addressBytes(first)[65536 + 3] == std::byte{0x11}, "the heap mirror was not filled"); + Require(build(0, {}) == first && MirrorCounters().blocksCopied == made.blocksCopied && MirrorCounters().heapRefills == made.heapRefills, "an unchanged heap range was read into its mirror again"); + guest[65536 + 3] = 0x22; + Require(build(0, {}) == first && MirrorCounters().heapRefills == made.heapRefills + 1 && MirrorCounters().blocksCopied == made.blocksCopied + 1, "a written heap block was not read again alone"); + Require(access.addressBytes(first)[65536 + 3] == std::byte{0x22}, "the heap mirror missed the CPU write"); + const auto one = MirrorCounters(); + guest[5] = 0x33; + guest[65536 + 7] = 0x44; + Require(build(0, {}) == first && MirrorCounters().blocksCopied == one.blocksCopied + 2, "two written heap blocks were not read again"); + Require(access.addressBytes(first)[5] == std::byte{0x33} && access.addressBytes(first)[65536 + 7] == std::byte{0x44}, "the heap mirror missed the CPU writes"); + registry(false, false); + sweep(); + const auto swept = MirrorCounters(); + Require(swept.heapMirrors == before.heapMirrors && swept.heapBytes == before.heapBytes, "the heap mirror outlived its range"); + + registry(true, true); + const auto device = build(address + 16, [&](GuestBufferMemory& leased) { + std::uint32_t adjustment = 0; + const auto view = leased.Descriptor(address + 16, 32, adjustment); + access.bytes(view.buffer)[view.offset + adjustment] = std::byte{0x77}; + guest[16 + 8] = 0x55; + }); + Require(guest[16] == 0x77 && guest[16 + 8] == 0x55, "a writable heap mirror's write-back lost the GPU's store or rolled back the CPU's"); + Require(build(0, {}) == device && access.addressBytes(device)[16] == std::byte{0x77} && access.addressBytes(device)[16 + 8] == std::byte{0x55}, "the writable heap mirror missed the stores"); + registry(false, true); + sweep(); + Require(MirrorCounters().heapMirrors == before.heapMirrors, "the writable heap mirror outlived its range"); + +#ifdef _WIN32 + // A build that throws after queuing a mirror's changed blocks (another mirror's range became + // inaccessible) must not mark those blocks current: the next build copies them. + { + constexpr std::size_t half = 2 * 65536; + void* pair = GuestArena::GuestArenaAllocate_nid_postfix(2 * half, 65536); + Require(VirtualAlloc(pair, 2 * half, MEM_COMMIT, PAGE_READWRITE) != nullptr, "cannot commit the heap mirror pair"); + std::memset(pair, 0x11, 2 * half); + auto* first = static_cast(pair); + auto* second = first + half; + const auto registerPair = [&](bool add) { + auto* mutation = GuestAllocations::GuestAllocationsBegin_nid_postfix(); + for (auto* range : {first, second}) { + if (add) GuestAllocations::GuestAllocationsAdd_nid_postfix(mutation, range, half, true, false); + else GuestAllocations::GuestAllocationsRemove_nid_postfix(mutation, range); + } + GuestAllocations::GuestAllocationsEnd_nid_postfix(mutation); + }; + const auto firstDevice = [&] { + GuestBufferMemory leased(context); + leased.AcquireRegistered(); + leased.Upload(true); + const auto ranges = leased.AddressRanges(); + const auto found = std::find_if(ranges.begin(), ranges.end(), [&](const auto& range) { return range.begin == reinterpret_cast(first); }); + Require(found != ranges.end(), "the first heap range is missing from the BDA table"); + const auto device = found->deviceAddress; + leased.WriteBack(); + return device; + }; + registerPair(true); + const auto device = firstDevice(); + first[5] = 0x66; + DWORD previous = 0; + // A protection change outside the registry, as a heap decommit is: the page cache is told. + Require(VirtualProtect(second, 65536, PAGE_NOACCESS, &previous) != 0, "cannot protect the second heap range"); + GuestAllocations::GuestAllocationsInvalidate_nid_postfix(reinterpret_cast(second), 65536); + bool threw = false; + try { + GuestBufferMemory leased(context); + leased.AcquireRegistered(); + } catch (const std::runtime_error&) { + threw = true; + } + Require(VirtualProtect(second, 65536, PAGE_READWRITE, &previous) != 0, "cannot unprotect the second heap range"); + GuestAllocations::GuestAllocationsInvalidate_nid_postfix(reinterpret_cast(second), 65536); + Require(threw, "a build over an inaccessible heap mirror range did not fail"); + Require(firstDevice() == device && access.addressBytes(device)[5] == std::byte{0x66}, "an interrupted build left a changed heap block marked current"); + registerPair(false); + sweep(); + VirtualFree(pair, 2 * half, MEM_DECOMMIT); + GuestArena::GuestArenaRelease_nid_postfix(pair, 2 * half); + } +#endif +#ifdef _WIN32 + // A writable range with a read-only page (a direct memory page mapped at several addresses is + // read-only in every view): the mirror holds the page's bytes, a store next to it is written + // back, and a GPU change inside it fails the write-back instead of being dropped. A descriptor's + // copy outside an address-based build holds the page's bytes too. + { + constexpr std::size_t size = 2 * 65536; + void* raw = GuestArena::GuestArenaAllocate_nid_postfix(size, 65536); + Require(VirtualAlloc(raw, size, MEM_COMMIT, PAGE_READWRITE) != nullptr, "cannot commit the aliased range"); + auto* bytes8 = static_cast(raw); + std::memset(raw, 0x11, size); + const auto base = reinterpret_cast(raw); + const auto page = base + 65536; + DWORD previous = 0; + Require(VirtualProtect(reinterpret_cast(page), 4096, PAGE_READONLY, &previous) != 0, "cannot make the aliased page read-only"); + GuestAllocations::GuestAllocationsInvalidate_nid_postfix(page, 4096); + const auto registerRange = [&](bool add) { + auto* mutation = GuestAllocations::GuestAllocationsBegin_nid_postfix(); + if (add) GuestAllocations::GuestAllocationsAdd_nid_postfix(mutation, raw, size, true, true); + else GuestAllocations::GuestAllocationsRemove_nid_postfix(mutation, raw); + GuestAllocations::GuestAllocationsEnd_nid_postfix(mutation); + }; + registerRange(true); + const auto storeAt = [&](std::uint64_t at) { + GuestBufferMemory leased(context); + leased.AcquireRegistered(); + leased.AddWritable(page - 16, 32); + leased.Upload(true); + const auto ranges = leased.AddressRanges(); + const auto found = std::find_if(ranges.begin(), ranges.end(), [&](const auto& range) { return range.begin == base; }); + Require(found != ranges.end() && access.addressBytes(found->deviceAddress)[65536 + 8] == std::byte{0x11}, "a writable range with a read-only page was not mirrored with its bytes"); + std::uint32_t adjustment = 0; + const auto view = leased.Descriptor(page - 16, 32, adjustment); + access.bytes(view.buffer)[view.offset + adjustment + static_cast(at - (page - 16))] = std::byte{0x77}; + leased.WriteBack(); + }; + const auto made = MirrorCounters().heapMirrors; + storeAt(page - 8); + Require(MirrorCounters().heapMirrors == made + 1 && bytes8[65536 - 8] == 0x77, "a store next to a read-only page was not written back"); + bool refused = false; + try { + storeAt(page + 4); + } catch (const std::runtime_error& error) { + refused = std::string(error.what()).find("aliased writes are not implemented") != std::string::npos; + } + Require(refused && bytes8[65536 + 4] == 0x11, "a GPU change of a read-only page was not refused"); + { + GuestBufferMemory plain(context); + plain.AddReadable(page, 16); + plain.Upload(false); + std::uint32_t adjustment = 0; + const auto view = plain.Descriptor(page, 16, adjustment); + Require(access.bytes(view.buffer)[view.offset + adjustment] == std::byte{0x11}, "a descriptor over a read-only page of a writable range read zeros"); + plain.WriteBack(); + } + registerRange(false); + sweep(); + Require(VirtualProtect(reinterpret_cast(page), 4096, PAGE_READWRITE, &previous) != 0, "cannot restore the aliased page"); + GuestAllocations::GuestAllocationsInvalidate_nid_postfix(page, 4096); + VirtualFree(raw, size, MEM_DECOMMIT); + GuestArena::GuestArenaRelease_nid_postfix(raw, size); + } +#endif +#ifdef _WIN32 + VirtualFree(block, bytes, MEM_DECOMMIT); +#endif + GuestArena::GuestArenaRelease_nid_postfix(block, bytes); +} + } void RunBdaResourceTests(const Context& context, const BdaTestAccess& access) { @@ -96,6 +304,11 @@ void RunBdaResourceTests(const Context& context, const BdaTestAccess& access) { Require(range.begin == snapshots[0].address && range.end == range.begin + source.size(), "64-bit guest address was truncated"); const auto fault = access.bytes(access.descriptor(5).buffer); for (const auto byte : fault) Require(byte == std::byte{}, "fault buffer was not initialized"); + // A store into a read-only table range names the range and why stores cannot reach it. + const ShaderRecompiler::BdaAbi::Fault denied{ShaderRecompiler::BdaAbi::FaultState::Ready, ShaderRecompiler::BdaAbi::FaultReason::Permission, snapshots[0].address + 4, 4, 0, 0x88, 0}; + std::memcpy(fault.data(), &denied, sizeof(denied)); + reject([&] { resources.WriteBack(); }, "read-only in the BDA table"); + std::memset(fault.data(), 0, fault.size()); resources.WriteBack(); } { @@ -158,4 +371,9 @@ void RunBdaResourceTests(const Context& context, const BdaTestAccess& access) { const auto dropped = AddressSpaceCounters(); if (dropped.enabled) Require(dropped.waiterDrops == after.waiterDrops + 1 && LeaseCounters().cacheDrops == dropped.waiterDrops, "the free did not drop the cached address space"); } + heapMirrorTests(context, access); + // The copies left to an address-based build: nothing within the limit, the largest first past it. + Require(AddressCopyOverflow({{0x1000, 0x3000, 0x2000, "uncommitted pages"}}, 0x2000).empty(), "copies within the limit were refused"); + const auto copies = AddressCopyOverflow({{0x1000, 0x2000, 0x1000, "not mirrored"}, {0x10000, 0x30000, 0x18000, "uncommitted pages"}}, 0x2000); + Require(!copies.empty() && copies.find("0x10000+0x20000 (0.1 MiB committed, uncommitted pages)") < copies.find("0x1000+0x1000"), "the copy limit does not name the largest copy first"); } diff --git a/core/libs/prx/libSceAgcDriver/tests/BdaTests.hpp b/core/libs/prx/libSceAgcDriver/tests/BdaTests.hpp index 0ebb477a9..157ec6e93 100644 --- a/core/libs/prx/libSceAgcDriver/tests/BdaTests.hpp +++ b/core/libs/prx/libSceAgcDriver/tests/BdaTests.hpp @@ -8,6 +8,8 @@ struct BdaTestAccess { std::function(VkBuffer)> bytes; std::function descriptor; + // The bytes of the buffer behind a device address, from that address on. + std::function(VkDeviceAddress)> addressBytes; }; void RunBdaResourceTests(const AgcDriver::Graphics::Context& context, const BdaTestAccess& access); diff --git a/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp b/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp index d8c22f68f..752475279 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp @@ -1273,6 +1273,8 @@ int main() { mock = MockVulkan{}; auto bdaContext = mockContext(); bdaContext.bufferDeviceAddress = true; + // The fault buffer (with its written-page slots) is larger than the mock's 4 KiB default. + bdaContext.limits.maxStorageBufferRange = 1u << 27; RunBdaResourceTests(bdaContext, { [](VkBuffer buffer) -> std::span { return mock.memories.at(mock.bufferMemory.at(buffer)); }, [](std::uint32_t binding) { @@ -1280,6 +1282,12 @@ int main() { if (it->binding == binding) return it->buffers.at(0); } throw std::runtime_error("missing BDA test descriptor"); + }, + [](VkDeviceAddress address) { + // The inverse of mockGetBufferDeviceAddress. + const auto offset = address - 0x100000000000ULL; + const auto buffer = reinterpret_cast(offset / 0x10000); + return std::span(mock.memories.at(mock.bufferMemory.at(buffer))).subspan(offset % 0x10000); } }); Require(mock.live == 0, "BDA resources leaked Vulkan objects");