feat(agc-driver): heap ranges the host imports did not take are mirrored
An address-based build maps every readable registered range. A heap range past APS5_HOST_IMPORT_MIB (or refused by the driver) was copied by every such build: about 3200 ranges, 670 MiB and 500 ms per build on the story-mode load, which then stalled. Such ranges are now served by persistent host-cached mirrors, as the main image is, and the load reaches its next failure. - a heap mirror keeps no shadow; it keeps a write-tracking generation per 64 KiB block and a build copies only the blocks stamped since. The generation is stored by the copy, so a build that throws before it leaves no block marked current with stale bytes - the build gathers its heap mirrors, sorts them and collects each run of adjacent ranges once (a collect costs about 4 us whatever its size; ~3000 mirrors took 16 ms per build, now about 3 ms) - writable ranges are mirrored too: each build keeps the bytes its merged written ranges held at upload and write-back stores only what the GPU changed from them, so a CPU store made meanwhile stays; recorded work writing a range completes before a refresh reads it - the mirrors are limited by APS5_HEAP_MIRROR_MIB (24 GiB by default); a mirror past it, or one whose buffer the driver refuses, stops the process. Heap mirrors are not given to descriptor sub-ranges, image mirror serials or reusable direct regions - past APS5_ADDRESS_COPY_MAX_MIB (64 MiB by default) of committed bytes still copied, a build stops the process, naming the largest ranges and why each is copied (unreadable pages, an image mirror the driver refused, a range outside the write-watched arena, mirrors disabled) - a direct memory page mapped at several addresses is read-only in every view while its range stays writable: copies, descriptor regions and mirrors take every readable page, write-back stores the writable ones and fails on a GPU change inside a read-only page instead of dropping it - a store through the BDA table into a mirrored or copied range faults as before (only writable ranges imported in place are writable there); the fault now names the table range and that rule - the BDA resource tests run with a storage range limit that holds the fault buffer and cover a mirror's fill, reuse, per-block refill, an interrupted refresh, a descriptor store beside a concurrent CPU store, and the copy limit Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
STL1te
parent
c8913314bc
commit
119560206d
|
|
@ -6,6 +6,7 @@
|
|||
#include "prx/libc/include/GuestAllocations.hpp"
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
|
|
@ -123,6 +124,29 @@ struct AddressSpaceStats {
|
|||
};
|
||||
AddressSpaceStats AddressSpaceCounters();
|
||||
|
||||
// The image and heap mirrors (see GuestBufferMemory.cpp): heap mirrors held and their bytes, then
|
||||
// cumulative: mirrors made, 64 KiB blocks copied into them after their fill, heap mirrors read again.
|
||||
struct MirrorStats {
|
||||
std::uint64_t heapMirrors = 0;
|
||||
std::uint64_t heapBytes = 0;
|
||||
std::uint64_t rebuilds = 0;
|
||||
std::uint64_t blocksCopied = 0;
|
||||
std::uint64_t heapRefills = 0;
|
||||
};
|
||||
MirrorStats MirrorCounters();
|
||||
|
||||
// A registered range an address-based build copies instead of serving in place: its committed bytes
|
||||
// and why no import or mirror serves it.
|
||||
struct AddressCopy {
|
||||
std::uint64_t begin;
|
||||
std::uint64_t end;
|
||||
std::uint64_t committed;
|
||||
const char* reason;
|
||||
};
|
||||
// The FATAL message for copies whose committed bytes exceed `limit` (APS5_ADDRESS_COPY_MAX_MIB),
|
||||
// naming the largest ones; empty within the limit.
|
||||
std::string AddressCopyOverflow(std::vector<AddressCopy> copies, std::uint64_t limit);
|
||||
|
||||
class GuestBufferMemory {
|
||||
public:
|
||||
explicit GuestBufferMemory(const Context& context);
|
||||
|
|
@ -301,6 +325,9 @@ private:
|
|||
// Records the import-to-buffer copies of the given gpuCopy regions into the open batch, with
|
||||
// the barriers that order them after earlier recorded writes and before the shaders reading them.
|
||||
void recordGpuCopies(std::span<Region* const> copies, bool addressable);
|
||||
// Keeps the bytes of every merged written range a writable heap mirror serves, as uploaded: the
|
||||
// mirror keeps no shadow, and write-back stores only what the GPU changed from them.
|
||||
void takeHeapReferences();
|
||||
Context context;
|
||||
bool stagingAllowed = false;
|
||||
GuestAllocations::Lease lease;
|
||||
|
|
@ -315,6 +342,8 @@ private:
|
|||
// regions follow the registry's order), so AddSnapshot can search instead of scanning.
|
||||
bool regionsSorted = false;
|
||||
std::vector<std::pair<std::uint64_t, std::uint64_t>> writes;
|
||||
// By the begin of the merged written range (see takeHeapReferences).
|
||||
std::vector<std::pair<std::uint64_t, std::vector<std::byte>>> heapReferences;
|
||||
// UploadPrepare ran (regions are frozen); `uploaded` once UploadFinish ran.
|
||||
bool prepared = false;
|
||||
bool uploaded = false;
|
||||
|
|
|
|||
|
|
@ -254,6 +254,21 @@ void BdaResources::CheckFault() const {
|
|||
}
|
||||
std::ostringstream message;
|
||||
message << "BDA access failed: address=0x" << std::hex << report.address << " instruction=0x" << report.instruction << std::dec << " bytes=" << report.bytes << " stage=" << report.stage << " reason=" << static_cast<std::uint32_t>(report.reason);
|
||||
if (report.reason == ShaderRecompiler::BdaAbi::FaultReason::Permission && table != nullptr) {
|
||||
// Only a writable range imported in place is writable through the table (see
|
||||
// GuestBufferMemory::addressRange): a store into a range served by a mirror or a copy would
|
||||
// not reach guest memory, so it faults. Name the range and how it is served.
|
||||
const auto bytes = table->Bytes();
|
||||
ShaderRecompiler::BdaAbi::Header header{};
|
||||
std::memcpy(&header, bytes.data(), sizeof(header));
|
||||
for (std::uint32_t index = 0; index < header.count; ++index) {
|
||||
ShaderRecompiler::BdaAbi::Range range{};
|
||||
std::memcpy(&range, bytes.data() + sizeof(header) + index * sizeof(range), sizeof(range));
|
||||
if (report.address < range.begin || report.address >= range.end) continue;
|
||||
message << std::hex << "; the store hit 0x" << range.begin << "+0x" << range.end - range.begin << ", read-only in the BDA table: stores through GPU-selected descriptors reach only writable ranges imported in place, not ones served by a mirror or a copy (past APS5_HOST_IMPORT_MIB, or refused by the driver)";
|
||||
break;
|
||||
}
|
||||
}
|
||||
throw std::runtime_error(message.str());
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -36,10 +36,20 @@ namespace AgcDriver::Graphics {
|
|||
// mprotect, which replaces the registered Range object, and a mirror whose Range is gone is rebuilt.
|
||||
// Writable ranges keep a shadow of the guest bytes and copy the 64 KiB blocks that differ on each use.
|
||||
// A mirror never holds the Range shared_ptr: a pinned range makes the guest's mprotect spin and fail.
|
||||
// Heap mirrors serve the readable registered heap ranges the host imports did not take (past
|
||||
// APS5_HOST_IMPORT_MIB, or refused by the driver), which each address-based build copied instead.
|
||||
// They keep no shadow: the arena's write tracking says which 64 KiB blocks changed since the mirror
|
||||
// took them, and those blocks are read again. A writable one is written back per build from the
|
||||
// bytes its written ranges held at upload (takeHeapReferences); stores through the BDA table fault
|
||||
// there, as in a copy.
|
||||
struct ImageMirror {
|
||||
std::uint64_t base = 0;
|
||||
std::uint64_t bytes = 0;
|
||||
bool writable = false;
|
||||
bool heap = false;
|
||||
// Heap mirrors only: per tracker block of the range (block 0 holds `base`), the generation
|
||||
// collected before the block was last read into the buffer.
|
||||
std::vector<std::uint64_t> generations;
|
||||
std::weak_ptr<const GuestAllocations::Range> range;
|
||||
std::shared_ptr<Buffer> buffer;
|
||||
// Writable ranges only: the guest bytes the mirror holds, compared on refresh and at write-back.
|
||||
|
|
@ -306,8 +316,14 @@ struct ImageMirrors {
|
|||
VkDevice device = VK_NULL_HANDLE;
|
||||
std::map<std::uint64_t, std::shared_ptr<ImageMirror>> entries;
|
||||
std::set<std::uint64_t> failed;
|
||||
// Bytes of the heap mirrors in `entries` (against APS5_HEAP_MIRROR_MIB).
|
||||
std::uint64_t heapBytes = 0;
|
||||
// APS5_PROFILE_DRAW counters, reported every 10 s as [buffers] image mirrors.
|
||||
std::uint64_t builds = 0;
|
||||
// Heap mirrors read again because their range changed, and heap ranges left to copies because
|
||||
// the arena does not track their writes.
|
||||
std::uint64_t heapRefills = 0;
|
||||
std::uint64_t heapUnwatched = 0;
|
||||
std::uint64_t subranges = 0;
|
||||
std::uint64_t rebuilds = 0;
|
||||
std::uint64_t refreshes = 0;
|
||||
|
|
@ -361,6 +377,21 @@ bool mirrorsEnabled() {
|
|||
return !disabled;
|
||||
}
|
||||
|
||||
// The heap mirrors' limit: past it the process stops rather than copying gigabytes per dispatch.
|
||||
std::uint64_t heapMirrorBudget() {
|
||||
static const std::uint64_t bytes = [] {
|
||||
const char* value = std::getenv("APS5_HEAP_MIRROR_MIB");
|
||||
return (value ? std::strtoull(value, nullptr, 10) : 24576ull) << 20u;
|
||||
}();
|
||||
return bytes;
|
||||
}
|
||||
|
||||
[[noreturn]] void heapMirrorFatal(const GuestAllocations::Range& range, std::uint64_t held, const char* reason) {
|
||||
std::fprintf(stderr, "FATAL: heap mirror of 0x%llx+0x%llx: %s; heap mirrors hold %llu MiB of %llu MiB (APS5_HEAP_MIRROR_MIB)\n", static_cast<unsigned long long>(range.address), static_cast<unsigned long long>(range.bytes), reason, static_cast<unsigned long long>(held >> 20u), static_cast<unsigned long long>(heapMirrorBudget() >> 20u));
|
||||
std::fflush(stderr);
|
||||
std::abort();
|
||||
}
|
||||
|
||||
bool sameRange(const std::weak_ptr<const GuestAllocations::Range>& mirrored, const std::shared_ptr<const GuestAllocations::Range>& range) {
|
||||
return !mirrored.owner_before(range) && !range.owner_before(mirrored);
|
||||
}
|
||||
|
|
@ -370,13 +401,22 @@ struct RefreshBlock {
|
|||
ImageMirror* mirror;
|
||||
std::uint64_t address;
|
||||
std::size_t length;
|
||||
// Heap mirrors: the generation the block's copy makes current, stored only once the copy ran,
|
||||
// so a build that throws before its copies leaves the block stale in the tracker's eyes too.
|
||||
std::uint64_t generation = 0;
|
||||
};
|
||||
|
||||
// Compares a block's guest bytes with the mirror's shadow and copies a changed block into the device
|
||||
// buffer and the shadow. Blocks are distinct, so blocks of one refresh can be compared in parallel.
|
||||
// buffer and the shadow (a heap mirror's block is known to have changed and is copied). Blocks are
|
||||
// distinct, so blocks of one refresh can be compared in parallel.
|
||||
bool compareBlock(const RefreshBlock& block) {
|
||||
const auto offset = static_cast<std::size_t>(block.address - block.mirror->base);
|
||||
const auto* guest = reinterpret_cast<const std::byte*>(block.address);
|
||||
if (block.mirror->heap) {
|
||||
std::memcpy(block.mirror->buffer->Bytes().data() + offset, guest, block.length);
|
||||
block.mirror->generations[static_cast<std::size_t>(block.address / 65536 - block.mirror->base / 65536)] = block.generation;
|
||||
return true;
|
||||
}
|
||||
if (std::memcmp(block.mirror->shadow.data() + offset, guest, block.length) == 0) return false;
|
||||
std::memcpy(block.mirror->buffer->Bytes().data() + offset, guest, block.length);
|
||||
std::memcpy(block.mirror->shadow.data() + offset, guest, block.length);
|
||||
|
|
@ -480,9 +520,10 @@ private:
|
|||
// completes first, so its results reach the guest and the shadow before the compare and no batch
|
||||
// writes a block being replaced; GPU reads of the mirror by earlier recorded work (an address-based
|
||||
// dispatch whose lease is released at completion) see the newer bytes, as they would on hardware.
|
||||
// Returns whether there is anything to compare.
|
||||
bool prepareRefresh(ImageMirror& mirror, std::uint64_t address, std::uint64_t bytes) {
|
||||
if (!mirror.writable || bytes == 0) return false;
|
||||
// prepareRange does it for [address, address + bytes), prepareRefresh for a writable image mirror and
|
||||
// returns whether there is anything to compare. Heap mirrors are prepared a run of adjacent ranges at
|
||||
// a time, before their write tracking is asked what changed (refreshHeapMirrors).
|
||||
void prepareRange(std::uint64_t address, std::uint64_t bytes) {
|
||||
// The compare reads the pages directly; a mirror is only reused while its Range exists, but a
|
||||
// lease of a build in flight can keep a replaced Range alive past a protection change.
|
||||
Require(GuestMemory::Accessible(reinterpret_cast<const void*>(address), static_cast<std::size_t>(bytes)), "image mirror range became inaccessible");
|
||||
|
|
@ -498,6 +539,11 @@ bool prepareRefresh(ImageMirror& mirror, std::uint64_t address, std::uint64_t by
|
|||
recorder->Sync();
|
||||
}
|
||||
++state.refreshes;
|
||||
}
|
||||
|
||||
bool prepareRefresh(ImageMirror& mirror, std::uint64_t address, std::uint64_t bytes) {
|
||||
if ((!mirror.writable && !mirror.heap) || bytes == 0) return false;
|
||||
prepareRange(address, bytes);
|
||||
return true;
|
||||
}
|
||||
|
||||
|
|
@ -528,11 +574,49 @@ void refreshMirror(ImageMirror& mirror, std::uint64_t address, std::uint64_t byt
|
|||
compareBlocks(blocks);
|
||||
}
|
||||
|
||||
// Appends to `blocks` the blocks of one build's heap mirrors that a CPU or driver store changed
|
||||
// since the mirrors took them. The mirrors of adjacent ranges are prepared and collected as one run: a
|
||||
// collect walks the run's page tables at a fixed cost of microseconds, which paid per mirror (thousands
|
||||
// on a story-mode load) was most of an address-based build. The copies follow and store the run's
|
||||
// collected generation (see RefreshBlock).
|
||||
void refreshHeapMirrors(std::vector<ImageMirror*>& mirrors, std::vector<RefreshBlock>& blocks) {
|
||||
constexpr std::uint64_t block = 65536;
|
||||
std::sort(mirrors.begin(), mirrors.end(), [](const ImageMirror* left, const ImageMirror* right) { return left->base < right->base; });
|
||||
thread_local std::vector<std::uint8_t> changed;
|
||||
for (std::size_t first = 0; first < mirrors.size();) {
|
||||
auto last = first + 1;
|
||||
while (last < mirrors.size() && mirrors[last]->base == mirrors[last - 1]->base + mirrors[last - 1]->bytes) ++last;
|
||||
const auto begin = mirrors[first]->base;
|
||||
const auto bytes = mirrors[last - 1]->base + mirrors[last - 1]->bytes - begin;
|
||||
prepareRange(begin, bytes);
|
||||
const auto generation = GuestMemory::CollectWrites(begin, static_cast<std::size_t>(bytes));
|
||||
Require(generation != 0, "a heap mirror's range is no longer write-watched");
|
||||
for (auto index = first; index < last; ++index) {
|
||||
auto& mirror = *mirrors[index];
|
||||
changed.assign(mirror.generations.size(), 0);
|
||||
GuestMemory::ChangedBlocks(mirror.base, static_cast<std::size_t>(mirror.bytes), mirror.generations, changed);
|
||||
const auto aligned = mirror.base / block * block;
|
||||
const auto end = mirror.base + mirror.bytes;
|
||||
bool refilled = false;
|
||||
for (std::size_t at = 0; at < changed.size(); ++at) {
|
||||
if (changed[at] == 0) continue;
|
||||
const auto from = std::max(mirror.base, aligned + at * block);
|
||||
blocks.push_back({&mirror, from, static_cast<std::size_t>(std::min(end, aligned + (at + 1) * block) - from), generation});
|
||||
refilled = true;
|
||||
}
|
||||
if (refilled) ++Mirrors().heapRefills;
|
||||
}
|
||||
first = last;
|
||||
}
|
||||
}
|
||||
|
||||
// The mirror for a leased image range: the existing one, or a new one when none exists or the
|
||||
// registered Range object changed (a protection change). Null when the range cannot be mirrored. An
|
||||
// existing writable mirror's blocks are appended to `blocks` for the caller's one compare of every
|
||||
// mirror of the build (prepareRefresh ran); a new mirror is filled here.
|
||||
std::shared_ptr<ImageMirror> acquireMirror(const Context& context, const std::shared_ptr<const GuestAllocations::Range>& range, std::vector<RefreshBlock>& blocks) {
|
||||
// mirror of the build (prepareRefresh ran); the caller refreshes an existing heap mirror with the
|
||||
// build's others (refreshHeapMirrors); a new mirror is filled here. A heap mirror past APS5_HEAP_MIRROR_MIB, or whose buffer the driver
|
||||
// refuses, stops the process.
|
||||
std::shared_ptr<ImageMirror> acquireMirror(const Context& context, const std::shared_ptr<const GuestAllocations::Range>& range, std::vector<RefreshBlock>& blocks, bool heap) {
|
||||
auto& state = Mirrors();
|
||||
std::shared_ptr<ImageMirror> mirror;
|
||||
{
|
||||
|
|
@ -540,26 +624,47 @@ std::shared_ptr<ImageMirror> acquireMirror(const Context& context, const std::sh
|
|||
if (state.device != context.device) {
|
||||
state.entries.clear();
|
||||
state.failed.clear();
|
||||
state.heapBytes = 0;
|
||||
state.device = context.device;
|
||||
}
|
||||
if (state.failed.contains(range->address)) return nullptr;
|
||||
const auto found = state.entries.find(range->address);
|
||||
if (found != state.entries.end() && sameRange(found->second->range, range) && found->second->bytes == range->bytes && found->second->writable == range->writable) mirror = found->second;
|
||||
if (found != state.entries.end() && sameRange(found->second->range, range) && found->second->bytes == range->bytes && found->second->writable == range->writable && found->second->heap == heap) mirror = found->second;
|
||||
}
|
||||
if (mirror != nullptr) {
|
||||
if (prepareRefresh(*mirror, range->address, range->bytes)) appendBlocks(blocks, *mirror, range->address, range->bytes);
|
||||
if (!heap && prepareRefresh(*mirror, range->address, range->bytes)) appendBlocks(blocks, *mirror, range->address, range->bytes);
|
||||
return mirror;
|
||||
}
|
||||
// The registered flags were taken from the pages; a range that disagrees is copied by this build.
|
||||
if (!GuestMemory::Accessible(reinterpret_cast<const void*>(range->address), range->bytes, range->writable)) return nullptr;
|
||||
// The registered flags were taken from the pages; an image range that disagrees is copied by this
|
||||
// build. A heap mirror needs readable pages only: a writable range may hold read-only pages (a
|
||||
// direct memory page mapped at several addresses is read-only in every view), whose bytes the GPU
|
||||
// reads like the others and whose write-back storeChanged refuses.
|
||||
if (!GuestMemory::Accessible(reinterpret_cast<const void*>(range->address), range->bytes, range->writable && !heap)) return nullptr;
|
||||
mirror = std::make_shared<ImageMirror>();
|
||||
mirror->base = range->address;
|
||||
mirror->bytes = range->bytes;
|
||||
mirror->writable = range->writable;
|
||||
mirror->heap = heap;
|
||||
mirror->range = range;
|
||||
if (heap) {
|
||||
// Pending GPU results reach the range before the collect, so the fill below is current.
|
||||
prepareRefresh(*mirror, range->address, range->bytes);
|
||||
const auto generation = GuestMemory::CollectWrites(range->address, range->bytes);
|
||||
constexpr std::uint64_t block = 65536;
|
||||
mirror->generations.assign(static_cast<std::size_t>(((range->address + range->bytes + block - 1) / block) - range->address / block), generation);
|
||||
std::lock_guard lock(state.mutex);
|
||||
if (generation == 0) {
|
||||
++state.heapUnwatched;
|
||||
return nullptr;
|
||||
}
|
||||
std::uint64_t held = state.heapBytes;
|
||||
if (const auto found = state.entries.find(range->address); found != state.entries.end() && found->second->heap) held -= found->second->bytes;
|
||||
if (held + range->bytes > heapMirrorBudget()) heapMirrorFatal(*range, held, "past the budget");
|
||||
}
|
||||
try {
|
||||
mirror->buffer = std::make_shared<Buffer>(context, range->bytes, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT);
|
||||
} catch (const std::runtime_error& error) {
|
||||
if (heap) heapMirrorFatal(*range, state.heapBytes, error.what());
|
||||
std::fprintf(stderr, "[gpu] image mirror of 0x%llx+0x%llx failed: %s; falling back to copies\n", static_cast<unsigned long long>(range->address), static_cast<unsigned long long>(range->bytes), error.what());
|
||||
std::lock_guard lock(state.mutex);
|
||||
state.failed.insert(range->address);
|
||||
|
|
@ -571,17 +676,20 @@ std::shared_ptr<ImageMirror> acquireMirror(const Context& context, const std::sh
|
|||
const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::MirrorRefresh);
|
||||
GuestMemory::Read(range->address, mirror->buffer->Bytes());
|
||||
}
|
||||
if (range->writable) mirror->shadow.assign(mirror->buffer->Bytes().begin(), mirror->buffer->Bytes().end());
|
||||
if (range->writable && !heap) mirror->shadow.assign(mirror->buffer->Bytes().begin(), mirror->buffer->Bytes().end());
|
||||
std::lock_guard lock(state.mutex);
|
||||
++state.rebuilds;
|
||||
mirror->serial = (1ull << 63u) | ++state.serials;
|
||||
// A replaced mirror lives on in the regions of recorded work that bound it.
|
||||
if (const auto found = state.entries.find(range->address); found != state.entries.end() && found->second->heap) state.heapBytes -= found->second->bytes;
|
||||
if (heap) state.heapBytes += mirror->bytes;
|
||||
state.entries[range->address] = mirror;
|
||||
return mirror;
|
||||
}
|
||||
|
||||
// The mirror containing [begin, end) whose registered range still exists, or null. A mirror whose
|
||||
// Range was replaced is only rebuilt by the next address-based build; until then the range is copied.
|
||||
// The image mirror containing [begin, end) whose registered range still exists, or null. A mirror
|
||||
// whose Range was replaced is only rebuilt by the next address-based build; until then the range is
|
||||
// copied. Heap mirrors are not given out: only an address-based build refreshes them.
|
||||
std::shared_ptr<ImageMirror> findMirror(const Context& context, std::uint64_t begin, std::uint64_t end) {
|
||||
auto& state = Mirrors();
|
||||
std::lock_guard lock(state.mutex);
|
||||
|
|
@ -590,7 +698,7 @@ std::shared_ptr<ImageMirror> findMirror(const Context& context, std::uint64_t be
|
|||
if (found == state.entries.begin()) return nullptr;
|
||||
--found;
|
||||
const auto& mirror = found->second;
|
||||
if (begin < mirror->base || end > mirror->base + mirror->bytes || mirror->range.expired()) return nullptr;
|
||||
if (mirror->heap || begin < mirror->base || end > mirror->base + mirror->bytes || mirror->range.expired()) return nullptr;
|
||||
return mirror;
|
||||
}
|
||||
|
||||
|
|
@ -598,7 +706,14 @@ std::shared_ptr<ImageMirror> findMirror(const Context& context, std::uint64_t be
|
|||
void sweepMirrors() {
|
||||
auto& state = Mirrors();
|
||||
std::lock_guard lock(state.mutex);
|
||||
for (auto it = state.entries.begin(); it != state.entries.end();) it = it->second->range.expired() ? state.entries.erase(it) : std::next(it);
|
||||
for (auto it = state.entries.begin(); it != state.entries.end();) {
|
||||
if (!it->second->range.expired()) {
|
||||
++it;
|
||||
continue;
|
||||
}
|
||||
if (it->second->heap) state.heapBytes -= it->second->bytes;
|
||||
it = state.entries.erase(it);
|
||||
}
|
||||
}
|
||||
|
||||
void reportMirrors() {
|
||||
|
|
@ -610,16 +725,20 @@ void reportMirrors() {
|
|||
state.lastReport = now;
|
||||
std::size_t count = 0;
|
||||
std::size_t writable = 0;
|
||||
std::size_t heaps = 0;
|
||||
std::uint64_t bytes = 0;
|
||||
std::uint64_t heapBytes = 0;
|
||||
{
|
||||
std::lock_guard lock(state.mutex);
|
||||
for (const auto& [base, mirror] : state.entries) {
|
||||
++count;
|
||||
bytes += mirror->bytes;
|
||||
if (mirror->writable) ++writable;
|
||||
if (mirror->heap) ++heaps;
|
||||
}
|
||||
heapBytes = state.heapBytes;
|
||||
}
|
||||
std::fprintf(stderr, "[buffers] image mirrors: %zu ranges (%.1f MiB, %zu writable), %llu address-based builds served, %llu descriptor sub-ranges bound, %llu rebuilds, %llu refreshes: %llu blocks compared, %llu copied, %llu refresh syncs\n", count, bytes / 1048576.0, writable, static_cast<unsigned long long>(state.builds), static_cast<unsigned long long>(state.subranges), static_cast<unsigned long long>(state.rebuilds), static_cast<unsigned long long>(state.refreshes), static_cast<unsigned long long>(state.blocksCompared), static_cast<unsigned long long>(state.blocksCopied), static_cast<unsigned long long>(state.syncs));
|
||||
std::fprintf(stderr, "[buffers] image mirrors: %zu ranges (%.1f MiB, %zu writable, %zu heap %.1f MiB), %llu address-based builds served, %llu descriptor sub-ranges bound, %llu rebuilds, %llu refreshes: %llu blocks compared, %llu copied, %llu refresh syncs; heap refills %llu, unwatched heaps %llu\n", count, bytes / 1048576.0, writable, heaps, heapBytes / 1048576.0, static_cast<unsigned long long>(state.builds), static_cast<unsigned long long>(state.subranges), static_cast<unsigned long long>(state.rebuilds), static_cast<unsigned long long>(state.refreshes), static_cast<unsigned long long>(state.blocksCompared), static_cast<unsigned long long>(state.blocksCopied), static_cast<unsigned long long>(state.syncs), static_cast<unsigned long long>(state.heapRefills), static_cast<unsigned long long>(state.heapUnwatched));
|
||||
}
|
||||
|
||||
// Deferred lease release. The lease an address-based build takes (AcquireRegistered) is dropped by its
|
||||
|
|
@ -778,6 +897,30 @@ AddressSpaceStats AddressSpaceCounters() {
|
|||
return {addressSpaceCacheEnabled(), load(cache.hits), load(cache.rebuiltFirst), load(cache.rebuiltGeneration), load(cache.rebuiltEpoch), load(cache.rebuiltDevice), load(cache.rebuiltWaiterDrop), load(cache.unpublished), load(cache.dissolvedOverlap), load(cache.dissolvedImports), load(cache.waiterDrops)};
|
||||
}
|
||||
|
||||
std::string AddressCopyOverflow(std::vector<AddressCopy> copies, std::uint64_t limit) {
|
||||
std::uint64_t total = 0;
|
||||
for (const auto& copy : copies) total += copy.committed;
|
||||
if (total <= limit) return {};
|
||||
std::sort(copies.begin(), copies.end(), [](const AddressCopy& left, const AddressCopy& right) { return left.committed > right.committed; });
|
||||
char text[256];
|
||||
std::snprintf(text, sizeof(text), "an address-based build copies %llu MiB of %zu registered ranges, past %llu MiB (APS5_ADDRESS_COPY_MAX_MIB); the largest:", static_cast<unsigned long long>(total >> 20u), copies.size(), static_cast<unsigned long long>(limit >> 20u));
|
||||
std::string message = text;
|
||||
for (std::size_t index = 0; index < copies.size() && index < 8; ++index) {
|
||||
const auto& copy = copies[index];
|
||||
std::snprintf(text, sizeof(text), " 0x%llx+0x%llx (%.1f MiB committed, %s)", static_cast<unsigned long long>(copy.begin), static_cast<unsigned long long>(copy.end - copy.begin), copy.committed / 1048576.0, copy.reason);
|
||||
message += text;
|
||||
}
|
||||
return message;
|
||||
}
|
||||
|
||||
MirrorStats MirrorCounters() {
|
||||
auto& state = Mirrors();
|
||||
std::lock_guard lock(state.mutex);
|
||||
MirrorStats stats{0, state.heapBytes, state.rebuilds, state.blocksCopied, state.heapRefills};
|
||||
for (const auto& [base, mirror] : state.entries) stats.heapMirrors += mirror->heap ? 1 : 0;
|
||||
return stats;
|
||||
}
|
||||
|
||||
const HostImport* HostImportFor(const Context& context, std::uint64_t address, std::size_t bytes) {
|
||||
if (context.hostImportAlignment == 0 || bytes == 0 || bytes > std::numeric_limits<std::uint64_t>::max() - address) return nullptr;
|
||||
auto& state = Imports();
|
||||
|
|
@ -859,6 +1002,8 @@ void GuestBufferMemory::AcquireRegistered() {
|
|||
// APS5_NO_SORTED_SNAPSHOT_LOOKUP=1: AddSnapshot scans the regions as before instead of searching.
|
||||
static const bool sortedLookup = std::getenv("APS5_NO_SORTED_SNAPSHOT_LOOKUP") == nullptr;
|
||||
std::vector<RefreshBlock> blocks;
|
||||
// The build's heap mirrors, refreshed together before the blocks are copied.
|
||||
std::vector<ImageMirror*> heaps;
|
||||
bool mirrored = false;
|
||||
auto& spaces = Spaces();
|
||||
// Read before the lease below is acquired: a mutation ending in between leaves the space
|
||||
|
|
@ -890,7 +1035,8 @@ void GuestBufferMemory::AcquireRegistered() {
|
|||
for (const auto& region : space->base) {
|
||||
if (region.mirror == nullptr) continue;
|
||||
mirrored = true;
|
||||
if (region.mirror->writable && prepareRefresh(*region.mirror, region.begin, region.end - region.begin)) appendBlocks(blocks, *region.mirror, region.begin, region.end - region.begin);
|
||||
if (region.mirror->heap) heaps.push_back(region.mirror.get());
|
||||
else if (region.mirror->writable && prepareRefresh(*region.mirror, region.begin, region.end - region.begin)) appendBlocks(blocks, *region.mirror, region.begin, region.end - region.begin);
|
||||
}
|
||||
lap(timing.mirrorsUs, at);
|
||||
}
|
||||
|
|
@ -914,6 +1060,8 @@ void GuestBufferMemory::AcquireRegistered() {
|
|||
// The import lock is taken per range: a mirror's preparation may wait for recorded work, which
|
||||
// must not happen under it.
|
||||
regions.reserve(lease.size());
|
||||
// The ranges left to copies, with why no import or mirror serves them (the copy limit below).
|
||||
std::vector<AddressCopy> copies;
|
||||
for (const auto& range : lease) {
|
||||
if (!range->readable) continue;
|
||||
validate(range->address, range->bytes);
|
||||
|
|
@ -932,20 +1080,41 @@ void GuestBufferMemory::AcquireRegistered() {
|
|||
regions.push_back(std::move(region));
|
||||
continue;
|
||||
}
|
||||
if (!range->releasable && mirrorsEnabled()) {
|
||||
// Main image: served by a persistent mirror (see ImageMirror); the writable ones are
|
||||
// compared with guest memory below, all at once.
|
||||
region.mirror = acquireMirror(context, range, blocks);
|
||||
if (mirrorsEnabled()) {
|
||||
// Main image and heaps: served by a persistent mirror (see ImageMirror); the writable
|
||||
// image ones are compared with guest memory below, all at once.
|
||||
region.mirror = acquireMirror(context, range, blocks, range->releasable);
|
||||
lap(timing.mirrorsUs, at);
|
||||
if (region.mirror != nullptr) {
|
||||
mirrored = true;
|
||||
if (region.mirror->heap) heaps.push_back(region.mirror.get());
|
||||
regions.push_back(std::move(region));
|
||||
continue;
|
||||
}
|
||||
}
|
||||
addCopiedRange({range->address, range->address + range->bytes, range->writable});
|
||||
const auto& copied = regions.back();
|
||||
std::uint64_t committed = copied.end - copied.begin;
|
||||
if (copied.sparse) {
|
||||
committed = 0;
|
||||
for (const auto& [first, last] : copied.backed) committed += last - first;
|
||||
}
|
||||
// A heap range with readable pages is mirrored unless the arena does not track its writes.
|
||||
const char* reason = !mirrorsEnabled() ? "mirrors disabled by APS5_NO_LEASE_MIRROR" : copied.sparse ? "unreadable pages" : !range->releasable ? "image mirror refused" : "outside the write-watched arena";
|
||||
copies.push_back({copied.begin, copied.end, committed, reason});
|
||||
}
|
||||
regionsSorted = sortedLookup;
|
||||
// The ranges left to copies are copied by every build of this space; past the limit the
|
||||
// process stops rather than copying them per dispatch.
|
||||
static const std::uint64_t copyLimit = [] {
|
||||
const char* value = std::getenv("APS5_ADDRESS_COPY_MAX_MIB");
|
||||
return (value ? std::strtoull(value, nullptr, 10) : 64ull) << 20u;
|
||||
}();
|
||||
if (const auto overflow = AddressCopyOverflow(std::move(copies), copyLimit); !overflow.empty()) {
|
||||
std::fprintf(stderr, "FATAL: %s\n", overflow.c_str());
|
||||
std::fflush(stderr);
|
||||
std::abort();
|
||||
}
|
||||
lap(timing.importsUs, at);
|
||||
if (addressSpaceCacheEnabled()) {
|
||||
// Published only while the imports the regions point at stand: the table entries below
|
||||
|
|
@ -991,6 +1160,8 @@ void GuestBufferMemory::AcquireRegistered() {
|
|||
}
|
||||
}
|
||||
}
|
||||
refreshHeapMirrors(heaps, blocks);
|
||||
lap(timing.mirrorsUs, at);
|
||||
const auto copiedBefore = Mirrors().blocksCopied;
|
||||
compareBlocks(blocks);
|
||||
lap(timing.compareUs, at);
|
||||
|
|
@ -1018,7 +1189,8 @@ void GuestBufferMemory::addCopiedRange(const CopiedRange& range) {
|
|||
// cannot write them, so that equals a snapshot taken now without the extra copy. Reserved heaps
|
||||
// are registered whole while the guest commits pages on demand: committed pages only.
|
||||
region.hostBacked = !range.writable;
|
||||
auto committed = GuestMemory::DescribeCommitted(range.begin, range.end - range.begin, range.writable);
|
||||
// Readable pages are copied in, a writable range's read-only ones included (see storeChanged).
|
||||
auto committed = GuestMemory::DescribeCommitted(range.begin, range.end - range.begin);
|
||||
if (!committed.whole) {
|
||||
region.sparse = true;
|
||||
region.backed = std::move(committed.ranges);
|
||||
|
|
@ -1094,11 +1266,11 @@ void GuestBufferMemory::addDescriptorRegion(std::uint64_t address, std::size_t b
|
|||
// the shader stores to them; what is written back is decided by Writes() alone.
|
||||
Region region{address, address + bytes, true, {}, nullptr};
|
||||
region.atomic = atomic;
|
||||
auto committed = GuestMemory::DescribeCommitted(address, bytes, true);
|
||||
auto committed = GuestMemory::DescribeCommitted(address, bytes);
|
||||
if (!committed.whole) {
|
||||
// A GPU heap bound whole while the guest commits its pages on demand, or a descriptor left
|
||||
// pointing at unmapped memory: only committed pages are copied in and stored back. Shaders
|
||||
// must not touch the rest, which reads as zeros.
|
||||
// pointing at unmapped memory: only readable pages are copied in and stored back (writable
|
||||
// ones only, see storeChanged). Shaders must not touch the rest, which reads as zeros.
|
||||
region.sparse = true;
|
||||
region.backed = std::move(committed.ranges);
|
||||
}
|
||||
|
|
@ -1670,6 +1842,7 @@ void GuestBufferMemory::UploadFinish(bool addressable) {
|
|||
if (profile) copyUs += microsecondsSince(copyStart);
|
||||
}
|
||||
if (!gpuCopies.empty()) recordGpuCopies(gpuCopies, addressable);
|
||||
takeHeapReferences();
|
||||
if (!profile) return;
|
||||
const auto loopUs = microsecondsSince(loopStart);
|
||||
readUs.fetch_add(refreshUs, std::memory_order_relaxed);
|
||||
|
|
@ -2003,14 +2176,64 @@ void GuestBufferMemory::MarkDirectWrites() const {
|
|||
}
|
||||
}
|
||||
|
||||
void GuestBufferMemory::WriteBack() {
|
||||
Require(uploaded && !committed, "guest memory cannot be committed twice or before upload");
|
||||
namespace {
|
||||
|
||||
// Stores what the GPU changed in [address, address + current.size()) against `original`. Only the
|
||||
// writable pages are stored: a read-only page of a writable range (a direct memory page mapped at
|
||||
// several addresses is read-only in every view, aliased writes are not implemented) must come back
|
||||
// unchanged, else the write-back fails rather than dropping the GPU's store.
|
||||
void storeChanged(std::uint64_t address, std::span<const std::byte> current, std::span<const std::byte> original) {
|
||||
const auto writable = GuestMemory::DescribeCommitted(address, current.size(), true);
|
||||
if (writable.whole) {
|
||||
GuestMemory::WriteChanged(address, current, original);
|
||||
return;
|
||||
}
|
||||
const auto unchanged = [&](std::uint64_t from, std::uint64_t to) {
|
||||
const auto at = static_cast<std::size_t>(from - address);
|
||||
if (std::memcmp(current.data() + at, original.data() + at, static_cast<std::size_t>(to - from)) == 0) return;
|
||||
char message[192];
|
||||
std::snprintf(message, sizeof(message), "the GPU changed guest memory 0x%llx+0x%llx that is not writable (a direct memory page shared by several views; aliased writes are not implemented)", static_cast<unsigned long long>(from), static_cast<unsigned long long>(to - from));
|
||||
throw std::runtime_error(message);
|
||||
};
|
||||
auto cursor = address;
|
||||
for (const auto& [first, last] : writable.ranges) {
|
||||
if (cursor < first) unchanged(cursor, first);
|
||||
const auto at = static_cast<std::size_t>(first - address);
|
||||
const auto length = static_cast<std::size_t>(last - first);
|
||||
GuestMemory::WriteChanged(first, current.subspan(at, length), original.subspan(at, length));
|
||||
cursor = last;
|
||||
}
|
||||
if (cursor < address + current.size()) unchanged(cursor, address + current.size());
|
||||
}
|
||||
|
||||
// The written ranges sorted, overlapping ones merged.
|
||||
std::vector<std::pair<std::uint64_t, std::uint64_t>> mergeWrites(std::vector<std::pair<std::uint64_t, std::uint64_t>> writes) {
|
||||
std::sort(writes.begin(), writes.end());
|
||||
std::vector<std::pair<std::uint64_t, std::uint64_t>> merged;
|
||||
for (const auto& range : writes) {
|
||||
if (!merged.empty() && range.first < merged.back().second) merged.back().second = std::max(merged.back().second, range.second);
|
||||
else merged.push_back(range);
|
||||
}
|
||||
return merged;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
void GuestBufferMemory::takeHeapReferences() {
|
||||
heapReferences.clear();
|
||||
for (const auto& [begin, end] : mergeWrites(writes)) {
|
||||
const auto* found = owner(begin);
|
||||
if (found == nullptr || found->mirror == nullptr || !found->mirror->heap || !found->mirror->writable) continue;
|
||||
Require(end <= found->end, "written range exceeds its heap mirror");
|
||||
const auto& mirror = *found->mirror;
|
||||
const auto bytes = mirror.buffer->Bytes().subspan(static_cast<std::size_t>(begin - mirror.base), static_cast<std::size_t>(end - begin));
|
||||
heapReferences.emplace_back(begin, std::vector<std::byte>(bytes.begin(), bytes.end()));
|
||||
}
|
||||
}
|
||||
|
||||
void GuestBufferMemory::WriteBack() {
|
||||
Require(uploaded && !committed, "guest memory cannot be committed twice or before upload");
|
||||
const auto merged = mergeWrites(writes);
|
||||
// The CPU keeps running while the GPU works, so storing whole ranges would roll back its writes to
|
||||
// bytes the shader never touched. Only runs that differ from the uploaded copy are stored.
|
||||
for (const auto& [begin, end] : merged) {
|
||||
|
|
@ -2033,6 +2256,14 @@ void GuestBufferMemory::WriteBack() {
|
|||
const auto offset = static_cast<std::size_t>(begin - mirror.base);
|
||||
const auto length = static_cast<std::size_t>(end - begin);
|
||||
const auto current = mirror.buffer->Bytes().subspan(offset, length);
|
||||
if (mirror.heap) {
|
||||
// The build's reference instead of a shadow; the stores stamp the blocks, and the
|
||||
// next refresh takes them back from guest memory.
|
||||
const auto reference = std::find_if(heapReferences.begin(), heapReferences.end(), [&](const auto& entry) { return entry.first == begin; });
|
||||
Require(reference != heapReferences.end() && reference->second.size() == length, "heap mirror write-back has no reference");
|
||||
storeChanged(begin, current, reference->second);
|
||||
continue;
|
||||
}
|
||||
GuestMemory::WriteChanged(begin, current, std::span<const std::byte>(mirror.shadow).subspan(offset, length));
|
||||
std::memcpy(mirror.shadow.data() + offset, current.data(), length);
|
||||
continue;
|
||||
|
|
@ -2065,7 +2296,7 @@ void GuestBufferMemory::WriteBack() {
|
|||
if (!region.sparse) {
|
||||
const auto first = static_cast<std::size_t>(begin - region.begin);
|
||||
const auto length = static_cast<std::size_t>(end - begin);
|
||||
GuestMemory::WriteChanged(begin, region.buffer->Bytes().subspan(first, length), std::span<const std::byte>(region.uploaded).subspan(first, length));
|
||||
storeChanged(begin, region.buffer->Bytes().subspan(first, length), std::span<const std::byte>(region.uploaded).subspan(first, length));
|
||||
continue;
|
||||
}
|
||||
std::size_t packed = 0;
|
||||
|
|
@ -2074,7 +2305,7 @@ void GuestBufferMemory::WriteBack() {
|
|||
const auto to = std::min(last, end);
|
||||
if (from < to) {
|
||||
const auto length = static_cast<std::size_t>(to - from);
|
||||
GuestMemory::WriteChanged(from, region.buffer->Bytes().subspan(static_cast<std::size_t>(from - region.begin), length), std::span<const std::byte>(region.uploaded).subspan(packed + static_cast<std::size_t>(from - first), length));
|
||||
storeChanged(from, region.buffer->Bytes().subspan(static_cast<std::size_t>(from - region.begin), length), std::span<const std::byte>(region.uploaded).subspan(packed + static_cast<std::size_t>(from - first), length));
|
||||
}
|
||||
packed += static_cast<std::size_t>(last - first);
|
||||
}
|
||||
|
|
@ -2082,6 +2313,7 @@ void GuestBufferMemory::WriteBack() {
|
|||
committed = true;
|
||||
lease.clear();
|
||||
space.reset();
|
||||
heapReferences.clear();
|
||||
}
|
||||
|
||||
std::optional<std::vector<std::pair<std::uint64_t, std::uint64_t>>> GuestBufferMemory::DirectRegions() const {
|
||||
|
|
@ -2089,16 +2321,17 @@ std::optional<std::vector<std::pair<std::uint64_t, std::uint64_t>>> GuestBufferM
|
|||
std::vector<std::pair<std::uint64_t, std::uint64_t>> result;
|
||||
if (space != nullptr) {
|
||||
for (const auto& region : space->base) {
|
||||
if (region.direct == nullptr && (region.mirror == nullptr || region.mirror->writable)) return std::nullopt;
|
||||
if (region.direct == nullptr && (region.mirror == nullptr || region.mirror->writable || region.mirror->heap)) return std::nullopt;
|
||||
result.emplace_back(region.begin, region.end);
|
||||
}
|
||||
}
|
||||
for (const auto& region : regions) {
|
||||
// A read-only mirror is as fixed as an import: its bytes and VkBuffer never change while its
|
||||
// Range exists (see ImageMirrorSerial). A writable one is refreshed and written back per build.
|
||||
// Range exists (see ImageMirrorSerial). A writable one is refreshed and written back per build,
|
||||
// a heap one refreshed per build.
|
||||
// A staged region is keyed by its import like one bound in place: the import is the source
|
||||
// of its copy-in and the destination of its copy-back, both recorded per use.
|
||||
const bool fixedMirror = region.mirror != nullptr && !region.mirror->writable;
|
||||
const bool fixedMirror = region.mirror != nullptr && !region.mirror->writable && !region.mirror->heap;
|
||||
const bool staged = region.gpuCopy && region.deviceLocal;
|
||||
if (region.direct == nullptr && !fixedMirror && !staged) return std::nullopt;
|
||||
if (staged) {
|
||||
|
|
|
|||
|
|
@ -2,10 +2,19 @@
|
|||
#include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp"
|
||||
#include "prx/libc/include/GuestAllocations.hpp"
|
||||
#include "prx/libc/include/GuestHeap.hpp"
|
||||
#include "prx/libc/include/GuestArena.hpp"
|
||||
#ifdef _WIN32
|
||||
#ifndef NOMINMAX
|
||||
#define NOMINMAX
|
||||
#endif
|
||||
#include <windows.h>
|
||||
#endif
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <cstring>
|
||||
#include <functional>
|
||||
#include <limits>
|
||||
#include <string>
|
||||
|
||||
namespace {
|
||||
|
||||
|
|
@ -26,6 +35,205 @@ ShaderRecompiler::DescriptorBinding binding(Role role, std::uint32_t slot) {
|
|||
return {ShaderRecompiler::DescriptorKind::StorageBuffer, role, 0, slot, 1, {}, false};
|
||||
}
|
||||
|
||||
// Heap ranges that no import serves are mirrored. A read-only one: the build binds the mirror, the
|
||||
// next build with the range unchanged copies nothing into it, and the blocks a CPU write changed are
|
||||
// read again. A writable one: a descriptor's store reaches guest memory at write-back, and only the
|
||||
// bytes the GPU changed (a CPU store made while the GPU works stays).
|
||||
void heapMirrorTests(const Context& context, const BdaTestAccess& access) {
|
||||
// Heap mirrors refresh from the arena's write tracking; without it they are never made.
|
||||
Require(GuestArena::GuestArenaAvailable_nid_postfix() && GuestArena::GuestArenaWriteWatched_nid_postfix(), "the heap mirror tests need the write-watched guest arena");
|
||||
constexpr std::size_t bytes = 2 * 65536;
|
||||
void* block = GuestArena::GuestArenaAllocate_nid_postfix(bytes, 65536);
|
||||
#ifdef _WIN32
|
||||
Require(VirtualAlloc(block, bytes, MEM_COMMIT, PAGE_READWRITE) != nullptr, "cannot commit the heap mirror block");
|
||||
#endif
|
||||
auto* guest = static_cast<std::uint8_t*>(block);
|
||||
std::memset(block, 0x11, bytes);
|
||||
const auto address = reinterpret_cast<std::uintptr_t>(block);
|
||||
const auto registry = [&](bool add, bool writable) {
|
||||
auto* mutation = GuestAllocations::GuestAllocationsBegin_nid_postfix();
|
||||
if (add) GuestAllocations::GuestAllocationsAdd_nid_postfix(mutation, block, bytes, true, writable);
|
||||
else GuestAllocations::GuestAllocationsRemove_nid_postfix(mutation, block);
|
||||
GuestAllocations::GuestAllocationsEnd_nid_postfix(mutation);
|
||||
};
|
||||
// An address-based build that writes 32 bytes at `written` (nothing for 0), with `gpu` run
|
||||
// between its upload and its write-back; returns the device address serving the range.
|
||||
const auto build = [&](std::uint64_t written, const std::function<void(GuestBufferMemory&)>& gpu) {
|
||||
GuestBufferMemory leased(context);
|
||||
leased.AcquireRegistered();
|
||||
if (written != 0) leased.AddWritable(written, 32);
|
||||
leased.Upload(true);
|
||||
const auto ranges = leased.AddressRanges();
|
||||
const auto found = std::find_if(ranges.begin(), ranges.end(), [&](const auto& range) { return range.begin == address && range.end == address + bytes; });
|
||||
Require(found != ranges.end() && found->permissions == ShaderRecompiler::BdaAbi::Read, "the heap range is missing from the BDA table");
|
||||
const auto device = found->deviceAddress;
|
||||
if (gpu) gpu(leased);
|
||||
leased.WriteBack();
|
||||
return device;
|
||||
};
|
||||
// An address-based build sweeps the mirrors whose ranges are gone (an earlier test's too).
|
||||
const auto sweep = [&] {
|
||||
GuestBufferMemory leased(context);
|
||||
leased.AcquireRegistered();
|
||||
leased.Upload(true);
|
||||
leased.WriteBack();
|
||||
};
|
||||
sweep();
|
||||
registry(true, false);
|
||||
const auto before = MirrorCounters();
|
||||
const auto first = build(0, {});
|
||||
const auto made = MirrorCounters();
|
||||
Require(made.heapMirrors == before.heapMirrors + 1 && made.heapBytes == before.heapBytes + bytes, "a read-only heap range was not mirrored");
|
||||
Require(access.addressBytes(first)[65536 + 3] == std::byte{0x11}, "the heap mirror was not filled");
|
||||
Require(build(0, {}) == first && MirrorCounters().blocksCopied == made.blocksCopied && MirrorCounters().heapRefills == made.heapRefills, "an unchanged heap range was read into its mirror again");
|
||||
guest[65536 + 3] = 0x22;
|
||||
Require(build(0, {}) == first && MirrorCounters().heapRefills == made.heapRefills + 1 && MirrorCounters().blocksCopied == made.blocksCopied + 1, "a written heap block was not read again alone");
|
||||
Require(access.addressBytes(first)[65536 + 3] == std::byte{0x22}, "the heap mirror missed the CPU write");
|
||||
const auto one = MirrorCounters();
|
||||
guest[5] = 0x33;
|
||||
guest[65536 + 7] = 0x44;
|
||||
Require(build(0, {}) == first && MirrorCounters().blocksCopied == one.blocksCopied + 2, "two written heap blocks were not read again");
|
||||
Require(access.addressBytes(first)[5] == std::byte{0x33} && access.addressBytes(first)[65536 + 7] == std::byte{0x44}, "the heap mirror missed the CPU writes");
|
||||
registry(false, false);
|
||||
sweep();
|
||||
const auto swept = MirrorCounters();
|
||||
Require(swept.heapMirrors == before.heapMirrors && swept.heapBytes == before.heapBytes, "the heap mirror outlived its range");
|
||||
|
||||
registry(true, true);
|
||||
const auto device = build(address + 16, [&](GuestBufferMemory& leased) {
|
||||
std::uint32_t adjustment = 0;
|
||||
const auto view = leased.Descriptor(address + 16, 32, adjustment);
|
||||
access.bytes(view.buffer)[view.offset + adjustment] = std::byte{0x77};
|
||||
guest[16 + 8] = 0x55;
|
||||
});
|
||||
Require(guest[16] == 0x77 && guest[16 + 8] == 0x55, "a writable heap mirror's write-back lost the GPU's store or rolled back the CPU's");
|
||||
Require(build(0, {}) == device && access.addressBytes(device)[16] == std::byte{0x77} && access.addressBytes(device)[16 + 8] == std::byte{0x55}, "the writable heap mirror missed the stores");
|
||||
registry(false, true);
|
||||
sweep();
|
||||
Require(MirrorCounters().heapMirrors == before.heapMirrors, "the writable heap mirror outlived its range");
|
||||
|
||||
#ifdef _WIN32
|
||||
// A build that throws after queuing a mirror's changed blocks (another mirror's range became
|
||||
// inaccessible) must not mark those blocks current: the next build copies them.
|
||||
{
|
||||
constexpr std::size_t half = 2 * 65536;
|
||||
void* pair = GuestArena::GuestArenaAllocate_nid_postfix(2 * half, 65536);
|
||||
Require(VirtualAlloc(pair, 2 * half, MEM_COMMIT, PAGE_READWRITE) != nullptr, "cannot commit the heap mirror pair");
|
||||
std::memset(pair, 0x11, 2 * half);
|
||||
auto* first = static_cast<std::uint8_t*>(pair);
|
||||
auto* second = first + half;
|
||||
const auto registerPair = [&](bool add) {
|
||||
auto* mutation = GuestAllocations::GuestAllocationsBegin_nid_postfix();
|
||||
for (auto* range : {first, second}) {
|
||||
if (add) GuestAllocations::GuestAllocationsAdd_nid_postfix(mutation, range, half, true, false);
|
||||
else GuestAllocations::GuestAllocationsRemove_nid_postfix(mutation, range);
|
||||
}
|
||||
GuestAllocations::GuestAllocationsEnd_nid_postfix(mutation);
|
||||
};
|
||||
const auto firstDevice = [&] {
|
||||
GuestBufferMemory leased(context);
|
||||
leased.AcquireRegistered();
|
||||
leased.Upload(true);
|
||||
const auto ranges = leased.AddressRanges();
|
||||
const auto found = std::find_if(ranges.begin(), ranges.end(), [&](const auto& range) { return range.begin == reinterpret_cast<std::uintptr_t>(first); });
|
||||
Require(found != ranges.end(), "the first heap range is missing from the BDA table");
|
||||
const auto device = found->deviceAddress;
|
||||
leased.WriteBack();
|
||||
return device;
|
||||
};
|
||||
registerPair(true);
|
||||
const auto device = firstDevice();
|
||||
first[5] = 0x66;
|
||||
DWORD previous = 0;
|
||||
// A protection change outside the registry, as a heap decommit is: the page cache is told.
|
||||
Require(VirtualProtect(second, 65536, PAGE_NOACCESS, &previous) != 0, "cannot protect the second heap range");
|
||||
GuestAllocations::GuestAllocationsInvalidate_nid_postfix(reinterpret_cast<std::uintptr_t>(second), 65536);
|
||||
bool threw = false;
|
||||
try {
|
||||
GuestBufferMemory leased(context);
|
||||
leased.AcquireRegistered();
|
||||
} catch (const std::runtime_error&) {
|
||||
threw = true;
|
||||
}
|
||||
Require(VirtualProtect(second, 65536, PAGE_READWRITE, &previous) != 0, "cannot unprotect the second heap range");
|
||||
GuestAllocations::GuestAllocationsInvalidate_nid_postfix(reinterpret_cast<std::uintptr_t>(second), 65536);
|
||||
Require(threw, "a build over an inaccessible heap mirror range did not fail");
|
||||
Require(firstDevice() == device && access.addressBytes(device)[5] == std::byte{0x66}, "an interrupted build left a changed heap block marked current");
|
||||
registerPair(false);
|
||||
sweep();
|
||||
VirtualFree(pair, 2 * half, MEM_DECOMMIT);
|
||||
GuestArena::GuestArenaRelease_nid_postfix(pair, 2 * half);
|
||||
}
|
||||
#endif
|
||||
#ifdef _WIN32
|
||||
// A writable range with a read-only page (a direct memory page mapped at several addresses is
|
||||
// read-only in every view): the mirror holds the page's bytes, a store next to it is written
|
||||
// back, and a GPU change inside it fails the write-back instead of being dropped. A descriptor's
|
||||
// copy outside an address-based build holds the page's bytes too.
|
||||
{
|
||||
constexpr std::size_t size = 2 * 65536;
|
||||
void* raw = GuestArena::GuestArenaAllocate_nid_postfix(size, 65536);
|
||||
Require(VirtualAlloc(raw, size, MEM_COMMIT, PAGE_READWRITE) != nullptr, "cannot commit the aliased range");
|
||||
auto* bytes8 = static_cast<std::uint8_t*>(raw);
|
||||
std::memset(raw, 0x11, size);
|
||||
const auto base = reinterpret_cast<std::uintptr_t>(raw);
|
||||
const auto page = base + 65536;
|
||||
DWORD previous = 0;
|
||||
Require(VirtualProtect(reinterpret_cast<void*>(page), 4096, PAGE_READONLY, &previous) != 0, "cannot make the aliased page read-only");
|
||||
GuestAllocations::GuestAllocationsInvalidate_nid_postfix(page, 4096);
|
||||
const auto registerRange = [&](bool add) {
|
||||
auto* mutation = GuestAllocations::GuestAllocationsBegin_nid_postfix();
|
||||
if (add) GuestAllocations::GuestAllocationsAdd_nid_postfix(mutation, raw, size, true, true);
|
||||
else GuestAllocations::GuestAllocationsRemove_nid_postfix(mutation, raw);
|
||||
GuestAllocations::GuestAllocationsEnd_nid_postfix(mutation);
|
||||
};
|
||||
registerRange(true);
|
||||
const auto storeAt = [&](std::uint64_t at) {
|
||||
GuestBufferMemory leased(context);
|
||||
leased.AcquireRegistered();
|
||||
leased.AddWritable(page - 16, 32);
|
||||
leased.Upload(true);
|
||||
const auto ranges = leased.AddressRanges();
|
||||
const auto found = std::find_if(ranges.begin(), ranges.end(), [&](const auto& range) { return range.begin == base; });
|
||||
Require(found != ranges.end() && access.addressBytes(found->deviceAddress)[65536 + 8] == std::byte{0x11}, "a writable range with a read-only page was not mirrored with its bytes");
|
||||
std::uint32_t adjustment = 0;
|
||||
const auto view = leased.Descriptor(page - 16, 32, adjustment);
|
||||
access.bytes(view.buffer)[view.offset + adjustment + static_cast<std::size_t>(at - (page - 16))] = std::byte{0x77};
|
||||
leased.WriteBack();
|
||||
};
|
||||
const auto made = MirrorCounters().heapMirrors;
|
||||
storeAt(page - 8);
|
||||
Require(MirrorCounters().heapMirrors == made + 1 && bytes8[65536 - 8] == 0x77, "a store next to a read-only page was not written back");
|
||||
bool refused = false;
|
||||
try {
|
||||
storeAt(page + 4);
|
||||
} catch (const std::runtime_error& error) {
|
||||
refused = std::string(error.what()).find("aliased writes are not implemented") != std::string::npos;
|
||||
}
|
||||
Require(refused && bytes8[65536 + 4] == 0x11, "a GPU change of a read-only page was not refused");
|
||||
{
|
||||
GuestBufferMemory plain(context);
|
||||
plain.AddReadable(page, 16);
|
||||
plain.Upload(false);
|
||||
std::uint32_t adjustment = 0;
|
||||
const auto view = plain.Descriptor(page, 16, adjustment);
|
||||
Require(access.bytes(view.buffer)[view.offset + adjustment] == std::byte{0x11}, "a descriptor over a read-only page of a writable range read zeros");
|
||||
plain.WriteBack();
|
||||
}
|
||||
registerRange(false);
|
||||
sweep();
|
||||
Require(VirtualProtect(reinterpret_cast<void*>(page), 4096, PAGE_READWRITE, &previous) != 0, "cannot restore the aliased page");
|
||||
GuestAllocations::GuestAllocationsInvalidate_nid_postfix(page, 4096);
|
||||
VirtualFree(raw, size, MEM_DECOMMIT);
|
||||
GuestArena::GuestArenaRelease_nid_postfix(raw, size);
|
||||
}
|
||||
#endif
|
||||
#ifdef _WIN32
|
||||
VirtualFree(block, bytes, MEM_DECOMMIT);
|
||||
#endif
|
||||
GuestArena::GuestArenaRelease_nid_postfix(block, bytes);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
void RunBdaResourceTests(const Context& context, const BdaTestAccess& access) {
|
||||
|
|
@ -96,6 +304,11 @@ void RunBdaResourceTests(const Context& context, const BdaTestAccess& access) {
|
|||
Require(range.begin == snapshots[0].address && range.end == range.begin + source.size(), "64-bit guest address was truncated");
|
||||
const auto fault = access.bytes(access.descriptor(5).buffer);
|
||||
for (const auto byte : fault) Require(byte == std::byte{}, "fault buffer was not initialized");
|
||||
// A store into a read-only table range names the range and why stores cannot reach it.
|
||||
const ShaderRecompiler::BdaAbi::Fault denied{ShaderRecompiler::BdaAbi::FaultState::Ready, ShaderRecompiler::BdaAbi::FaultReason::Permission, snapshots[0].address + 4, 4, 0, 0x88, 0};
|
||||
std::memcpy(fault.data(), &denied, sizeof(denied));
|
||||
reject([&] { resources.WriteBack(); }, "read-only in the BDA table");
|
||||
std::memset(fault.data(), 0, fault.size());
|
||||
resources.WriteBack();
|
||||
}
|
||||
{
|
||||
|
|
@ -158,4 +371,9 @@ void RunBdaResourceTests(const Context& context, const BdaTestAccess& access) {
|
|||
const auto dropped = AddressSpaceCounters();
|
||||
if (dropped.enabled) Require(dropped.waiterDrops == after.waiterDrops + 1 && LeaseCounters().cacheDrops == dropped.waiterDrops, "the free did not drop the cached address space");
|
||||
}
|
||||
heapMirrorTests(context, access);
|
||||
// The copies left to an address-based build: nothing within the limit, the largest first past it.
|
||||
Require(AddressCopyOverflow({{0x1000, 0x3000, 0x2000, "uncommitted pages"}}, 0x2000).empty(), "copies within the limit were refused");
|
||||
const auto copies = AddressCopyOverflow({{0x1000, 0x2000, 0x1000, "not mirrored"}, {0x10000, 0x30000, 0x18000, "uncommitted pages"}}, 0x2000);
|
||||
Require(!copies.empty() && copies.find("0x10000+0x20000 (0.1 MiB committed, uncommitted pages)") < copies.find("0x1000+0x1000"), "the copy limit does not name the largest copy first");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,6 +8,8 @@
|
|||
struct BdaTestAccess {
|
||||
std::function<std::span<std::byte>(VkBuffer)> bytes;
|
||||
std::function<VkDescriptorBufferInfo(std::uint32_t)> descriptor;
|
||||
// The bytes of the buffer behind a device address, from that address on.
|
||||
std::function<std::span<std::byte>(VkDeviceAddress)> addressBytes;
|
||||
};
|
||||
|
||||
void RunBdaResourceTests(const AgcDriver::Graphics::Context& context, const BdaTestAccess& access);
|
||||
|
|
|
|||
|
|
@ -1273,6 +1273,8 @@ int main() {
|
|||
mock = MockVulkan{};
|
||||
auto bdaContext = mockContext();
|
||||
bdaContext.bufferDeviceAddress = true;
|
||||
// The fault buffer (with its written-page slots) is larger than the mock's 4 KiB default.
|
||||
bdaContext.limits.maxStorageBufferRange = 1u << 27;
|
||||
RunBdaResourceTests(bdaContext, {
|
||||
[](VkBuffer buffer) -> std::span<std::byte> { return mock.memories.at(mock.bufferMemory.at(buffer)); },
|
||||
[](std::uint32_t binding) {
|
||||
|
|
@ -1280,6 +1282,12 @@ int main() {
|
|||
if (it->binding == binding) return it->buffers.at(0);
|
||||
}
|
||||
throw std::runtime_error("missing BDA test descriptor");
|
||||
},
|
||||
[](VkDeviceAddress address) {
|
||||
// The inverse of mockGetBufferDeviceAddress.
|
||||
const auto offset = address - 0x100000000000ULL;
|
||||
const auto buffer = reinterpret_cast<VkBuffer>(offset / 0x10000);
|
||||
return std::span<std::byte>(mock.memories.at(mock.bufferMemory.at(buffer))).subspan(offset % 0x10000);
|
||||
}
|
||||
});
|
||||
Require(mock.live == 0, "BDA resources leaked Vulkan objects");
|
||||
|
|
|
|||
Loading…
Reference in New Issue