diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp index 2a74aebf1..e1b80ff50 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp @@ -35,6 +35,7 @@ struct HostImport { bool dmaBuf = false; bool joined = false; std::shared_ptr chunk; + mutable std::uint64_t lastUse = 0; }; enum class ImportWatch : std::uint8_t { Watch, Unwatch }; @@ -59,6 +60,10 @@ void SetImportWatch(const Context& context, ImportWatch watch); // the import's buffer. const HostImport* HostImportFor(const Context& context, std::uint64_t address, std::size_t bytes); void SetHostImportWindow(std::uint64_t bytes); +// Tests: the window of one large registered range (64 MiB) and the import budget (0: the configured one). +void SetHostImportRangeWindow(std::uint64_t bytes); +void SetHostImportBudget(std::uint64_t bytes); +std::uint64_t HostImportLiveBytes(); bool ImportMappedRanges(const Context& context, const GuestAllocations::Mapped& ranges, std::uint64_t generation, bool adoptDevice); // Whether an existing import covers [address, address + bytes), without reconciling the imports // with the registry or making one (HostImportFor may take a registry lease): a hint for choices @@ -379,6 +384,7 @@ private: // Import registry epoch when `direct` pointers were taken at acquire time; they are reused while // no import was destroyed since. std::uint64_t importsEpoch = 0; + std::uint64_t importFloor = 0; std::vector regions; // Whether `regions` is in ascending address order (true right after AcquireRegistered, whose // regions follow the registry's order), so AddSnapshot can search instead of scanning. diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp index 5813a3041..8f0be0a68 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp @@ -172,6 +172,8 @@ struct HostImports { VkDevice watchDevice = VK_NULL_HANDLE; bool unwatchImports = false; bool unwatchDmaBufImports = false; + std::uint64_t useCounter = 0; + std::uint64_t buildFloor = 0; }; HostImports& Imports() { @@ -582,6 +584,13 @@ std::shared_ptr leasedRangeOwner(const GuestAlloc return found != lease.end() && (*found)->address == base ? *found : nullptr; } +std::shared_ptr leasedRangeOwner(const GuestAllocations::Lease& lease, std::uint64_t begin, std::uint64_t end) { + const auto found = std::upper_bound(lease.begin(), lease.end(), begin, [](std::uint64_t value, const auto& range) { return value < range->address; }); + if (found == lease.begin()) return nullptr; + const auto& range = *std::prev(found); + return begin >= range->address && end <= range->address + range->bytes ? range : nullptr; +} + struct ImportRun { std::uint64_t base = 0; std::uint64_t bytes = 0; @@ -683,22 +692,131 @@ bool wouldRetireImport(const HostImports& state, const ImportRun& run) { } bool sameRange(const HostImport& entry, const GuestAllocations::Lease& lease) { + if (entry.ranges.size() == 1 && !entry.joined && entry.bytes < entry.ranges.front()->bytes) { + const auto current = leasedRangeOwner(lease, entry.base, entry.base + entry.bytes); + return current != nullptr && !entry.ranges.front().owner_before(current) && !current.owner_before(entry.ranges.front()); + } return sameRun(entry, entry.joined ? joinedRun(lease, entry.base, entry.base + entry.bytes) : importRun(lease, entry.base)); } +bool pointerHeld(const HostImports& state, const HostImport& entry) { + return entry.lastUse >= state.buildFloor; +} + +bool evictable(const HostImports& state, const HostImport& entry) { + constexpr std::uint64_t protectedUses = 256; + return !pointerHeld(state, entry) && entry.lastUse + protectedUses < state.useCounter; +} + +std::uint64_t retireLeastRecent(const Context& context, HostImports& state, const GuestAllocations::Lease& lease, std::uint64_t wanted) { + std::uint64_t freed = 0; + while (freed < wanted) { + auto oldest = state.imports.end(); + for (auto it = state.imports.begin(); it != state.imports.end(); ++it) { + if (evictable(state, it->second) && (oldest == state.imports.end() || it->second.lastUse < oldest->second.lastUse)) oldest = it; + } + if (oldest == state.imports.end()) break; + freed += oldest->second.bytes; + retireImport(context.device, state, oldest, lease); + } + return freed; +} + +std::uint64_t physicalMemoryBytes() { +#ifdef _WIN32 + MEMORYSTATUSEX status{}; + status.dwLength = sizeof(status); + return GlobalMemoryStatusEx(&status) ? status.ullTotalPhys : 0; +#else + const auto pages = sysconf(_SC_PHYS_PAGES); + const auto pageSize = sysconf(_SC_PAGESIZE); + return pages > 0 && pageSize > 0 ? static_cast(pages) * static_cast(pageSize) : 0; +#endif +} + +std::atomic budgetOverride{0}; + +// Pinned imports count against the driver's system memory heap, which imports and type-3 mirrors share; past it +// ordinary host allocations fail. The budget is APS5_HOST_IMPORT_MIB, else the larger of 6 GiB and half the physical +// memory, at most the largest host heap less 2 GiB. +std::uint64_t importBudget(const Context& context) { + if (const auto set = budgetOverride.load(std::memory_order_relaxed); set != 0) return set; + static const std::uint64_t configured = []() -> std::uint64_t { + const char* value = std::getenv("APS5_HOST_IMPORT_MIB"); + return value != nullptr ? std::strtoull(value, nullptr, 10) << 20u : 0u; + }(); + if (configured != 0) return configured; + std::uint64_t hostHeap = 0; + for (std::uint32_t index = 0; index < context.memory.memoryHeapCount; ++index) { + if ((context.memory.memoryHeaps[index].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) == 0) hostHeap = std::max(hostHeap, context.memory.memoryHeaps[index].size); + } + const auto fromMemory = std::max(6144ull << 20u, physicalMemoryBytes() / 2); + constexpr std::uint64_t reserve = 2048ull << 20u; + return hostHeap > reserve ? std::min(fromMemory, hostHeap - reserve) : fromMemory; +} + +std::atomic rangeImportWindow{64ull << 20u}; + +// The import of an access to a registered range larger than the range window: the cells the access touches, grown to +// the union of the range's imports it overlaps, so accesses across neighbouring cell edges share one import instead of +// replacing each other's. +ImportRun windowRun(const HostImports& state, ImportRun run, std::uint64_t begin, std::uint64_t end, std::uint64_t alignment) { + const auto window = rangeImportWindow.load(std::memory_order_relaxed); + if (run.ranges.size() != 1 || window == 0 || alignment == 0 || window % alignment != 0) return run; + const auto& range = *run.ranges.front(); + if (range.bytes <= window || begin < range.address || end > range.address + range.bytes || end <= begin) return run; + const auto lastCell = range.bytes / window - 1; + const auto cell = [&](std::uint64_t address) { return std::min((address - range.address) / window, lastCell); }; + auto windowBegin = range.address + cell(begin) * window; + const auto endCell = cell(end - 1); + auto windowEnd = endCell == lastCell ? range.address + range.bytes : range.address + (endCell + 1) * window; + for (bool grown = true; grown;) { + grown = false; + auto it = state.imports.upper_bound(windowBegin); + if (it != state.imports.begin()) --it; + for (; it != state.imports.end() && it->first < windowEnd; ++it) { + const auto importEnd = it->second.base + it->second.bytes; + if (importEnd <= windowBegin || it->second.base < range.address || importEnd > range.address + range.bytes) continue; + if (it->second.base < windowBegin || importEnd > windowEnd) grown = true; + windowBegin = std::min(windowBegin, it->second.base); + windowEnd = std::max(windowEnd, importEnd); + } + } + if (windowBegin % alignment != 0 || (windowEnd - windowBegin) % alignment != 0) return run; + run.base = windowBegin; + run.bytes = windowEnd - windowBegin; + return run; +} + const HostImport* findImport(HostImports& state, std::uint64_t begin, std::uint64_t end); bool importsStale(const Context& context, const HostImports& state); -const HostImport* importRunAt(const Context& context, HostImports& state, ImportRun run, bool joined, std::uint64_t address, const GuestAllocations::Lease& lease); +const HostImport* importRunAt(const Context& context, HostImports& state, ImportRun run, bool joined, std::uint64_t address, const GuestAllocations::Lease& lease, std::uint64_t end, bool evict); -const HostImport* importAllocation(const Context& context, HostImports& state, std::uint64_t address, const GuestAllocations::Lease& lease) { - if (!importsStale(context, state)) { - if (const auto* current = findImport(state, address, address + 1)) return current; - } else if (const auto* current = findImport(state, address, address + 1); current != nullptr && current->joined && sameRange(*current, lease)) { - return current; +// The bytes retireLeastRecent could free outside [begin, end). +std::uint64_t evictableBytes(const HostImports& state, std::uint64_t begin, std::uint64_t end) { + std::uint64_t bytes = 0; + for (const auto& [base, entry] : state.imports) { + if ((entry.base + entry.bytes <= begin || entry.base >= end) && evictable(state, entry)) bytes += entry.bytes; + } + return bytes; +} + +const HostImport* importAllocation(const Context& context, HostImports& state, std::uint64_t address, const GuestAllocations::Lease& lease, std::uint64_t end = 0, bool evict = true) { + const bool fresh = !importsStale(context, state); + if (fresh && end != 0) { + if (const auto* current = findImport(state, address, end)) return current; + } else if (!fresh) { + if (const auto* current = findImport(state, address, address + 1); current != nullptr && current->joined && sameRange(*current, lease)) return current; } auto run = importRun(lease, address); + // Without an end the caller views the whole run through the import (AcquireRegistered): a window that holds the + // first byte is shorter than that, so only an import of the whole run is a hit; a window is replaced below. + if (fresh && end == 0) { + if (const auto* current = findImport(state, address, run.ranges.empty() ? address + 1 : run.base + run.bytes)) return current; + } + if (end != 0) run = windowRun(state, std::move(run), address, end, context.hostImportAlignment); if (run.ranges.empty()) return nullptr; - return importRunAt(context, state, std::move(run), false, address, lease); + return importRunAt(context, state, std::move(run), false, address, lease, end, evict); } const HostImport* importJoined(const Context& context, HostImports& state, std::uint64_t begin, std::uint64_t end, const GuestAllocations::Lease& lease) { @@ -707,10 +825,10 @@ const HostImport* importJoined(const Context& context, HostImports& state, std:: auto run = joinedRun(lease, begin, end); if (run.ranges.empty()) return nullptr; if (const auto failed = state.failedJoins.find(run.base); failed != state.failedJoins.end() && sameRun(failed->second, run)) return nullptr; - return importRunAt(context, state, std::move(run), true, begin, lease); + return importRunAt(context, state, std::move(run), true, begin, lease, end, true); } -const HostImport* importRunAt(const Context& context, HostImports& state, ImportRun run, bool joined, std::uint64_t address, const GuestAllocations::Lease& lease) { +const HostImport* importRunAt(const Context& context, HostImports& state, ImportRun run, bool joined, std::uint64_t address, const GuestAllocations::Lease& lease, std::uint64_t end, bool evict) { const auto base = run.base; const auto bytes = run.bytes; auto& failures = joined ? state.failedJoins : state.failed; @@ -719,30 +837,28 @@ const HostImport* importRunAt(const Context& context, HostImports& state, Import failures.insert_or_assign(base, ImportFailure{base, bytes, GuestAllocations::Mapped(run.ranges.begin(), run.ranges.end())}); return nullptr; } - if (const auto found = state.imports.find(base); found != state.imports.end()) { - if (sameRun(found->second, run)) return &found->second; - retireImport(context.device, state, found, lease); - } - for (auto it = state.imports.lower_bound(base); it != state.imports.end() && it->first < base + bytes;) { - const auto next = std::next(it); - retireImport(context.device, state, it, lease); - it = next; - } - if (auto it = state.imports.lower_bound(base); it != state.imports.begin() && std::prev(it)->second.base + std::prev(it)->second.bytes > base) retireImport(context.device, state, std::prev(it), lease); + if (const auto found = state.imports.find(base); found != state.imports.end() && sameRun(found->second, run)) return &found->second; if (alignment == 0 || base % alignment != 0 || bytes % alignment != 0) return nullptr; if (const auto failed = failures.find(base); failed != failures.end()) { if (sameRun(failed->second, run)) return nullptr; failures.erase(failed); } - // Pinned imports count against the driver's system memory budget; past it ordinary host - // allocations fail, so imports stop at APS5_HOST_IMPORT_MIB (default 6 GiB, which covers the - // registered memory of address-based shaders; past it they copy gigabytes per dispatch). - static const std::uint64_t budget = [] { - const char* value = std::getenv("APS5_HOST_IMPORT_MIB"); - return (value ? std::strtoull(value, nullptr, 10) : 6144ull) << 20u; - }(); - const auto live = state.liveBytes; - if (live + bytes > budget) { + // The imports this one replaces overlap it; none may be held by a build, and the import must fit the budget (with + // what may be evicted) before any of them is retired. + auto first = state.imports.lower_bound(base); + if (first != state.imports.begin() && std::prev(first)->second.base + std::prev(first)->second.bytes > base) --first; + std::uint64_t replaced = 0; + // A join may replace imports this build already looked up: it runs before any region confirms an import + // (UploadFinish's pre-pass re-resolves every pointer after it), and a retired import stays alive through the + // hold (KeepThroughHold). Refusing it there would keep the run copied for good. + const auto held = [&](const HostImport& entry) { return !joined && pointerHeld(state, entry); }; + for (auto it = first; it != state.imports.end() && it->first < base + bytes; ++it) { + if (held(it->second)) return nullptr; + replaced += it->second.bytes; + } + const auto budget = importBudget(context); + const auto live = state.liveBytes - replaced; + if (live + bytes > budget && (!evict || live + bytes - evictableBytes(state, base, base + bytes) > budget)) { // A refused import turns every later use of the range into CPU copies, so say so. static std::uint64_t refused = 0, refusedBytes = 0; static auto lastReport = std::chrono::steady_clock::now() - std::chrono::seconds(60); @@ -750,10 +866,16 @@ const HostImport* importRunAt(const Context& context, HostImports& state, Import refusedBytes += bytes; if (std::chrono::steady_clock::now() - lastReport > std::chrono::seconds(10)) { lastReport = std::chrono::steady_clock::now(); - std::fprintf(stderr, "[gpu] host import budget: %llu MiB live of %llu MiB (APS5_HOST_IMPORT_MIB); %llu imports (%llu MiB) refused so far, last 0x%llx+0x%llx\n", static_cast(live >> 20u), static_cast(budget >> 20u), static_cast(refused), static_cast(refusedBytes >> 20u), static_cast(base), static_cast(bytes)); + std::fprintf(stderr, "[gpu] host import budget: %llu MiB live of %llu MiB (APS5_HOST_IMPORT_MIB); %llu imports (%llu MiB) refused so far, last 0x%llx+0x%llx\n", static_cast(state.liveBytes >> 20u), static_cast(budget >> 20u), static_cast(refused), static_cast(refusedBytes >> 20u), static_cast(base), static_cast(bytes)); } return nullptr; } + for (auto it = first; it != state.imports.end() && it->first < base + bytes;) { + const auto next = std::next(it); + retireImport(context.device, state, it, lease); + it = next; + } + if (state.liveBytes + bytes > budget) retireLeastRecent(context, state, lease, state.liveBytes + bytes - budget); HostImport entry{base, bytes, VK_NULL_HANDLE, VK_NULL_HANDLE, 0}; entry.range = run.ranges.front(); entry.ranges = std::move(run.ranges); @@ -795,7 +917,7 @@ const HostImport* importRunAt(const Context& context, HostImports& state, Import return nullptr; } state.unaliasedSpans.insert(base); - return importAllocation(context, state, address, lease); + return importAllocation(context, state, address, lease, end, evict); } } #else @@ -863,15 +985,10 @@ const HostImport* importRunAt(const Context& context, HostImports& state, Import state.liveBytes += bytes; if (trace) std::fprintf(stderr, "[gpu] host import of 0x%llx+0x%llx ok (%zu live, %.1f MiB live, %.1f MiB ever)%s\n", static_cast(base), static_cast(bytes), state.imports.size() + 1, state.liveBytes / 1048576.0, importedBytes / 1048576.0, joined ? ", joined at bind" : ""); entry.joined = joined; + entry.lastUse = ++state.useCounter; return &state.imports.emplace(base, entry).first->second; } -// The lease is in address order (it is built from the registry map), so a range is found by binary search. -const GuestAllocations::Range* leasedRangeAt(const GuestAllocations::Lease& lease, std::uint64_t base) { - const auto found = std::lower_bound(lease.begin(), lease.end(), base, [](const auto& range, std::uint64_t value) { return range->address < value; }); - return found != lease.end() && (*found)->address == base ? found->get() : nullptr; -} - // The readable registered range containing [begin, end), or null. const GuestAllocations::Range* containingRange(const GuestAllocations::Lease& lease, std::uint64_t begin, std::uint64_t end) { const auto found = std::upper_bound(lease.begin(), lease.end(), begin, [](std::uint64_t value, const auto& range) { return value < range->address; }); @@ -909,14 +1026,13 @@ void refreshImports(const Context& context, HostImports& state, const GuestAlloc if (generation == state.refreshedGeneration) return; state.refreshedGeneration = generation; for (auto it = state.imports.begin(); it != state.imports.end();) { - const auto* range = leasedRangeAt(lease, it->first); - if (range != nullptr && sameRange(it->second, lease)) { + if (sameRange(it->second, lease)) { if (it->second.unwatched) GuestMemory::Unwatch(it->first, it->second.bytes); ++it; continue; } static const bool trace = std::getenv("APS5_TRACE_HOST_IMPORT") != nullptr; - if (trace && range != nullptr && range->bytes == it->second.bytes) std::fprintf(stderr, "[gpu] host import of 0x%llx+0x%llx retired: the range was mapped again\n", static_cast(it->first), static_cast(it->second.bytes)); + if (trace && leasedRangeOwner(lease, it->first, it->first + it->second.bytes) != nullptr) std::fprintf(stderr, "[gpu] host import of 0x%llx+0x%llx retired: the range was mapped again\n", static_cast(it->first), static_cast(it->second.bytes)); const auto next = std::next(it); retireImport(context.device, state, it, lease); it = next; @@ -930,7 +1046,15 @@ const HostImport* findImport(HostImports& state, std::uint64_t begin, std::uint6 if (found == state.imports.begin()) return nullptr; --found; const auto& entry = found->second; - return begin >= entry.base && end <= entry.base + entry.bytes ? &entry : nullptr; + if (begin < entry.base || end > entry.base + entry.bytes) return nullptr; + entry.lastUse = ++state.useCounter; + return &entry; +} + +const HostImport* importAccess(const Context& context, HostImports& state, std::uint64_t begin, std::uint64_t end, const GuestAllocations::Lease& lease) { + if (!runCovers(lease, begin, end)) return findImport(state, begin, end); + const auto* entry = importAllocation(context, state, begin, lease, end); + return entry != nullptr && findImport(state, begin, end) == entry ? entry : nullptr; } // Whether the imports need reconciling before a lookup: the registry changed since the last walk. @@ -2063,6 +2187,20 @@ void SetHostImportWindow(std::uint64_t bytes) { importWindow.store(bytes, std::memory_order_relaxed); } +void SetHostImportRangeWindow(std::uint64_t bytes) { + rangeImportWindow.store(bytes, std::memory_order_relaxed); +} + +void SetHostImportBudget(std::uint64_t bytes) { + budgetOverride.store(bytes, std::memory_order_relaxed); +} + +std::uint64_t HostImportLiveBytes() { + auto& state = Imports(); + std::lock_guard lock(state.mutex); + return state.liveBytes; +} + const HostImport* HostImportFor(const Context& context, std::uint64_t address, std::size_t bytes) { if (context.hostImportAlignment == 0 || bytes == 0 || bytes > std::numeric_limits::max() - address) return nullptr; ensurePinWaiter(); @@ -2070,13 +2208,14 @@ const HostImport* HostImportFor(const Context& context, std::uint64_t address, s std::lock_guard lock(state.mutex); // A hit is only valid while the registry has not changed since the imports were reconciled. if (!importsStale(context, state)) { - if (const auto* entry = findImport(state, address, address + bytes)) return entry; + if (const auto* entry = findImport(state, address, address + bytes)) { + entry->lastUse = ++state.useCounter; + return entry; + } } const auto lease = GuestAllocations::GuestAllocationsAcquire_nid_postfix(); refreshImports(context, state, lease); - if (!runCovers(lease, address, address + bytes)) return findImport(state, address, address + bytes); - const auto* entry = importAllocation(context, state, address, lease); - return entry != nullptr && findImport(state, address, address + bytes) == entry ? entry : nullptr; + return importAccess(context, state, address, address + bytes, lease); } bool ImportMappedRanges(const Context& context, const GuestAllocations::Mapped& ranges, std::uint64_t generation, bool adoptDevice) { @@ -2099,7 +2238,7 @@ bool ImportMappedRanges(const Context& context, const GuestAllocations::Mapped& if (range == nullptr || leasedRangeOwner(current, range->address) != range) continue; try { if (!GuestMemory::GpuMutex().HeldByThisThread() && wouldRetireImport(state, importRun(current, range->address))) return false; - if (importAllocation(context, state, range->address, current) == nullptr && GuestAllocations::GuestAllocationsGeneration_nid_postfix() != generation) state.failed.erase(importRun(current, range->address).base); + if (importAllocation(context, state, range->address, current, 0, false) == nullptr && GuestAllocations::GuestAllocationsGeneration_nid_postfix() != generation) state.failed.erase(importRun(current, range->address).base); } catch (const std::exception& error) { std::fprintf(stderr, "[gpu] import of GPU memory mapped at 0x%llx+0x%zx failed: %s\n", static_cast(range->address), range->bytes, error.what()); } @@ -2199,6 +2338,13 @@ void GuestBufferMemory::AcquireRegistered() { else if (current->generation != generation) spaces.rebuiltGeneration.fetch_add(1, std::memory_order_relaxed); else if (current->importsEpoch != state.epoch) spaces.rebuiltEpoch.fetch_add(1, std::memory_order_relaxed); else hit = true; + importFloor = state.useCounter + 1; + if (hit) { + const auto stamp = ++state.useCounter; + for (const auto& region : current->base) { + if (region.direct != nullptr) region.direct->lastUse = stamp; + } + } } if (hit) { for (auto* mirror : current->sparseMirrors) { @@ -2237,6 +2383,8 @@ void GuestBufferMemory::AcquireRegistered() { auto& state = Imports(); std::lock_guard lock(state.mutex); if (importable) refreshImports(context, state, lease); + if (importFloor == 0) importFloor = state.useCounter + 1; + state.buildFloor = importFloor; // Taken before the pass: the imports it takes are the registry's ones (just reconciled, so // none is retired by a make below), and a mirror's preparation may sync the recorder, whose // completions can retire an import already taken; that moves the epoch, and Upload then @@ -2258,7 +2406,7 @@ void GuestBufferMemory::AcquireRegistered() { if (importable) { auto& state = Imports(); std::lock_guard lock(state.mutex); - entry = importAllocation(context, state, range->address, lease); + entry = importAllocation(context, state, range->address, lease, 0, false); } lap(timing.importsUs, at); if (entry != nullptr) { @@ -2907,6 +3055,7 @@ void GuestBufferMemory::UploadPrepare(bool addressable) { if (context.hostImportAlignment != 0) { auto& state = Imports(); std::lock_guard lock(state.mutex); + if (importFloor == 0) importFloor = state.useCounter + 1; const bool stale = importsStale(context, state); for (auto& region : regions) { if (region.mirror != nullptr) continue; @@ -2989,6 +3138,11 @@ void GuestBufferMemory::UploadFinish(bool addressable) { } uploaded = true; for (const auto& [begin, end] : writes) StorageTexture::FlushPending(begin, static_cast(end - begin), nullptr, "written buffer range"); + if (context.hostImportAlignment != 0) { + auto& state = Imports(); + std::lock_guard lock(state.mutex); + state.buildFloor = importFloor != 0 ? importFloor : state.useCounter + 1; + } // Registered ranges to import from. Address-based shaders hold their lease until write-back; // descriptor-only uploads acquire one only when the registry changed or an import must be made // (a lease copies every registered range's shared_ptr under the registry lock), so the guest can @@ -3086,9 +3240,7 @@ void GuestBufferMemory::UploadFinish(bool addressable) { region.direct = nullptr; if (entry == nullptr) entry = findImport(state, region.begin, region.end); if (entry == nullptr) { - if (runCovers(importRanges(), region.begin, region.end)) entry = importAllocation(context, state, region.begin, importRanges()); - if (entry != nullptr && region.end > entry->base + entry->bytes) entry = importJoined(context, state, region.begin, region.end, importRanges()); - if (entry != nullptr && (region.begin < entry->base || region.end > entry->base + entry->bytes)) entry = nullptr; + entry = importAccess(context, state, region.begin, region.end, importRanges()); } // A staged region (see stagingEligible) is copied out of the import even when aligned; // without a recorder to record the copies it binds in place like any other. diff --git a/core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp b/core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp index 85d560560..08fe07ffa 100644 --- a/core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp @@ -325,6 +325,155 @@ void importJoinTests(const Context& context) { ::operator delete(block, std::align_val_t{window}); } +// One registered range of `cells` import windows of `window` bytes, with the window and the import budget set for the +// test and reset after it. +class WindowedRange { +public: + WindowedRange(const Context& context, std::size_t window, std::size_t cells, std::uint64_t budget = 0) : window(window), bytes(window * cells), importing(context) { + block = ::operator new(bytes, std::align_val_t{piece}); + std::memset(block, 0x33, bytes); + importing.hostImportAlignment = piece; + SetHostImportRangeWindow(window); + SetHostImportBudget(budget); + change(true); + } + ~WindowedRange() { + change(false); + SetHostImportBudget(0); + SetHostImportRangeWindow(std::uint64_t{64} << 20u); + ::operator delete(block, std::align_val_t{piece}); + } + WindowedRange(const WindowedRange&) = delete; + WindowedRange& operator=(const WindowedRange&) = delete; + + std::uintptr_t First() const { return reinterpret_cast(block); } + + static constexpr std::size_t piece = 65536; + const std::size_t window; + const std::size_t bytes; + Context importing; + +private: + void change(bool add) { + auto* mutation = GuestAllocations::GuestAllocationsBegin_nid_postfix(); + if (add) GuestAllocations::GuestAllocationsAdd_nid_postfix(mutation, block, bytes, true, false, true); + else GuestAllocations::GuestAllocationsRemove_nid_postfix(mutation, block); + GuestAllocations::GuestAllocationsEnd_nid_postfix(mutation); + } + + void* block = nullptr; +}; + +// A registered range larger than the import window, its first window imported by an earlier access: an address-based +// build (AcquireRegistered) must import the whole range, so a view in its last window binds (Demon's Souls threw +// "guest buffer view ... exceeds its GPU owner" with the 64 MiB windows). +void importWholeRangeOverWindowTests(const Context& context) { + WindowedRange range(context, 2 * WindowedRange::piece, 2); + const auto first = range.First(); + const auto* windowed = HostImportFor(range.importing, first + 16, 32); + Require(windowed != nullptr && windowed->base == first && windowed->bytes == range.window, "an access to the first window of a large range was not imported as that window"); + GuestBufferMemory leased(range.importing); + leased.AcquireRegistered(); + leased.Upload(true); + std::uint32_t adjustment = 0; + const auto view = leased.Descriptor(first + range.bytes - WindowedRange::piece + 16, 32, adjustment); + Require(view.range == 32 + adjustment, "a view in the last window of a large registered range has no descriptor"); + const auto* whole = HostImportFor(range.importing, first, range.bytes); + Require(whole != nullptr && whole->base == first && whole->bytes == range.bytes && view.buffer == whole->buffer, "an address-based build was served by the range's first window, not an import of the whole range"); + leased.WriteBack(); +} + +// An address-based build whose whole-range import does not fit the budget leaves the range's window alone: before, +// every such build retired the window and the next access imported it again. +void importRefusedWholeRangeTests(const Context& context) { + WindowedRange range(context, 2 * WindowedRange::piece, 4, 3 * 2 * WindowedRange::piece); + const auto first = range.First(); + const auto* windowed = HostImportFor(range.importing, first + 16, 32); + Require(windowed != nullptr && windowed->bytes == range.window, "the first window of a range over the budget was not imported"); + const auto buffer = windowed->buffer; + for (int build = 0; build < 3; ++build) { + GuestBufferMemory leased(range.importing); + leased.AcquireRegistered(); + leased.Upload(true); + leased.WriteBack(); + Require(HostImportCovers(range.importing, first + 16, 32), "an address-based build refused its whole-range import and retired the range's window (build " + std::to_string(build) + ")"); + } + const auto* kept = HostImportFor(range.importing, first + 16, 32); + Require(kept != nullptr && kept->buffer == buffer, "the range's window was imported again after a refused whole-range import"); +} + +// Two uses across neighbouring cell edges of one range, each in a build of its own, share one import: the second grows +// the first's window to their union, and later rounds of both keep it. Before, each replaced the other's window at +// every use. +void importNeighbouringEdgesTests(const Context& context) { + WindowedRange range(context, 2 * WindowedRange::piece, 3); + const auto first = range.First(); + const auto use = [&](std::uintptr_t address) { + GuestBufferMemory leased(range.importing); + leased.AddReadable(address, 32); + leased.Upload(false); + leased.WriteBack(); + }; + const auto acrossFirst = first + range.window - 16; + const auto acrossSecond = first + 2 * range.window - 16; + use(acrossFirst); + Require(HostImportCovers(range.importing, acrossFirst, 32), "a use across the first cell edge was not imported"); + use(acrossSecond); + Require(HostImportCovers(range.importing, first, range.bytes), "a use across the second cell edge did not grow the first use's window to their union"); + const auto serial = HostImportSerial(range.importing, first, range.bytes, false); + for (int round = 0; round < 10; ++round) { + use(acrossFirst); + use(acrossSecond); + Require(HostImportSerial(range.importing, first, range.bytes, false) == serial, "a use across a cell edge replaced the shared window (round " + std::to_string(round) + ")"); + } +} + +// A range mapped while the imports are at the budget is refused at map time without retiring an import: the map +// observer holds no GPU lock, so it must not evict (before, it evicted the least recently used import to fit). +void importAtMapOverBudgetTests(const Context& context) { + constexpr std::size_t bytes = 65536; + auto importing = context; + importing.hostImportAlignment = bytes; + void* kept = ::operator new(bytes, std::align_val_t{bytes}); + void* used = ::operator new(bytes, std::align_val_t{bytes}); + void* mapped = ::operator new(bytes, std::align_val_t{bytes}); + const auto change = [&](void* block, bool add) { + auto* mutation = GuestAllocations::GuestAllocationsBegin_nid_postfix(); + if (add) GuestAllocations::GuestAllocationsAdd_nid_postfix(mutation, block, bytes, true, false, true); + else GuestAllocations::GuestAllocationsRemove_nid_postfix(mutation, block); + GuestAllocations::GuestAllocationsEnd_nid_postfix(mutation); + }; + change(kept, true); + change(used, true); + const auto keptAddress = reinterpret_cast(kept); + const auto usedAddress = reinterpret_cast(used); + Require(HostImportFor(importing, keptAddress, 16) != nullptr, "the import to keep was not made"); + for (int use = 0; use < 300; ++use) Require(HostImportFor(importing, usedAddress, 16) != nullptr, "the import in use was not made"); + { + GuestBufferMemory leased(importing); + leased.AddReadable(usedAddress, 16); + leased.Upload(false); + leased.WriteBack(); + } + const auto live = HostImportLiveBytes(); + SetHostImportBudget(live + bytes - 1); + change(mapped, true); + GuestAllocations::Mapped ranges; + { + const auto lease = GuestAllocations::GuestAllocationsAcquire_nid_postfix(); + for (const auto& range : lease) { + if (range->address == reinterpret_cast(mapped)) ranges.push_back(range); + } + } + Require(ranges.size() == 1, "the mapped range is not registered"); + static_cast(ImportMappedRanges(importing, ranges, GuestAllocations::GuestAllocationsGeneration_nid_postfix(), true)); + SetHostImportBudget(0); + Require(HostImportLiveBytes() == live, "a range mapped at the budget retired an import at map time (an eviction without the GPU lock)"); + Require(HostImportCovers(importing, keptAddress, 16) && !HostImportCovers(importing, reinterpret_cast(mapped), 16), "a range mapped at the budget was imported, or the import to keep was retired"); + for (auto* block : {kept, used, mapped}) change(block, false); + for (auto* block : {kept, used, mapped}) ::operator delete(block, std::align_val_t{bytes}); +} + ShaderRecompiler::DescriptorBinding binding(Role role, std::uint32_t slot) { return {ShaderRecompiler::DescriptorKind::StorageBuffer, role, 0, slot, 1, {}, false}; } @@ -1323,6 +1472,10 @@ void RunBdaResourceTests(const Context& context, const BdaTestAccess& access) { spaceImportRunTests(context); importWindowTests(context); importJoinTests(context); + importWholeRangeOverWindowTests(context); + importRefusedWholeRangeTests(context); + importNeighbouringEdgesTests(context); + importAtMapOverBudgetTests(context); heapMirrorTests(context, access); importedHeapMirrorTests(context, access); importWatchTests(); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/HostImportWindows.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/HostImportWindows.cpp new file mode 100644 index 000000000..57396daef --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/HostImportWindows.cpp @@ -0,0 +1,145 @@ +#include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "VulkanTestDevice.hpp" +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +namespace Graphics = AgcDriver::Graphics; +using Graphics::Require; + +constexpr std::uint64_t MiB = 1ull << 20u; +constexpr std::uint64_t WindowBytes = 64 * MiB; +constexpr std::uint64_t BlockBytes = 2 * WindowBytes; +constexpr std::uint64_t OtherBytes = 64 * 1024; +constexpr std::uint64_t GuestAlignment = 64 * 1024; +constexpr std::uint64_t ProbeBytes = 4096; + +std::string Hex(std::uint64_t value) { + char text[32]; + std::snprintf(text, sizeof(text), "0x%llx", static_cast(value)); + return text; +} + +class GuestBlock { +public: + explicit GuestBlock(std::uint64_t size) : bytes(size) { +#ifdef _WIN32 + block = static_cast(VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); +#else + block = static_cast(std::aligned_alloc(GuestAlignment, bytes)); +#endif + Require(block != nullptr, "host import windows: cannot allocate a guest block of " + Hex(bytes)); + std::memset(block, 0, bytes); + GuestAllocations::Mutation().Add(block, bytes, true, true, true); + } + + ~GuestBlock() { + GuestAllocations::Mutation().Remove(block); +#ifdef _WIN32 + VirtualFree(block, 0, MEM_RELEASE); +#else + std::free(block); +#endif + } + + GuestBlock(const GuestBlock&) = delete; + GuestBlock& operator=(const GuestBlock&) = delete; + + std::uint8_t* Data() const { return block; } + std::uint64_t Address() const { return static_cast(reinterpret_cast(block)); } + +private: + std::uint8_t* block = nullptr; + std::uint64_t bytes = 0; +}; + +enum class Outcome : std::uint8_t { NoDevice, NoImport, Passed }; + +Graphics::Context Probe(const AgcDriver::VulkanDevice& device) { + Graphics::Context probe{}; + probe.device = device.Device(); + probe.hostImportAlignment = 1; + return probe; +} + +bool Fill(AgcDriver::VulkanDevice& device, std::uint64_t address) { + const std::array pattern{0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u}; + std::lock_guard lock(AgcDriver::GuestMemory::GpuMutex()); + const bool filled = device.FillBuffer(address, ProbeBytes, pattern); + device.WaitIdle(); + return filled; +} + +Outcome ClearingDropsImports() { + GuestBlock guest(BlockBytes); + auto device = OpenVulkanTestDevice(); + if (!device) return Outcome::NoDevice; + if (!Fill(*device, guest.Address() + WindowBytes)) return Outcome::NoImport; + const auto probe = Probe(*device); + Require(Graphics::HostImportCovers(probe, guest.Address() + WindowBytes, ProbeBytes), "host import windows: the second window of " + Hex(guest.Address()) + " is not imported"); + Graphics::ClearHostImports(device->Device()); + Require(!Graphics::HostImportCovers(probe, guest.Address() + WindowBytes, ProbeBytes), "host import windows: ClearHostImports kept the window of " + Hex(guest.Address())); + return Outcome::Passed; +} + +Outcome WindowsFollowTheirRange() { + GuestBlock guest(BlockBytes); + auto device = OpenVulkanTestDevice(); + if (!device) return Outcome::NoDevice; + const auto probe = Probe(*device); + const auto address = guest.Address() + WindowBytes + WindowBytes / 2; + if (!Fill(*device, address)) return Outcome::NoImport; + Require(Graphics::HostImportCovers(probe, guest.Address() + WindowBytes, ProbeBytes), "host import windows: the second window of " + Hex(guest.Address()) + " is not imported"); + Require(!Graphics::HostImportCovers(probe, guest.Address(), ProbeBytes), "host import windows: the first window of " + Hex(guest.Address()) + " is imported along with the second"); + const auto first = Graphics::HostImportSerial(probe, address, ProbeBytes, false); + Require(first != 0, "host import windows: the window at " + Hex(address) + " has no serial"); + + GuestBlock other(OtherBytes); + Require(Fill(*device, address), "host import windows: the window at " + Hex(address) + " cannot be filled after an unrelated registration"); + Require(Graphics::HostImportSerial(probe, address, ProbeBytes, false) == first, "host import windows: the window at " + Hex(address) + " was dropped by an unrelated registration, although its range still owns it"); + + GuestAllocations::Mutation().Remove(guest.Data()); + GuestAllocations::Mutation().Add(guest.Data(), BlockBytes, true, true, true); + Require(Fill(*device, address), "host import windows: the window at " + Hex(address) + " cannot be filled after its range was mapped again"); + const auto second = Graphics::HostImportSerial(probe, address, ProbeBytes, false); + Require(second != 0 && second != first, "host import windows: the window of the range mapped again kept the serial " + Hex(first)); + return Outcome::Passed; +} + +} + +int main() { + try { + const auto counting = ClearingDropsImports(); + if (counting == Outcome::NoDevice) return VulkanTestSkipped; + if (counting == Outcome::NoImport) { + std::printf("skipped, the device does not import guest memory\n"); + return VulkanTestSkipped; + } + const auto windows = WindowsFollowTheirRange(); + Require(windows == Outcome::Passed, "host import windows: the device stopped importing guest memory between two scenarios"); + std::puts("host import windows tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/docs/dev/ARCHITECTURE.md b/docs/dev/ARCHITECTURE.md index 5227da96e..4eaebccf3 100644 --- a/docs/dev/ARCHITECTURE.md +++ b/docs/dev/ARCHITECTURE.md @@ -105,6 +105,7 @@ flowchart LR - A draw's `Graphics::Pipeline` comes from `CachedPipeline`. When the device has `VK_EXT_graphics_pipeline_library` with fast linking, `VK_EXT_extended_dynamic_state` and `VK_KHR_dynamic_rendering`, draws render with `vkCmdBeginRenderingKHR` instead of render passes and the pipeline is linked from four libraries (vertex input, pre-rasterization shaders, fragment shader, fragment output) kept per device in [`PipelineLibrary.cpp`](../../core/libs/prx/libSceAgcDriver/Graphics/src/PipelineLibrary.cpp). A library built for one pipeline is reused by the next: another blend, vertex layout or target format builds only that part (the shader libraries name no attachment format), and cull mode, front face, depth and stencil state are dynamic, so changing them only links. The fast link is what the draw waits for; a background thread then links the same libraries with link-time optimization and the pipeline binds that one from the next draw on. Push constants then use a 256-byte block with a 128-byte slot per side: the pre-rasterization stages stack from byte 0, the pixel shader always starts at byte 128 (`StagePushOffset`), so its library does not depend on how much push data its vertex shader uses. Pipelines with per-face depth bias, a separate alpha blend into an 8_8 target with the alternate component order (both draw twice), rect lists, geometry shaders or a stage without a variant id are created whole, still with dynamic rendering; devices without the extensions (or with `APS5_NO_GPL=1`) keep render passes and whole pipelines. - A draw whose `PA_SC_AA_CONFIG` selects 2, 4 or 8 samples renders into host-only multisampled images. [`MultisampleTarget.cpp`](../../core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleTarget.cpp) keeps one image per device, surface address, format, extent and sample count (`MultisampleTargetView`); the pipeline, render pass and library keys carry `State::samples`, and a multisampled depth target is a multisampled depth surface. The guest sample planes and FMASK are never read or written, so the registry also guards the target's guest range: a CPU read (`GuestMemory::SetReadGuard`), a DMA copy, and a sampled or storage image read of it throw. A `CB_COLOR_CONTROL` resolve mode over a multisampled CB0 runs as a render pass whose resolve attachment is CB1's resident image (`ResolveMultisampleTarget`). A zero fill of the target's CMASK (`NoteColorMetadataFill`, from fill kernels and CP DMA fills) clears the image at its next use. - A texture over a depth surface samples the depth image itself when it views one 2D slice of the plane's own format (`DepthSurfaceTexture` in [`DepthSurface.cpp`](../../core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp)). A view of several slices (a 2D array or a cube) or an integer view of a D16 plane is served by a `DepthPlaneCopy`, an image the slices are copied into at every use: from the depth surface bound at each slice's address, else from guest memory. Such a texture is never kept in the resource templates (`RefreshedPerUse`). +- Guest memory reaches the GPU through host imports (`VK_EXT_external_memory_host`) where the driver accepts them, and through copies otherwise ([`GuestBufferMemory.cpp`](../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp)). A registered range larger than the import window (64 MiB) is imported per window for buffer and fill accesses and whole for an address-based build. Live imports stay under a budget, with the least recently used import retired first (see TechnicalDebt, Host import windows and budget). ### Untracked storage image comparisons diff --git a/docs/dev/debt/libSceAgcDriver.md b/docs/dev/debt/libSceAgcDriver.md index cf1926f52..d14eb624f 100644 --- a/docs/dev/debt/libSceAgcDriver.md +++ b/docs/dev/debt/libSceAgcDriver.md @@ -114,3 +114,4 @@ - [Descriptor pool recycling](../../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) avoids individually freed sets: [RADV 26.2.2](https://gitlab.freedesktop.org/mesa/mesa/-/blob/mesa-26.2.2/src/amd/vulkan/radv_descriptor_pool.c) initializes its free-set heap with 32 bytes beyond the backing allocation. An allocation-only RX 6600 test reproduces acceptance of a 4,128-byte set in a 4,096-byte pool; the path without individual frees rejects it. Pools reset after their last set is released. A cached set can retain unused slots in an exhausted pool until eviction; peak gameplay pool memory remains unmeasured. - [16-bit depth with stencil](../../../core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp) (libSceAgcDriver) is stored as `D24_UNORM_S8_UINT`, or `D32_SFLOAT_S8_UINT`, on a device without `D16_UNORM_S8_UINT` as an attachment. Depth comparisons, clears and the depth image see the wider format's precision, so results at the 16-bit quantization boundary can differ from the console; the constant depth bias is scaled by 256 for D24, and a non-zero constant on the D32 float fallback throws. Not measured on console hardware. - [Flip packets raise the graphics end-of-pipe event](../../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp) (libSceAgcDriver) - a flip delivers the queue's end-of-pipe event (`sceAgcDriverAddEqEvent`) once the work recorded before it completes, as a `RELEASE_MEM` with `INT_SEL` does. Minecraft (PPSA17221) registers the graphics event, never requests an interrupt in its releases, and waits on the event after every flip while holding its render lock, so without this it never presents a second frame. Whether the console raises the event from the flip itself or from an interrupt its flip sequence contains is not measured; a title that waits only on release interrupts sees one extra event per flip, with `data` counting it. +- [Host import windows and budget](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp) (libSceAgcDriver): a registered range larger than 64 MiB is imported one 64 MiB window at a time for buffer and fill accesses; an access across window edges imports the cells it touches, grown to the union of the range's imports it overlaps. An address-based build (`AcquireRegistered`) imports the range whole and replaces its windows. Live imports stay under a budget: `APS5_HOST_IMPORT_MIB`, else the larger of 6 GiB and half the physical memory, at most the largest host heap less 2 GiB (imports and type-3 mirrors share it; #167's rule). A new import beyond it retires the least recently used imports not used in the last 256 lookups; one that would not fit even then is refused before anything is retired, and imports made when a range is mapped never evict. A refused import is tried again at its next use. The 64 MiB window, the 256-lookup protection and the 2 GiB reserve are choices, not measured.