From 2596f8c0ad50b0a62ab8efdc0284c85f76f43214 Mon Sep 17 00:00:00 2001 From: Dean Galvin Date: Fri, 2 Oct 2026 15:11:43 +1300 Subject: [PATCH 1/2] perf(agc): record occlusion counter dumps on the GPU Every ZPASS_DONE counter dump drained the device so the CPU could add up the occlusion queries of the finished batches and store the running total. PPSA21564 dumps around each occlusion-tested pass: about 1.5k drains per 10 s at its title screen, and in a level's early passes four drains a frame that took 5.6 s of every 10 s of the graphics worker. The running total now lives in a small buffer the GPU updates in batch order. A batch still ends its occlusion query at submit; a query that saw draws is remembered, and its pool is kept alive until it is folded. A dump ends the open batch's query, copies the remembered results next to the total, adds them with a one-thread compute shader that also stores the 16 DB counters into the dump's host import, and begins a fresh query for the rest of the batch. The dump is ordered with the work around it like any other GPU store, noted as a pending write and stamped. More than 64 remembered results are folded at a submit. Query pools are recycled. A dump into memory the GPU cannot see, under a pending copied write-back or over a label the CPU stores at a completion falls back to the drain, which settles the remembered results on the CPU. Without buffer device addresses the counting stays on the CPU. --- core/libs/prx/libSceAgcDriver/CMakeLists.txt | 15 ++ .../Execution/include/VulkanDevice.hpp | 1 + .../src/Driver/Packets/PacketExecution.cpp | 4 +- .../src/Driver/Packets/PacketMemory.cpp | 10 +- .../Execution/src/VulkanDevice.cpp | 19 ++ .../Graphics/include/Recorder.hpp | 21 +- .../Graphics/shaders/SampleCounter.comp | 29 +++ .../prx/libSceAgcDriver/Graphics/src/Draw.cpp | 1 + .../libSceAgcDriver/Graphics/src/Recorder.cpp | 211 +++++++++++++++++- .../prx/libSceAgcDriver/tests/Recorder.cpp | 63 ++++++ 10 files changed, 362 insertions(+), 12 deletions(-) create mode 100644 core/libs/prx/libSceAgcDriver/Graphics/shaders/SampleCounter.comp diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 7ee765a74..ee9f175b0 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -59,6 +59,19 @@ add_custom_command( VERBATIM ) +set(agcSampleCounterShaderSource ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/shaders/SampleCounter.comp) +set(agcSampleCounterShaderSpv ${agcTextureDetileGeneratedDir}/SampleCounter.spv) +set(agcSampleCounterShaderHeader ${agcTextureDetileGeneratedDir}/SampleCounter_spv.h) +add_custom_command( + OUTPUT "${agcSampleCounterShaderHeader}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${agcTextureDetileGeneratedDir}" + COMMAND "$" -V --target-env vulkan1.1 -o "${agcSampleCounterShaderSpv}" "${agcSampleCounterShaderSource}" + COMMAND ${CMAKE_COMMAND} -DINPUT=${agcSampleCounterShaderSpv} -DOUTPUT=${agcSampleCounterShaderHeader} + -DSYMBOL=SAMPLE_COUNTER_SPV -P "${CMAKE_CURRENT_SOURCE_DIR}/embed_spirv.cmake" + DEPENDS "${agcSampleCounterShaderSource}" glslang-standalone + VERBATIM +) + set(agcDriverSources ${CMAKE_CURRENT_SOURCE_DIR}/Unimplemented.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/DisplayBuffer.cpp @@ -144,6 +157,7 @@ set(agcDriverSources ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Sampler.cpp ${agcTextureDetileShaderHeader} ${agcMeshArgumentsShaderHeader} + ${agcSampleCounterShaderHeader} ${CMAKE_CURRENT_SOURCE_DIR}/Eq/src/Event.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Eq/src/Query.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Resource/src/Lifecycle.cpp @@ -403,6 +417,7 @@ add_test_executable(agc_driver_graphics_tests EXCLUDE_FROM_ALL Graphics/src/ShaderInputState.cpp ${agcTextureDetileShaderHeader} ${agcMeshArgumentsShaderHeader} + ${agcSampleCounterShaderHeader} Execution/src/GuestMemory.cpp Execution/src/Pm4.cpp ) diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp index 6ef93ea67..b2ae86195 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp @@ -82,6 +82,7 @@ public: // CPU (a copied buffer's write-back, a deferred label). Debug aid: APS5_FILL_SYNC=1 waits for // every recorded store over the range. bool FillBuffer(std::uint64_t address, std::size_t bytes, std::span pattern); + bool DumpSamplesOnGpu(std::uint64_t address); // Copies `bytes` of guest memory from `source` to `destination` (disjoint ranges) in place of the // engine's memcpy kernel (Driver.cpp copyBuffer). `path` 0: copied on the CPU at once, when // every test of the rule holds (each a pure query, nothing flushed or recorded before the diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp index dac8ff4ef..3ebdecd90 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp @@ -270,7 +270,7 @@ void Driver::execute(const Submission& submission) { } }); }); finishDrawPacket(drawn); - } else if (sampleDump) { + } else if (sampleDump && !wroteOnGpu) { dumpSampleCounters(packet[2] | (static_cast(packet[3]) << 32u)); } else if (opcode != 0x42 && opcode != 0x46 && opcode != 0x58) { if (!wroteOnGpu) { @@ -281,7 +281,7 @@ void Driver::execute(const Submission& submission) { } if (endOfPipeInterrupt && !interruptDeferred) AgcDriverDeliverEopInterrupt(submission.queue); } - if (drawPacket) Graphics::Recorder::CountRecordedWork(); + if (drawPacket || (sampleDump && wroteOnGpu)) Graphics::Recorder::CountRecordedWork(); cursor += count; } diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp index 1b7c0b9c0..f77bb313a 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp @@ -152,6 +152,14 @@ bool Driver::preparePacketMemory(const Submission& submission, QueueState& queue static const bool drawDrain = std::getenv("APS5_DRAW_DRAIN") != nullptr; drawPacket = Pm4::DrawOpcode(opcode); sampleDump = opcode == 0x46 && (packet[1] & 0x3fu) == 0x39u; + if (sampleDump && !drainAll) { + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Label); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + if (const auto localDevice = device.Load()) { + recordDeferredLabels(localDevice.get(), submission.queue); + wroteOnGpu = localDevice->DumpSamplesOnGpu(packet[2] | (static_cast(packet[3]) << 32u)); + } + } static const bool syncFlip = std::getenv("APS5_SYNC_FLIP") != nullptr; const bool drains = drainAll ? ((Pm4::AccessesMemory(header) && opcode != 0x16) || opcode == 0x42 || opcode == 0x46 || opcode == 0x58 || header == FlipPacketHeader) @@ -211,7 +219,7 @@ void Driver::dumpSampleCounters(std::uint64_t address) { auto* recorder = Graphics::Recorder::Active(); require(recorder != nullptr, "occlusion counters without the command recorder are not implemented"); recorder->CountSamples(); - samples = Graphics::Recorder::SamplesPassed(); + samples = recorder->SamplesTotal(); } constexpr std::uint64_t ready = 1ull << 63u; for (std::uint64_t db = 0; db < 16; ++db) { diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp index d5b74f184..783b80655 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp @@ -1175,6 +1175,25 @@ bool VulkanDevice::AfterRecordedWork(std::function action, bool reapFirs return state->recorder->AfterRecordedWork(std::move(action)); } +bool VulkanDevice::DumpSamplesOnGpu(std::uint64_t address) { + if (!state->recorder) return false; + auto& recorder = *state->recorder; + constexpr std::size_t bytes = 15 * 16 + 8; + if (OpportunisticReap()) recorder.Reap(); + if (state->CopiedWriterOverlaps(address, bytes) || (Graphics::Recorder::PendingCompletionLabels() != 0 && recorder.CompletionLabelIn(address, bytes))) return false; + const auto context = graphicsContext(); + Graphics::StorageTexture::FlushPending(address, bytes, nullptr, "occlusion counter dump", Graphics::PublishScope::PartialUnits); + const auto* import = Graphics::HostImportFor(context, address, bytes); + if (import == nullptr || import->address == 0) return false; + if (Graphics::AnyShadowedOverlaps(address, bytes)) Graphics::PublishShadow(address, bytes, Graphics::PublishScope::PartialUnits, Graphics::PublishReason::Label); + recorder.FlushStoresOverlapping(address, bytes); + recorder.FlushKeyStoresOverlapping(address, bytes); + if (!recorder.DumpSamples(import->address + (address - import->base))) return false; + recorder.NotePendingWrite(address, bytes); + GuestMemory::MarkWritten(address, bytes); + return true; +} + bool VulkanDevice::FillBuffer(std::uint64_t address, std::size_t bytes, std::span pattern) { if (!state->recorder || bytes == 0 || bytes % 16 != 0 || address % 16 != 0) return false; auto& recorder = *state->recorder; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp index 36d1b8529..6e3c1ddca 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp @@ -178,8 +178,10 @@ public: // Submits and waits for every batch, running completions in order. void Sync(); void CountSamples(); - static std::uint64_t SamplesPassed(); bool RecordMeshArguments(VkCommandBuffer commands, VkDeviceAddress record, VkDeviceAddress arguments, std::span rules); + std::uint64_t SamplesTotal(); + bool DumpSamples(VkDeviceAddress target); + void NoteSampledDraw(); // Waits only for the batches up to the newest one that writes the range (submitting the open // batch when it is that one); later batches stay in flight. Fences of one queue signal in // submission order, so completions still run in order. Debug aid: APS5_NO_SYNC_THROUGH=1 syncs all. @@ -276,6 +278,7 @@ public: // inside [address, address + bytes): a range query for large ranges (a fill of megabytes), // where a per-dword lookup would not do. bool PendingLabelIn(std::uint64_t address, std::size_t bytes) const; + bool CompletionLabelIn(std::uint64_t address, std::size_t bytes) const; // PendingLabel of the active recorder WITHOUT GuestMemory::GpuMutex: the table has a small mutex // of its own (every mutation holds both), so a WAIT_REG_MEM consults it without queueing behind // device work. Nothing is done under the table mutex but the lookup (it never takes the GPU mutex). @@ -505,6 +508,9 @@ private: std::chrono::steady_clock::time_point submittedAt{}; VkQueryPool queries = VK_NULL_HANDLE; VkQueryPool samples = VK_NULL_HANDLE; + std::shared_ptr samplePool; + bool sampleActive = false; + bool samplesDrawn = false; std::vector timedKeys; std::vector timedBytes; // The whole-batch timed range (BatchTimingKey) and its stamps once read (see Completed). @@ -603,10 +609,23 @@ private: void readGpuTiming(Batch& batch); void beginSamples(Batch& batch); void readSamples(Batch& batch); + bool gpuSampleCounter(); + void endSamples(Batch& batch); + void foldSamples(Batch& batch, VkDeviceAddress target); + struct SampleSegment { + std::shared_ptr pool; + VkQueryPool handle; + }; + std::vector pendingSamples; bool countingSamples = false; int meshArgumentState = 0; VkPipelineLayout meshArgumentLayout = VK_NULL_HANDLE; VkPipeline meshArgumentPipeline = VK_NULL_HANDLE; + int sampleCounterState = 0; + std::unique_ptr sampleCounter; + VkPipelineLayout sampleLayout = VK_NULL_HANDLE; + VkPipeline samplePipeline = VK_NULL_HANDLE; + std::shared_ptr samplePools; // BeginGpuTiming on the open batch without Commands() (RecordStore times its own run, which // Commands() would close). std::uint32_t beginTiming(std::uint64_t key); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/shaders/SampleCounter.comp b/core/libs/prx/libSceAgcDriver/Graphics/shaders/SampleCounter.comp new file mode 100644 index 000000000..66824ece2 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Graphics/shaders/SampleCounter.comp @@ -0,0 +1,29 @@ +#version 450 +#extension GL_EXT_buffer_reference : require + +layout(local_size_x = 1) in; +layout(buffer_reference, std430, buffer_reference_align = 8) buffer Words { uint words[]; }; +layout(push_constant) uniform Parameters { + Words counter; + Words target; + uint count; + uint store; +} parameters; + +void main() { + uint low = parameters.counter.words[0]; + uint high = parameters.counter.words[1]; + for (uint i = 0u; i < parameters.count; ++i) { + uint carry; + low = uaddCarry(low, parameters.counter.words[4u + 2u * i], carry); + high = high + parameters.counter.words[5u + 2u * i] + carry; + } + parameters.counter.words[0] = low; + parameters.counter.words[1] = high; + if (parameters.store != 0u) { + for (uint db = 0u; db < 16u; ++db) { + parameters.target.words[db * 4u] = db == 0u ? low : 0u; + parameters.target.words[db * 4u + 1u] = (db == 0u ? high : 0u) | 0x80000000u; + } + } +} diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp index a2f9d6047..cbad1d34d 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp @@ -962,6 +962,7 @@ struct IndirectRecord { // The draw commands of one draw: the vertex and index buffer binds, then the direct draw, the // GPU-side indirect draw from `argumentBuffer` or the CPU-read records with the driver's rules. void recordDrawCommands(const Context& context, VkCommandBuffer commands, const State& state, const Pm4::DrawParameters& draw, const DrawInputs& inputs, const IndirectRecord* indirect, VkBuffer argumentBuffer, VkDeviceSize argumentOffset) { + if (context.recorder != nullptr) context.recorder->NoteSampledDraw(); const auto* args = indirect != nullptr ? indirect->args : nullptr; if (state.stages.mesh && args != nullptr) { context.Function("vkCmdDrawMeshTasksIndirectEXT")(commands, argumentBuffer, argumentOffset, 1, sizeof(VkDrawMeshTasksIndirectCommandEXT)); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp index d5754cbe3..474ac0976 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp @@ -4,6 +4,7 @@ #include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" #include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" #include "prx/libSceAgcDriver/Graphics/shaders/MeshArguments_spv.h" +#include "prx/libSceAgcDriver/Graphics/shaders/SampleCounter_spv.h" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" #include "prx/libSceAgcDriver/Execution/include/Pm4Opcodes.hpp" @@ -170,6 +171,48 @@ std::mutex liveRecordersMutex; std::vector liveRecorders; std::atomic nextRecorderId{1}; std::atomic samplesPassed{0}; +constexpr std::uint32_t SampleFoldCapacity = 64; +constexpr std::uint32_t SampleCounterPushBytes = 24; + +struct SamplePoolCache { + VkDevice device; + PFN_vkDestroyQueryPool destroy; + std::mutex mutex; + std::vector free; + bool closed = false; + void put(VkQueryPool pool) { + { + std::lock_guard lock(mutex); + if (!closed && free.size() < 256) { + free.push_back(pool); + return; + } + } + destroy(device, pool, nullptr); + } + VkQueryPool take() { + std::lock_guard lock(mutex); + if (free.empty()) return VK_NULL_HANDLE; + const auto pool = free.back(); + free.pop_back(); + return pool; + } + void close() { + std::lock_guard lock(mutex); + closed = true; + for (const auto pool : free) destroy(device, pool, nullptr); + free.clear(); + } +}; + +struct SamplePool { + SamplePool(std::shared_ptr cache, VkQueryPool pool) : cache(std::move(cache)), pool(pool) {} + SamplePool(const SamplePool&) = delete; + SamplePool& operator=(const SamplePool&) = delete; + ~SamplePool() { cache->put(pool); } + std::shared_ptr cache; + VkQueryPool pool; +}; bool RecorderAlive(std::uint64_t id) { std::lock_guard lock(liveRecordersMutex); @@ -1097,6 +1140,13 @@ Recorder::~Recorder() { } if (meshArgumentPipeline != VK_NULL_HANDLE) context.Function("vkDestroyPipeline")(context.device, meshArgumentPipeline, nullptr); if (meshArgumentLayout != VK_NULL_HANDLE) context.Function("vkDestroyPipelineLayout")(context.device, meshArgumentLayout, nullptr); + if (sampleCounterState > 0) { + samplesPassed.store(SamplesTotal(), std::memory_order_release); + if (samplePools != nullptr) std::static_pointer_cast(samplePools)->close(); + context.Function("vkDestroyPipeline")(context.device, samplePipeline, nullptr); + context.Function("vkDestroyPipelineLayout")(context.device, sampleLayout, nullptr); + sampleCounter.reset(); + } for (auto& [commands, fence] : spare) { context.Function("vkFreeCommandBuffers")(context.device, context.pool, 1, &commands); context.Function("vkDestroyFence")(context.device, fence, nullptr); @@ -1810,14 +1860,124 @@ void Recorder::EndGpuTiming(std::uint32_t index, std::uint64_t bytes) { void Recorder::CountSamples() { GuestMemory::AssertGpuLockHeld("Recorder::CountSamples"); + gpuSampleCounter(); countingSamples = true; if (open == nullptr || open->samples != VK_NULL_HANDLE) return; if (open->renderPass.open) endOpenRenderPass(); beginSamples(*open); } -std::uint64_t Recorder::SamplesPassed() { - return samplesPassed.load(std::memory_order_acquire); +std::uint64_t Recorder::SamplesTotal() { + if (sampleCounterState <= 0) return samplesPassed.load(std::memory_order_acquire); + std::uint64_t total = 0; + std::memcpy(&total, sampleCounter->Bytes().data(), sizeof(total)); + for (const auto& segment : pendingSamples) { + std::uint64_t samples = 0; + Check(context.Function("vkGetQueryPoolResults")(context.device, segment.handle, 0, 1, sizeof(samples), &samples, sizeof(samples), VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WAIT_BIT), "vkGetQueryPoolResults occlusion"); + total += samples; + } + pendingSamples.clear(); + std::memcpy(sampleCounter->Bytes().data(), &total, sizeof(total)); + return total; +} + +void Recorder::NoteSampledDraw() { + if (open != nullptr && open->sampleActive) open->samplesDrawn = true; +} + +bool Recorder::gpuSampleCounter() { + if (sampleCounterState != 0) return sampleCounterState > 0; + sampleCounterState = -1; + if (!context.bufferDeviceAddress) return false; + VkShaderModule module = VK_NULL_HANDLE; + try { + sampleCounter = std::make_unique(context, 16 + 8 * SampleFoldCapacity, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT); + std::memset(sampleCounter->Bytes().data(), 0, sampleCounter->Bytes().size()); + const auto initial = samplesPassed.load(std::memory_order_acquire); + std::memcpy(sampleCounter->Bytes().data(), &initial, sizeof(initial)); + const VkPushConstantRange push{VK_SHADER_STAGE_COMPUTE_BIT, 0, SampleCounterPushBytes}; + VkPipelineLayoutCreateInfo layoutInfo{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO}; + layoutInfo.pushConstantRangeCount = 1; + layoutInfo.pPushConstantRanges = &push; + Check(context.Function("vkCreatePipelineLayout")(context.device, &layoutInfo, nullptr, &sampleLayout), "vkCreatePipelineLayout sample counter"); + VkShaderModuleCreateInfo moduleInfo{VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO}; + moduleInfo.codeSize = sizeof(SAMPLE_COUNTER_SPV); + moduleInfo.pCode = SAMPLE_COUNTER_SPV; + Check(context.Function("vkCreateShaderModule")(context.device, &moduleInfo, nullptr, &module), "vkCreateShaderModule sample counter"); + VkComputePipelineCreateInfo pipelineInfo{VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO}; + pipelineInfo.stage = {VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO}; + pipelineInfo.stage.stage = VK_SHADER_STAGE_COMPUTE_BIT; + pipelineInfo.stage.module = module; + pipelineInfo.stage.pName = "main"; + pipelineInfo.layout = sampleLayout; + Check(context.Function("vkCreateComputePipelines")(context.device, context.pipelineCache, 1, &pipelineInfo, nullptr, &samplePipeline), "vkCreateComputePipelines sample counter"); + context.Function("vkDestroyShaderModule")(context.device, module, nullptr); + } catch (const std::exception& error) { + if (module != VK_NULL_HANDLE) context.Function("vkDestroyShaderModule")(context.device, module, nullptr); + if (sampleLayout != VK_NULL_HANDLE) context.Function("vkDestroyPipelineLayout")(context.device, sampleLayout, nullptr); + sampleLayout = VK_NULL_HANDLE; + sampleCounter.reset(); + std::fprintf(stderr, "[gpu] occlusion counter dumps drain the device: %s\n", error.what()); + return false; + } + sampleCounterState = 1; + return true; +} + +void Recorder::endSamples(Batch& batch) { + if (!batch.sampleActive) return; + context.Function("vkCmdEndQuery")(batch.commands, batch.samples, 0); + batch.sampleActive = false; + if (sampleCounterState > 0 && batch.samplesDrawn) pendingSamples.push_back({batch.samplePool, batch.samples}); + batch.samplesDrawn = false; +} + +void Recorder::foldSamples(Batch& batch, VkDeviceAddress target) { + const auto commands = batch.commands; + const auto copy = context.Function("vkCmdCopyQueryPoolResults"); + const auto bind = context.Function("vkCmdBindPipeline"); + const auto push = context.Function("vkCmdPushConstants"); + const auto dispatch = context.Function("vkCmdDispatch"); + std::size_t done = 0; + do { + const auto count = static_cast(std::min(SampleFoldCapacity, pendingSamples.size() - done)); + if (done != 0) recordBarrier(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT); + for (std::uint32_t i = 0; i < count; ++i) { + const auto& segment = pendingSamples[done + i]; + copy(commands, segment.handle, 0, 1, sampleCounter->Handle(), 16 + 8 * i, 8, VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WAIT_BIT); + if (segment.pool != nullptr) batch.kept.push_back(segment.pool); + } + done += count; + const bool last = done == pendingSamples.size(); + recordBarrier(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); + bind(commands, VK_PIPELINE_BIND_POINT_COMPUTE, samplePipeline); + struct { + VkDeviceAddress counter; + VkDeviceAddress target; + std::uint32_t count; + std::uint32_t store; + } parameters{sampleCounter->DeviceAddress(), target, count, last && target != 0 ? 1u : 0u}; + static_assert(sizeof(parameters) == SampleCounterPushBytes); + push(commands, sampleLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, SampleCounterPushBytes, ¶meters); + dispatch(commands, 1, 1, 1); + } while (done < pendingSamples.size()); + recordBarrier(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_INDIRECT_COMMAND_READ_BIT | VK_ACCESS_UNIFORM_READ_BIT | VK_ACCESS_HOST_READ_BIT); + CountBarriers(CommandClass::Copy, 2); + pendingSamples.clear(); +} + +bool Recorder::DumpSamples(VkDeviceAddress target) { + GuestMemory::AssertGpuLockHeld("Recorder::DumpSamples"); + if (target == 0 || !gpuSampleCounter()) return false; + countingSamples = true; + Commands(); + endSamples(*open); + foldSamples(*open, target); + if (open->samplePool != nullptr) open->kept.push_back(std::move(open->samplePool)); + open->samplePool.reset(); + open->samples = VK_NULL_HANDLE; + beginSamples(*open); + return true; } bool Recorder::RecordMeshArguments(VkCommandBuffer commands, VkDeviceAddress record, VkDeviceAddress arguments, std::span rules) { @@ -1866,15 +2026,37 @@ bool Recorder::RecordMeshArguments(VkCommandBuffer commands, VkDeviceAddress rec } void Recorder::beginSamples(Batch& batch) { - VkQueryPoolCreateInfo info{VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO}; - info.queryType = VK_QUERY_TYPE_OCCLUSION; - info.queryCount = 1; - Check(context.Function("vkCreateQueryPool")(context.device, &info, nullptr, &batch.samples), "vkCreateQueryPool occlusion"); + auto cache = std::static_pointer_cast(samplePools); + if (sampleCounterState > 0 && cache == nullptr) { + cache = std::make_shared(); + cache->device = context.device; + cache->destroy = context.Function("vkDestroyQueryPool"); + samplePools = cache; + } + batch.samples = sampleCounterState > 0 ? cache->take() : VK_NULL_HANDLE; + if (batch.samples == VK_NULL_HANDLE) { + VkQueryPoolCreateInfo info{VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO}; + info.queryType = VK_QUERY_TYPE_OCCLUSION; + info.queryCount = 1; + Check(context.Function("vkCreateQueryPool")(context.device, &info, nullptr, &batch.samples), "vkCreateQueryPool occlusion"); + } + if (sampleCounterState > 0) { + try { + batch.samplePool = std::make_shared(cache, batch.samples); + } catch (...) { + cache->put(batch.samples); + batch.samples = VK_NULL_HANDLE; + throw; + } + } context.Function("vkCmdResetQueryPool")(batch.commands, batch.samples, 0, 1); context.Function("vkCmdBeginQuery")(batch.commands, batch.samples, 0, context.occlusionQueryPrecise ? VK_QUERY_CONTROL_PRECISE_BIT : 0u); + batch.sampleActive = true; + batch.samplesDrawn = false; } void Recorder::readSamples(Batch& batch) { + if (sampleCounterState > 0) return; if (batch.samples == VK_NULL_HANDLE || !batch.submitted) return; std::uint64_t samples = 0; Check(context.Function("vkGetQueryPoolResults")(context.device, batch.samples, 0, 1, sizeof(samples), &samples, sizeof(samples), VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WAIT_BIT), "vkGetQueryPoolResults occlusion"); @@ -2349,6 +2531,17 @@ bool Recorder::PendingLabelIn(std::uint64_t address, std::size_t bytes) const { return false; } +bool Recorder::CompletionLabelIn(std::uint64_t address, std::size_t bytes) const { + std::lock_guard tableLock(labelTableMutex); + if (bytes == 0) return false; + const auto end = address + bytes; + for (auto it = labels.lower_bound(address); it != labels.end() && it->first < end; ++it) { + if (it->second.behindCompletion) return true; + } + if (const auto first = queuedLabels.lower_bound(address); first != queuedLabels.end() && first->first < end) return true; + return false; +} + std::optional Recorder::lookupLabel(std::uint64_t address, std::size_t bytes, std::uint64_t afterStamp, LabelRefusal* refusal) const { if (refusal != nullptr) *refusal = LabelRefusal::None; if ((labels.empty() && queuedLabels.empty()) || (bytes != 4 && bytes != 8) || address % 4 != 0) return std::nullopt; @@ -2509,7 +2702,8 @@ void Recorder::Submit() { GuestMemory::AssertGpuLockHeld("Recorder::Submit"); if (!open->keyStores.empty()) recordKeyStores(false); if (open->renderPass.open) endOpenRenderPass(); - if (open->samples != VK_NULL_HANDLE) context.Function("vkCmdEndQuery")(open->commands, open->samples, 0); + endSamples(*open); + if (pendingSamples.size() >= SampleFoldCapacity) foldSamples(*open, 0); const bool hostReadCovered = (open->run.open || !open->run.queued.empty()) && closeStoreRun(true); if (BarrierValidate()) validateBatchEnds.fetch_add(1, std::memory_order_relaxed); if (open->hostReadOwed && !hostReadCovered) { @@ -3086,8 +3280,9 @@ void Recorder::release(Batch& batch) noexcept { if (batch.queries != VK_NULL_HANDLE) context.Function("vkDestroyQueryPool")(context.device, batch.queries, nullptr); } batch.queries = VK_NULL_HANDLE; - if (batch.samples != VK_NULL_HANDLE) context.Function("vkDestroyQueryPool")(context.device, batch.samples, nullptr); + if (batch.samples != VK_NULL_HANDLE && batch.samplePool == nullptr) context.Function("vkDestroyQueryPool")(context.device, batch.samples, nullptr); batch.samples = VK_NULL_HANDLE; + batch.samplePool.reset(); // A completed (or never submitted) batch's objects are kept for reuse: the fence is signaled or // untouched, so resetting it cannot block, and the command buffer is no longer pending. if (batch.commands != VK_NULL_HANDLE && batch.fence != VK_NULL_HANDLE && spare.size() < 64 && function(resetFences, "vkResetFences")(context.device, 1, &batch.fence) == VK_SUCCESS) { diff --git a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp index 25013c719..732e684a1 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp @@ -1885,6 +1885,68 @@ void pendingKeyStoreTests(const Device& device, Recorder& recorder) { recorder.Sync(); } +void sampleDumpTests(const Device& device, Recorder& recorder) { + const auto& context = device.GetContext(); + if (context.hostImportAlignment == 0 || !context.bufferDeviceAddress) { + std::cout << "host imports or buffer device addresses unavailable: occlusion counter dumps on the GPU not tested\n"; + return; + } + constexpr std::size_t bytes = 65536; +#ifdef _WIN32 + void* block = VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); +#else + void* block = std::aligned_alloc(65536, bytes); +#endif + Require(block != nullptr, "cannot allocate the occlusion counter block"); + auto* words = static_cast(block); + constexpr std::uint64_t untouched = 0xaaaaaaaaaaaaaaaaull; + std::fill(words, words + 64, untouched); + const auto address = reinterpret_cast(block); + { + GuestAllocations::Mutation mutation; + mutation.Add(block, bytes, true, true); + } + struct Unregister { + const Context& context; + void* block; + std::uint64_t address; + ~Unregister() { + { + GuestAllocations::Mutation mutation; + mutation.Remove(block); + } + HostImportFor(context, address, bytes); + } + } unregister{context, block, address}; + const auto* import = HostImportFor(context, address, bytes); + if (import == nullptr || import->address == 0) { + std::cout << "host import of the occlusion counter block refused: occlusion counter dumps on the GPU not tested\n"; + return; + } + const auto target = import->address + (address - import->base); + recorder.Sync(); + constexpr std::uint64_t ready = 1ull << 63u; + Require(recorder.DumpSamples(target), "the occlusion counters were not dumped on the GPU"); + Require(words[0] == untouched && !recorder.Idle(), "the occlusion counter dump waited for the GPU or landed before its batch ran"); + recorder.Sync(); + const auto begin = recorder.SamplesTotal(); + for (std::size_t db = 0; db < 16; ++db) { + Require(words[db * 2] == ((db == 0 ? begin : 0) | ready), "an occlusion counter dump stored the wrong value"); + Require(words[db * 2 + 1] == untouched, "an occlusion counter dump stored over the next counter"); + } + Require(recorder.DumpSamples(target + 8), "the second occlusion counter dump was not made on the GPU"); + recorder.Submit(); + recorder.Sync(); + for (std::size_t db = 0; db < 16; ++db) { + Require(words[db * 2 + 1] == ((db == 0 ? begin : 0) | ready), "the counters changed with nothing drawn between two dumps"); + Require(words[db * 2] == ((db == 0 ? begin : 0) | ready), "the second dump stored over the first"); + } + Require(recorder.SamplesTotal() == begin, "the sample total moved with nothing drawn"); + for (int i = 0; i < 40; ++i) Require(recorder.DumpSamples(target), "a dump past one batch's query slots was not made on the GPU"); + recorder.Sync(); + Require(words[0] == (begin | ready) && recorder.SamplesTotal() == begin, "dumps past one batch's query slots moved the total"); +} + void metadataPassTests(const Device& device, Recorder& recorder) { const auto& base = device.GetContext(); if (base.hostImportAlignment == 0) { @@ -2406,6 +2468,7 @@ int main() { firstLayerViewTests(device, recorder); metadataPassTests(device, recorder); pendingKeyStoreTests(device, recorder); + sampleDumpTests(device, recorder); std::cout << "Recorder read tracking and label tests passed\n"; return 0; } catch (const std::exception& error) { From 584c5bc590463adce01e4a134bcfeda85ce21c7e Mon Sep 17 00:00:00 2001 From: Dean Galvin Date: Sat, 3 Oct 2026 20:34:03 +1300 Subject: [PATCH 2/2] perf(agc): wait for a pending key fill before materializing a register DCC clear The first draw into a target whose DCC keys hold the register clear code materializes the clear. The GPU clear of the resident image refuses while a recorded write to the keys is still pending, so whenever the title's own key fill had not run yet the clear went the CPU way: the clear texels stored into guest memory (16 MiB for PPSA21564's normal G-buffer) and the image uploaded again. With the occlusion dumps no longer draining the device, the fill is usually still pending at that draw, and that was every frame of PPSA21564's level. The draw now waits for the recorded writes of the key range alone, checks the keys again, and clears on the GPU when they still hold the register code, as it did when the fill had already landed. --- core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp index cbad1d34d..3d9239d39 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp @@ -107,7 +107,17 @@ void materializeRegisterClear(const Context& context, const ColorTarget& color, if (CurrentDccKeys(color.dccAddress, color.bytes) != DccKeys::ClearRegister) return; const auto texel = clearTexel(color, DccKeys::ClearRegister); const char* refusal = nullptr; - if (clearToTexel(resident, texel, color.elementBytes, refusal)) { + bool cleared = clearToTexel(resident, texel, color.elementBytes, refusal); + const auto keyBytes = static_cast(color.bytes / 256); + if (!cleared && keyBytes != 0) { + if (auto* recorder = Recorder::Active(); recorder != nullptr && recorder->PendingWriteOverlaps(color.dccAddress, keyBytes)) { + Recorder::CountSync(2); + recorder->SyncThrough(color.dccAddress, keyBytes); + if (CurrentDccKeys(color.dccAddress, color.bytes) != DccKeys::ClearRegister) return; + cleared = clearToTexel(resident, texel, color.elementBytes, refusal); + } + } + if (cleared) { MarkDccUncompressed(context, color.dccAddress, color.bytes); return; }