diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 7ee765a74..ee9f175b0 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -59,6 +59,19 @@ add_custom_command( VERBATIM ) +set(agcSampleCounterShaderSource ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/shaders/SampleCounter.comp) +set(agcSampleCounterShaderSpv ${agcTextureDetileGeneratedDir}/SampleCounter.spv) +set(agcSampleCounterShaderHeader ${agcTextureDetileGeneratedDir}/SampleCounter_spv.h) +add_custom_command( + OUTPUT "${agcSampleCounterShaderHeader}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${agcTextureDetileGeneratedDir}" + COMMAND "$" -V --target-env vulkan1.1 -o "${agcSampleCounterShaderSpv}" "${agcSampleCounterShaderSource}" + COMMAND ${CMAKE_COMMAND} -DINPUT=${agcSampleCounterShaderSpv} -DOUTPUT=${agcSampleCounterShaderHeader} + -DSYMBOL=SAMPLE_COUNTER_SPV -P "${CMAKE_CURRENT_SOURCE_DIR}/embed_spirv.cmake" + DEPENDS "${agcSampleCounterShaderSource}" glslang-standalone + VERBATIM +) + set(agcDriverSources ${CMAKE_CURRENT_SOURCE_DIR}/Unimplemented.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/DisplayBuffer.cpp @@ -144,6 +157,7 @@ set(agcDriverSources ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Sampler.cpp ${agcTextureDetileShaderHeader} ${agcMeshArgumentsShaderHeader} + ${agcSampleCounterShaderHeader} ${CMAKE_CURRENT_SOURCE_DIR}/Eq/src/Event.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Eq/src/Query.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Resource/src/Lifecycle.cpp @@ -403,6 +417,7 @@ add_test_executable(agc_driver_graphics_tests EXCLUDE_FROM_ALL Graphics/src/ShaderInputState.cpp ${agcTextureDetileShaderHeader} ${agcMeshArgumentsShaderHeader} + ${agcSampleCounterShaderHeader} Execution/src/GuestMemory.cpp Execution/src/Pm4.cpp ) diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp index 6ef93ea67..b2ae86195 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp @@ -82,6 +82,7 @@ public: // CPU (a copied buffer's write-back, a deferred label). Debug aid: APS5_FILL_SYNC=1 waits for // every recorded store over the range. bool FillBuffer(std::uint64_t address, std::size_t bytes, std::span pattern); + bool DumpSamplesOnGpu(std::uint64_t address); // Copies `bytes` of guest memory from `source` to `destination` (disjoint ranges) in place of the // engine's memcpy kernel (Driver.cpp copyBuffer). `path` 0: copied on the CPU at once, when // every test of the rule holds (each a pure query, nothing flushed or recorded before the diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp index dac8ff4ef..3ebdecd90 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp @@ -270,7 +270,7 @@ void Driver::execute(const Submission& submission) { } }); }); finishDrawPacket(drawn); - } else if (sampleDump) { + } else if (sampleDump && !wroteOnGpu) { dumpSampleCounters(packet[2] | (static_cast(packet[3]) << 32u)); } else if (opcode != 0x42 && opcode != 0x46 && opcode != 0x58) { if (!wroteOnGpu) { @@ -281,7 +281,7 @@ void Driver::execute(const Submission& submission) { } if (endOfPipeInterrupt && !interruptDeferred) AgcDriverDeliverEopInterrupt(submission.queue); } - if (drawPacket) Graphics::Recorder::CountRecordedWork(); + if (drawPacket || (sampleDump && wroteOnGpu)) Graphics::Recorder::CountRecordedWork(); cursor += count; } diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp index 1b7c0b9c0..f77bb313a 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp @@ -152,6 +152,14 @@ bool Driver::preparePacketMemory(const Submission& submission, QueueState& queue static const bool drawDrain = std::getenv("APS5_DRAW_DRAIN") != nullptr; drawPacket = Pm4::DrawOpcode(opcode); sampleDump = opcode == 0x46 && (packet[1] & 0x3fu) == 0x39u; + if (sampleDump && !drainAll) { + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Label); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + if (const auto localDevice = device.Load()) { + recordDeferredLabels(localDevice.get(), submission.queue); + wroteOnGpu = localDevice->DumpSamplesOnGpu(packet[2] | (static_cast(packet[3]) << 32u)); + } + } static const bool syncFlip = std::getenv("APS5_SYNC_FLIP") != nullptr; const bool drains = drainAll ? ((Pm4::AccessesMemory(header) && opcode != 0x16) || opcode == 0x42 || opcode == 0x46 || opcode == 0x58 || header == FlipPacketHeader) @@ -211,7 +219,7 @@ void Driver::dumpSampleCounters(std::uint64_t address) { auto* recorder = Graphics::Recorder::Active(); require(recorder != nullptr, "occlusion counters without the command recorder are not implemented"); recorder->CountSamples(); - samples = Graphics::Recorder::SamplesPassed(); + samples = recorder->SamplesTotal(); } constexpr std::uint64_t ready = 1ull << 63u; for (std::uint64_t db = 0; db < 16; ++db) { diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp index d5b74f184..783b80655 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp @@ -1175,6 +1175,25 @@ bool VulkanDevice::AfterRecordedWork(std::function action, bool reapFirs return state->recorder->AfterRecordedWork(std::move(action)); } +bool VulkanDevice::DumpSamplesOnGpu(std::uint64_t address) { + if (!state->recorder) return false; + auto& recorder = *state->recorder; + constexpr std::size_t bytes = 15 * 16 + 8; + if (OpportunisticReap()) recorder.Reap(); + if (state->CopiedWriterOverlaps(address, bytes) || (Graphics::Recorder::PendingCompletionLabels() != 0 && recorder.CompletionLabelIn(address, bytes))) return false; + const auto context = graphicsContext(); + Graphics::StorageTexture::FlushPending(address, bytes, nullptr, "occlusion counter dump", Graphics::PublishScope::PartialUnits); + const auto* import = Graphics::HostImportFor(context, address, bytes); + if (import == nullptr || import->address == 0) return false; + if (Graphics::AnyShadowedOverlaps(address, bytes)) Graphics::PublishShadow(address, bytes, Graphics::PublishScope::PartialUnits, Graphics::PublishReason::Label); + recorder.FlushStoresOverlapping(address, bytes); + recorder.FlushKeyStoresOverlapping(address, bytes); + if (!recorder.DumpSamples(import->address + (address - import->base))) return false; + recorder.NotePendingWrite(address, bytes); + GuestMemory::MarkWritten(address, bytes); + return true; +} + bool VulkanDevice::FillBuffer(std::uint64_t address, std::size_t bytes, std::span pattern) { if (!state->recorder || bytes == 0 || bytes % 16 != 0 || address % 16 != 0) return false; auto& recorder = *state->recorder; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp index 36d1b8529..6e3c1ddca 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp @@ -178,8 +178,10 @@ public: // Submits and waits for every batch, running completions in order. void Sync(); void CountSamples(); - static std::uint64_t SamplesPassed(); bool RecordMeshArguments(VkCommandBuffer commands, VkDeviceAddress record, VkDeviceAddress arguments, std::span rules); + std::uint64_t SamplesTotal(); + bool DumpSamples(VkDeviceAddress target); + void NoteSampledDraw(); // Waits only for the batches up to the newest one that writes the range (submitting the open // batch when it is that one); later batches stay in flight. Fences of one queue signal in // submission order, so completions still run in order. Debug aid: APS5_NO_SYNC_THROUGH=1 syncs all. @@ -276,6 +278,7 @@ public: // inside [address, address + bytes): a range query for large ranges (a fill of megabytes), // where a per-dword lookup would not do. bool PendingLabelIn(std::uint64_t address, std::size_t bytes) const; + bool CompletionLabelIn(std::uint64_t address, std::size_t bytes) const; // PendingLabel of the active recorder WITHOUT GuestMemory::GpuMutex: the table has a small mutex // of its own (every mutation holds both), so a WAIT_REG_MEM consults it without queueing behind // device work. Nothing is done under the table mutex but the lookup (it never takes the GPU mutex). @@ -505,6 +508,9 @@ private: std::chrono::steady_clock::time_point submittedAt{}; VkQueryPool queries = VK_NULL_HANDLE; VkQueryPool samples = VK_NULL_HANDLE; + std::shared_ptr samplePool; + bool sampleActive = false; + bool samplesDrawn = false; std::vector timedKeys; std::vector timedBytes; // The whole-batch timed range (BatchTimingKey) and its stamps once read (see Completed). @@ -603,10 +609,23 @@ private: void readGpuTiming(Batch& batch); void beginSamples(Batch& batch); void readSamples(Batch& batch); + bool gpuSampleCounter(); + void endSamples(Batch& batch); + void foldSamples(Batch& batch, VkDeviceAddress target); + struct SampleSegment { + std::shared_ptr pool; + VkQueryPool handle; + }; + std::vector pendingSamples; bool countingSamples = false; int meshArgumentState = 0; VkPipelineLayout meshArgumentLayout = VK_NULL_HANDLE; VkPipeline meshArgumentPipeline = VK_NULL_HANDLE; + int sampleCounterState = 0; + std::unique_ptr sampleCounter; + VkPipelineLayout sampleLayout = VK_NULL_HANDLE; + VkPipeline samplePipeline = VK_NULL_HANDLE; + std::shared_ptr samplePools; // BeginGpuTiming on the open batch without Commands() (RecordStore times its own run, which // Commands() would close). std::uint32_t beginTiming(std::uint64_t key); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/shaders/SampleCounter.comp b/core/libs/prx/libSceAgcDriver/Graphics/shaders/SampleCounter.comp new file mode 100644 index 000000000..66824ece2 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Graphics/shaders/SampleCounter.comp @@ -0,0 +1,29 @@ +#version 450 +#extension GL_EXT_buffer_reference : require + +layout(local_size_x = 1) in; +layout(buffer_reference, std430, buffer_reference_align = 8) buffer Words { uint words[]; }; +layout(push_constant) uniform Parameters { + Words counter; + Words target; + uint count; + uint store; +} parameters; + +void main() { + uint low = parameters.counter.words[0]; + uint high = parameters.counter.words[1]; + for (uint i = 0u; i < parameters.count; ++i) { + uint carry; + low = uaddCarry(low, parameters.counter.words[4u + 2u * i], carry); + high = high + parameters.counter.words[5u + 2u * i] + carry; + } + parameters.counter.words[0] = low; + parameters.counter.words[1] = high; + if (parameters.store != 0u) { + for (uint db = 0u; db < 16u; ++db) { + parameters.target.words[db * 4u] = db == 0u ? low : 0u; + parameters.target.words[db * 4u + 1u] = (db == 0u ? high : 0u) | 0x80000000u; + } + } +} diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp index a2f9d6047..3d9239d39 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp @@ -107,7 +107,17 @@ void materializeRegisterClear(const Context& context, const ColorTarget& color, if (CurrentDccKeys(color.dccAddress, color.bytes) != DccKeys::ClearRegister) return; const auto texel = clearTexel(color, DccKeys::ClearRegister); const char* refusal = nullptr; - if (clearToTexel(resident, texel, color.elementBytes, refusal)) { + bool cleared = clearToTexel(resident, texel, color.elementBytes, refusal); + const auto keyBytes = static_cast(color.bytes / 256); + if (!cleared && keyBytes != 0) { + if (auto* recorder = Recorder::Active(); recorder != nullptr && recorder->PendingWriteOverlaps(color.dccAddress, keyBytes)) { + Recorder::CountSync(2); + recorder->SyncThrough(color.dccAddress, keyBytes); + if (CurrentDccKeys(color.dccAddress, color.bytes) != DccKeys::ClearRegister) return; + cleared = clearToTexel(resident, texel, color.elementBytes, refusal); + } + } + if (cleared) { MarkDccUncompressed(context, color.dccAddress, color.bytes); return; } @@ -962,6 +972,7 @@ struct IndirectRecord { // The draw commands of one draw: the vertex and index buffer binds, then the direct draw, the // GPU-side indirect draw from `argumentBuffer` or the CPU-read records with the driver's rules. void recordDrawCommands(const Context& context, VkCommandBuffer commands, const State& state, const Pm4::DrawParameters& draw, const DrawInputs& inputs, const IndirectRecord* indirect, VkBuffer argumentBuffer, VkDeviceSize argumentOffset) { + if (context.recorder != nullptr) context.recorder->NoteSampledDraw(); const auto* args = indirect != nullptr ? indirect->args : nullptr; if (state.stages.mesh && args != nullptr) { context.Function("vkCmdDrawMeshTasksIndirectEXT")(commands, argumentBuffer, argumentOffset, 1, sizeof(VkDrawMeshTasksIndirectCommandEXT)); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp index d5754cbe3..474ac0976 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp @@ -4,6 +4,7 @@ #include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" #include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" #include "prx/libSceAgcDriver/Graphics/shaders/MeshArguments_spv.h" +#include "prx/libSceAgcDriver/Graphics/shaders/SampleCounter_spv.h" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" #include "prx/libSceAgcDriver/Execution/include/Pm4Opcodes.hpp" @@ -170,6 +171,48 @@ std::mutex liveRecordersMutex; std::vector liveRecorders; std::atomic nextRecorderId{1}; std::atomic samplesPassed{0}; +constexpr std::uint32_t SampleFoldCapacity = 64; +constexpr std::uint32_t SampleCounterPushBytes = 24; + +struct SamplePoolCache { + VkDevice device; + PFN_vkDestroyQueryPool destroy; + std::mutex mutex; + std::vector free; + bool closed = false; + void put(VkQueryPool pool) { + { + std::lock_guard lock(mutex); + if (!closed && free.size() < 256) { + free.push_back(pool); + return; + } + } + destroy(device, pool, nullptr); + } + VkQueryPool take() { + std::lock_guard lock(mutex); + if (free.empty()) return VK_NULL_HANDLE; + const auto pool = free.back(); + free.pop_back(); + return pool; + } + void close() { + std::lock_guard lock(mutex); + closed = true; + for (const auto pool : free) destroy(device, pool, nullptr); + free.clear(); + } +}; + +struct SamplePool { + SamplePool(std::shared_ptr cache, VkQueryPool pool) : cache(std::move(cache)), pool(pool) {} + SamplePool(const SamplePool&) = delete; + SamplePool& operator=(const SamplePool&) = delete; + ~SamplePool() { cache->put(pool); } + std::shared_ptr cache; + VkQueryPool pool; +}; bool RecorderAlive(std::uint64_t id) { std::lock_guard lock(liveRecordersMutex); @@ -1097,6 +1140,13 @@ Recorder::~Recorder() { } if (meshArgumentPipeline != VK_NULL_HANDLE) context.Function("vkDestroyPipeline")(context.device, meshArgumentPipeline, nullptr); if (meshArgumentLayout != VK_NULL_HANDLE) context.Function("vkDestroyPipelineLayout")(context.device, meshArgumentLayout, nullptr); + if (sampleCounterState > 0) { + samplesPassed.store(SamplesTotal(), std::memory_order_release); + if (samplePools != nullptr) std::static_pointer_cast(samplePools)->close(); + context.Function("vkDestroyPipeline")(context.device, samplePipeline, nullptr); + context.Function("vkDestroyPipelineLayout")(context.device, sampleLayout, nullptr); + sampleCounter.reset(); + } for (auto& [commands, fence] : spare) { context.Function("vkFreeCommandBuffers")(context.device, context.pool, 1, &commands); context.Function("vkDestroyFence")(context.device, fence, nullptr); @@ -1810,14 +1860,124 @@ void Recorder::EndGpuTiming(std::uint32_t index, std::uint64_t bytes) { void Recorder::CountSamples() { GuestMemory::AssertGpuLockHeld("Recorder::CountSamples"); + gpuSampleCounter(); countingSamples = true; if (open == nullptr || open->samples != VK_NULL_HANDLE) return; if (open->renderPass.open) endOpenRenderPass(); beginSamples(*open); } -std::uint64_t Recorder::SamplesPassed() { - return samplesPassed.load(std::memory_order_acquire); +std::uint64_t Recorder::SamplesTotal() { + if (sampleCounterState <= 0) return samplesPassed.load(std::memory_order_acquire); + std::uint64_t total = 0; + std::memcpy(&total, sampleCounter->Bytes().data(), sizeof(total)); + for (const auto& segment : pendingSamples) { + std::uint64_t samples = 0; + Check(context.Function("vkGetQueryPoolResults")(context.device, segment.handle, 0, 1, sizeof(samples), &samples, sizeof(samples), VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WAIT_BIT), "vkGetQueryPoolResults occlusion"); + total += samples; + } + pendingSamples.clear(); + std::memcpy(sampleCounter->Bytes().data(), &total, sizeof(total)); + return total; +} + +void Recorder::NoteSampledDraw() { + if (open != nullptr && open->sampleActive) open->samplesDrawn = true; +} + +bool Recorder::gpuSampleCounter() { + if (sampleCounterState != 0) return sampleCounterState > 0; + sampleCounterState = -1; + if (!context.bufferDeviceAddress) return false; + VkShaderModule module = VK_NULL_HANDLE; + try { + sampleCounter = std::make_unique(context, 16 + 8 * SampleFoldCapacity, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT); + std::memset(sampleCounter->Bytes().data(), 0, sampleCounter->Bytes().size()); + const auto initial = samplesPassed.load(std::memory_order_acquire); + std::memcpy(sampleCounter->Bytes().data(), &initial, sizeof(initial)); + const VkPushConstantRange push{VK_SHADER_STAGE_COMPUTE_BIT, 0, SampleCounterPushBytes}; + VkPipelineLayoutCreateInfo layoutInfo{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO}; + layoutInfo.pushConstantRangeCount = 1; + layoutInfo.pPushConstantRanges = &push; + Check(context.Function("vkCreatePipelineLayout")(context.device, &layoutInfo, nullptr, &sampleLayout), "vkCreatePipelineLayout sample counter"); + VkShaderModuleCreateInfo moduleInfo{VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO}; + moduleInfo.codeSize = sizeof(SAMPLE_COUNTER_SPV); + moduleInfo.pCode = SAMPLE_COUNTER_SPV; + Check(context.Function("vkCreateShaderModule")(context.device, &moduleInfo, nullptr, &module), "vkCreateShaderModule sample counter"); + VkComputePipelineCreateInfo pipelineInfo{VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO}; + pipelineInfo.stage = {VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO}; + pipelineInfo.stage.stage = VK_SHADER_STAGE_COMPUTE_BIT; + pipelineInfo.stage.module = module; + pipelineInfo.stage.pName = "main"; + pipelineInfo.layout = sampleLayout; + Check(context.Function("vkCreateComputePipelines")(context.device, context.pipelineCache, 1, &pipelineInfo, nullptr, &samplePipeline), "vkCreateComputePipelines sample counter"); + context.Function("vkDestroyShaderModule")(context.device, module, nullptr); + } catch (const std::exception& error) { + if (module != VK_NULL_HANDLE) context.Function("vkDestroyShaderModule")(context.device, module, nullptr); + if (sampleLayout != VK_NULL_HANDLE) context.Function("vkDestroyPipelineLayout")(context.device, sampleLayout, nullptr); + sampleLayout = VK_NULL_HANDLE; + sampleCounter.reset(); + std::fprintf(stderr, "[gpu] occlusion counter dumps drain the device: %s\n", error.what()); + return false; + } + sampleCounterState = 1; + return true; +} + +void Recorder::endSamples(Batch& batch) { + if (!batch.sampleActive) return; + context.Function("vkCmdEndQuery")(batch.commands, batch.samples, 0); + batch.sampleActive = false; + if (sampleCounterState > 0 && batch.samplesDrawn) pendingSamples.push_back({batch.samplePool, batch.samples}); + batch.samplesDrawn = false; +} + +void Recorder::foldSamples(Batch& batch, VkDeviceAddress target) { + const auto commands = batch.commands; + const auto copy = context.Function("vkCmdCopyQueryPoolResults"); + const auto bind = context.Function("vkCmdBindPipeline"); + const auto push = context.Function("vkCmdPushConstants"); + const auto dispatch = context.Function("vkCmdDispatch"); + std::size_t done = 0; + do { + const auto count = static_cast(std::min(SampleFoldCapacity, pendingSamples.size() - done)); + if (done != 0) recordBarrier(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT); + for (std::uint32_t i = 0; i < count; ++i) { + const auto& segment = pendingSamples[done + i]; + copy(commands, segment.handle, 0, 1, sampleCounter->Handle(), 16 + 8 * i, 8, VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WAIT_BIT); + if (segment.pool != nullptr) batch.kept.push_back(segment.pool); + } + done += count; + const bool last = done == pendingSamples.size(); + recordBarrier(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); + bind(commands, VK_PIPELINE_BIND_POINT_COMPUTE, samplePipeline); + struct { + VkDeviceAddress counter; + VkDeviceAddress target; + std::uint32_t count; + std::uint32_t store; + } parameters{sampleCounter->DeviceAddress(), target, count, last && target != 0 ? 1u : 0u}; + static_assert(sizeof(parameters) == SampleCounterPushBytes); + push(commands, sampleLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, SampleCounterPushBytes, ¶meters); + dispatch(commands, 1, 1, 1); + } while (done < pendingSamples.size()); + recordBarrier(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_INDIRECT_COMMAND_READ_BIT | VK_ACCESS_UNIFORM_READ_BIT | VK_ACCESS_HOST_READ_BIT); + CountBarriers(CommandClass::Copy, 2); + pendingSamples.clear(); +} + +bool Recorder::DumpSamples(VkDeviceAddress target) { + GuestMemory::AssertGpuLockHeld("Recorder::DumpSamples"); + if (target == 0 || !gpuSampleCounter()) return false; + countingSamples = true; + Commands(); + endSamples(*open); + foldSamples(*open, target); + if (open->samplePool != nullptr) open->kept.push_back(std::move(open->samplePool)); + open->samplePool.reset(); + open->samples = VK_NULL_HANDLE; + beginSamples(*open); + return true; } bool Recorder::RecordMeshArguments(VkCommandBuffer commands, VkDeviceAddress record, VkDeviceAddress arguments, std::span rules) { @@ -1866,15 +2026,37 @@ bool Recorder::RecordMeshArguments(VkCommandBuffer commands, VkDeviceAddress rec } void Recorder::beginSamples(Batch& batch) { - VkQueryPoolCreateInfo info{VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO}; - info.queryType = VK_QUERY_TYPE_OCCLUSION; - info.queryCount = 1; - Check(context.Function("vkCreateQueryPool")(context.device, &info, nullptr, &batch.samples), "vkCreateQueryPool occlusion"); + auto cache = std::static_pointer_cast(samplePools); + if (sampleCounterState > 0 && cache == nullptr) { + cache = std::make_shared(); + cache->device = context.device; + cache->destroy = context.Function("vkDestroyQueryPool"); + samplePools = cache; + } + batch.samples = sampleCounterState > 0 ? cache->take() : VK_NULL_HANDLE; + if (batch.samples == VK_NULL_HANDLE) { + VkQueryPoolCreateInfo info{VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO}; + info.queryType = VK_QUERY_TYPE_OCCLUSION; + info.queryCount = 1; + Check(context.Function("vkCreateQueryPool")(context.device, &info, nullptr, &batch.samples), "vkCreateQueryPool occlusion"); + } + if (sampleCounterState > 0) { + try { + batch.samplePool = std::make_shared(cache, batch.samples); + } catch (...) { + cache->put(batch.samples); + batch.samples = VK_NULL_HANDLE; + throw; + } + } context.Function("vkCmdResetQueryPool")(batch.commands, batch.samples, 0, 1); context.Function("vkCmdBeginQuery")(batch.commands, batch.samples, 0, context.occlusionQueryPrecise ? VK_QUERY_CONTROL_PRECISE_BIT : 0u); + batch.sampleActive = true; + batch.samplesDrawn = false; } void Recorder::readSamples(Batch& batch) { + if (sampleCounterState > 0) return; if (batch.samples == VK_NULL_HANDLE || !batch.submitted) return; std::uint64_t samples = 0; Check(context.Function("vkGetQueryPoolResults")(context.device, batch.samples, 0, 1, sizeof(samples), &samples, sizeof(samples), VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WAIT_BIT), "vkGetQueryPoolResults occlusion"); @@ -2349,6 +2531,17 @@ bool Recorder::PendingLabelIn(std::uint64_t address, std::size_t bytes) const { return false; } +bool Recorder::CompletionLabelIn(std::uint64_t address, std::size_t bytes) const { + std::lock_guard tableLock(labelTableMutex); + if (bytes == 0) return false; + const auto end = address + bytes; + for (auto it = labels.lower_bound(address); it != labels.end() && it->first < end; ++it) { + if (it->second.behindCompletion) return true; + } + if (const auto first = queuedLabels.lower_bound(address); first != queuedLabels.end() && first->first < end) return true; + return false; +} + std::optional Recorder::lookupLabel(std::uint64_t address, std::size_t bytes, std::uint64_t afterStamp, LabelRefusal* refusal) const { if (refusal != nullptr) *refusal = LabelRefusal::None; if ((labels.empty() && queuedLabels.empty()) || (bytes != 4 && bytes != 8) || address % 4 != 0) return std::nullopt; @@ -2509,7 +2702,8 @@ void Recorder::Submit() { GuestMemory::AssertGpuLockHeld("Recorder::Submit"); if (!open->keyStores.empty()) recordKeyStores(false); if (open->renderPass.open) endOpenRenderPass(); - if (open->samples != VK_NULL_HANDLE) context.Function("vkCmdEndQuery")(open->commands, open->samples, 0); + endSamples(*open); + if (pendingSamples.size() >= SampleFoldCapacity) foldSamples(*open, 0); const bool hostReadCovered = (open->run.open || !open->run.queued.empty()) && closeStoreRun(true); if (BarrierValidate()) validateBatchEnds.fetch_add(1, std::memory_order_relaxed); if (open->hostReadOwed && !hostReadCovered) { @@ -3086,8 +3280,9 @@ void Recorder::release(Batch& batch) noexcept { if (batch.queries != VK_NULL_HANDLE) context.Function("vkDestroyQueryPool")(context.device, batch.queries, nullptr); } batch.queries = VK_NULL_HANDLE; - if (batch.samples != VK_NULL_HANDLE) context.Function("vkDestroyQueryPool")(context.device, batch.samples, nullptr); + if (batch.samples != VK_NULL_HANDLE && batch.samplePool == nullptr) context.Function("vkDestroyQueryPool")(context.device, batch.samples, nullptr); batch.samples = VK_NULL_HANDLE; + batch.samplePool.reset(); // A completed (or never submitted) batch's objects are kept for reuse: the fence is signaled or // untouched, so resetting it cannot block, and the command buffer is no longer pending. if (batch.commands != VK_NULL_HANDLE && batch.fence != VK_NULL_HANDLE && spare.size() < 64 && function(resetFences, "vkResetFences")(context.device, 1, &batch.fence) == VK_SUCCESS) { diff --git a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp index 25013c719..732e684a1 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp @@ -1885,6 +1885,68 @@ void pendingKeyStoreTests(const Device& device, Recorder& recorder) { recorder.Sync(); } +void sampleDumpTests(const Device& device, Recorder& recorder) { + const auto& context = device.GetContext(); + if (context.hostImportAlignment == 0 || !context.bufferDeviceAddress) { + std::cout << "host imports or buffer device addresses unavailable: occlusion counter dumps on the GPU not tested\n"; + return; + } + constexpr std::size_t bytes = 65536; +#ifdef _WIN32 + void* block = VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); +#else + void* block = std::aligned_alloc(65536, bytes); +#endif + Require(block != nullptr, "cannot allocate the occlusion counter block"); + auto* words = static_cast(block); + constexpr std::uint64_t untouched = 0xaaaaaaaaaaaaaaaaull; + std::fill(words, words + 64, untouched); + const auto address = reinterpret_cast(block); + { + GuestAllocations::Mutation mutation; + mutation.Add(block, bytes, true, true); + } + struct Unregister { + const Context& context; + void* block; + std::uint64_t address; + ~Unregister() { + { + GuestAllocations::Mutation mutation; + mutation.Remove(block); + } + HostImportFor(context, address, bytes); + } + } unregister{context, block, address}; + const auto* import = HostImportFor(context, address, bytes); + if (import == nullptr || import->address == 0) { + std::cout << "host import of the occlusion counter block refused: occlusion counter dumps on the GPU not tested\n"; + return; + } + const auto target = import->address + (address - import->base); + recorder.Sync(); + constexpr std::uint64_t ready = 1ull << 63u; + Require(recorder.DumpSamples(target), "the occlusion counters were not dumped on the GPU"); + Require(words[0] == untouched && !recorder.Idle(), "the occlusion counter dump waited for the GPU or landed before its batch ran"); + recorder.Sync(); + const auto begin = recorder.SamplesTotal(); + for (std::size_t db = 0; db < 16; ++db) { + Require(words[db * 2] == ((db == 0 ? begin : 0) | ready), "an occlusion counter dump stored the wrong value"); + Require(words[db * 2 + 1] == untouched, "an occlusion counter dump stored over the next counter"); + } + Require(recorder.DumpSamples(target + 8), "the second occlusion counter dump was not made on the GPU"); + recorder.Submit(); + recorder.Sync(); + for (std::size_t db = 0; db < 16; ++db) { + Require(words[db * 2 + 1] == ((db == 0 ? begin : 0) | ready), "the counters changed with nothing drawn between two dumps"); + Require(words[db * 2] == ((db == 0 ? begin : 0) | ready), "the second dump stored over the first"); + } + Require(recorder.SamplesTotal() == begin, "the sample total moved with nothing drawn"); + for (int i = 0; i < 40; ++i) Require(recorder.DumpSamples(target), "a dump past one batch's query slots was not made on the GPU"); + recorder.Sync(); + Require(words[0] == (begin | ready) && recorder.SamplesTotal() == begin, "dumps past one batch's query slots moved the total"); +} + void metadataPassTests(const Device& device, Recorder& recorder) { const auto& base = device.GetContext(); if (base.hostImportAlignment == 0) { @@ -2406,6 +2468,7 @@ int main() { firstLayerViewTests(device, recorder); metadataPassTests(device, recorder); pendingKeyStoreTests(device, recorder); + sampleDumpTests(device, recorder); std::cout << "Recorder read tracking and label tests passed\n"; return 0; } catch (const std::exception& error) {