From a3ffb7330951dafc115bbae6bf511e64f09da257 Mon Sep 17 00:00:00 2001 From: Dean Galvin Date: Sat, 3 Oct 2026 09:56:05 +1300 Subject: [PATCH] perf(agc): deliver end-of-pipe interrupts after their work without draining A RELEASE_MEM with an interrupt select drained the device: the queue worker submitted everything recorded and waited for the GPU before it stored the label and delivered the interrupt. PPSA21564 issues about 300 of them per frame, so the worker and the GPU took turns. The label is now recorded like any other (on the GPU behind the work, or as a completion action behind pending write-backs), and the delivery becomes a completion action of the batch holding the work before the packet (Recorder::AfterRecordedWork, after the batch's earlier completions). It is counted with the completion labels, so idle workers and pollers reap it, and it starts the label flush deadline. With draws or dispatches recorded since the last submit the batch is submitted at once. A batch holding only label stores stays open and goes out with the next work, at the label flush deadline, at a wait, at a flip or at the end of the submission, like any other pending label. A label the GPU cannot store, or one that does not decode, still drains. The interrupt so never arrives before the work ahead of it completed and its label landed; it can only arrive later, by at most the flush deadline plus the work recorded into the same batch. --- .../Execution/include/Driver/Driver.hpp | 2 +- .../Execution/include/VulkanDevice.hpp | 1 + .../src/Driver/Packets/PacketExecution.cpp | 6 +-- .../src/Driver/Packets/PacketMemory.cpp | 39 ++++++++++++++++++- .../Execution/src/VulkanDevice.cpp | 6 +++ .../Graphics/include/Recorder.hpp | 1 + .../libSceAgcDriver/Graphics/src/Recorder.cpp | 12 ++++++ .../prx/libSceAgcDriver/tests/Recorder.cpp | 23 +++++++++++ .../libs/prx/libSceAgcDriver/tests/Submit.cpp | 25 ++++++++++++ 9 files changed, 110 insertions(+), 5 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp index b945e50cb..895548081 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp @@ -207,7 +207,7 @@ private: void recordQueuedLabelsByTry(std::uint32_t queue, std::atomic& counter); static void recordQueuedLabelsFromHook(); void flushBetweenPackets(std::uint32_t queue, std::uint32_t header, bool labelPacket); - bool preparePacketMemory(const Submission& submission, QueueState& queue, std::span packet, std::uint32_t header, std::uint32_t opcode, bool& wroteOnGpu, bool& endOfPipeInterrupt, bool& drawPacket, bool& sampleDump); + bool preparePacketMemory(const Submission& submission, QueueState& queue, std::span packet, std::uint32_t header, std::uint32_t opcode, bool& wroteOnGpu, bool& endOfPipeInterrupt, bool& interruptDeferred, bool& drawPacket, bool& sampleDump); void dumpSampleCounters(std::uint64_t address); template static void timed(double WorkerProfile::*bucket, TWork&& work); diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp index 02cd2611c..ba0a0a204 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp @@ -71,6 +71,7 @@ public: // `reapFirst` retires finished batches before the checks; a caller recording a group of labels // under one lock passes it for the first label only. int WriteLabelOnGpu(std::uint64_t address, std::span bytes, std::uint64_t stamp, std::uint32_t queue, bool reapFirst = true); + bool AfterRecordedWork(std::function action, bool reapFirst); // Pending-label table lookup and open-batch overlap test for WAIT_REG_MEM (see Recorder). std::optional PendingLabel(std::uint64_t address, std::size_t bytes, std::uint64_t afterStamp, Graphics::Recorder::LabelRefusal* refusal = nullptr) const; bool OpenWriteOverlaps(std::uint64_t address, std::size_t bytes) const; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp index 62e593360..dac8ff4ef 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp @@ -152,8 +152,8 @@ void Driver::execute(const Submission& submission) { }; if (profilePackets) packetProfile.flushMs += std::chrono::duration(packetTimer.start - flushStart).count(); - bool wroteOnGpu = false, endOfPipeInterrupt = false, drawPacket = false, sampleDump = false; - const bool drains = preparePacketMemory(submission, queue, packet, header, opcode, wroteOnGpu, endOfPipeInterrupt, drawPacket, sampleDump); + bool wroteOnGpu = false, endOfPipeInterrupt = false, interruptDeferred = false, drawPacket = false, sampleDump = false; + const bool drains = preparePacketMemory(submission, queue, packet, header, opcode, wroteOnGpu, endOfPipeInterrupt, interruptDeferred, drawPacket, sampleDump); traceLabel(packet, submission.queue); const bool waitPacket = opcode == 0x3c || opcode == 0x93 || header == RenderingWaitPacketHeader; @@ -279,7 +279,7 @@ void Driver::execute(const Submission& submission) { if (const auto label = Pm4::DecodeLabelWrite(packet)) noteLabelStore(label->address, label->Bytes(), ++eventSerial); } } - if (endOfPipeInterrupt) AgcDriverDeliverEopInterrupt(submission.queue); + if (endOfPipeInterrupt && !interruptDeferred) AgcDriverDeliverEopInterrupt(submission.queue); } if (drawPacket) Graphics::Recorder::CountRecordedWork(); cursor += count; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp index 4d514defe..e5301b851 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp @@ -4,12 +4,13 @@ #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/DeferredLabels.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libSceAgcDriver/Execution/include/Pm4.hpp" +#include "prx/libSceAgcDriver/Eq/include/Event.hpp" #include #include namespace AgcDriver::DriverDetail { -bool Driver::preparePacketMemory(const Submission& submission, QueueState& queue, std::span packet, std::uint32_t header, std::uint32_t opcode, bool& wroteOnGpu, bool& endOfPipeInterrupt, bool& drawPacket, bool& sampleDump) { +bool Driver::preparePacketMemory(const Submission& submission, QueueState& queue, std::span packet, std::uint32_t header, std::uint32_t opcode, bool& wroteOnGpu, bool& endOfPipeInterrupt, bool& interruptDeferred, bool& drawPacket, bool& sampleDump) { static const bool drainAll = std::getenv("APS5_DRAIN_ALL") != nullptr; static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; wroteOnGpu = false; @@ -17,6 +18,42 @@ bool Driver::preparePacketMemory(const Submission& submission, QueueState& queue bool orderedAlready = false; endOfPipeInterrupt = opcode == 0x49 && ((packet[2] >> 24u) & 7u) != 0; + interruptDeferred = false; + if (!drainAll && endOfPipeInterrupt) { + const auto label = Pm4::DecodeLabelWrite(packet); + const bool storesNothing = (packet[2] >> 29u) == 0 || (packet[3] | (static_cast(packet[4]) << 32u)) == 0; + if (label.has_value() || storesNothing) { + bumpEpoch(&EpochBumps::drains); + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Label); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + const auto localDevice = device.Load(); + recordDeferredLabels(localDevice.get(), submission.queue); + const bool workOpen = Graphics::Recorder::RecordedWorkSinceSubmit() != 0; + int reason = localDevice != nullptr ? 0 : 1; + if (label.has_value()) { + const auto bytes = label->Bytes(); + const auto stamp = ++eventSerial; + reason = localDevice != nullptr ? localDevice->WriteLabelOnGpu(label->address, bytes, stamp, submission.queue) : 4; + if (reason == 1) GuestMemory::Write(label->address, bytes, 4); + if (reason == 0 || reason == 1 || reason == 5 || reason == 6) { + noteLabelStore(label->address, bytes, stamp); + Graphics::Recorder::CloseLabelGroup(GuestMemory::TrackerGeneration()); + } + countLabelOutcome(reason); + ++immediateLabels; + } else { + ++noOpLabels; + } + if (reason == 0 || reason == 5 || reason == 6) { + const auto queueId = submission.queue; + interruptDeferred = localDevice->AfterRecordedWork([queueId] { AgcDriverDeliverEopInterrupt(queueId); }, submission.queue == 0); + if (interruptDeferred && workOpen) localDevice->SubmitRecorded(submission.queue == 0); + wroteOnGpu = true; + } else if (reason == 1) { + wroteOnGpu = true; + } + } + } if (!drainAll && !endOfPipeInterrupt && (opcode == 0x49 || opcode == 0x37)) { if (const auto label = Pm4::DecodeLabelWrite(packet)) { const auto bytes = label->Bytes(); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp index 2b859eb91..ce218b07d 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp @@ -1127,6 +1127,12 @@ int VulkanDevice::WriteLabelOnGpu(std::uint64_t address, std::span action, bool reapFirst) { + if (!state->recorder) return false; + if (reapFirst && OpportunisticReap()) state->recorder->Reap(); + return state->recorder->AfterRecordedWork(std::move(action)); +} + bool VulkanDevice::FillBuffer(std::uint64_t address, std::size_t bytes, std::span pattern) { if (!state->recorder || bytes == 0 || bytes % 16 != 0 || address % 16 != 0) return false; auto& recorder = *state->recorder; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp index 816f8844a..532d2f7e8 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp @@ -316,6 +316,7 @@ public: // completion store would land on memory the game may have reused by then (a stale label value // over a fresh command buffer). Debug aid: APS5_LABEL_STORE_ALWAYS=1 stores unconditionally. void AfterCompletions(std::uint64_t address, std::span bytes, std::uint64_t stamp, std::uint32_t queue, bool storedOnGpu); + bool AfterRecordedWork(std::function action); // A CPU store of GPU results into guest memory (GuestBufferMemory::WriteBack): recorded so a // completion label store can tell whether its bytes were overwritten. Under GuestMemory:: // GpuMutex (every write-back runs inside finish() or a synchronous draw's wait): the first diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp index 0c25fe4f6..ee7fb97b1 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp @@ -2359,6 +2359,18 @@ void Recorder::AfterCompletions(std::uint64_t address, std::span action) { + if (Idle()) return false; + Batch& batch = open != nullptr ? *open : *inFlight.back(); + batch.completions.push_back(std::move(action)); + ++batch.completionLabelCount; + completionLabels.fetch_add(1, std::memory_order_acq_rel); + if (&batch == open.get() && activeRecorder == this && pendingLabelSince.load(std::memory_order_relaxed) == NoPendingLabel) { + pendingLabelSince.store(std::chrono::steady_clock::now().time_since_epoch().count(), std::memory_order_release); + } + return true; +} + void Recorder::NoteWrittenBack(std::uint64_t address, std::size_t bytes) { Recorder* recorder = activeRecorder; if (recorder == nullptr || bytes == 0) return; diff --git a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp index 8ad28f8fe..cd4851a85 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp @@ -294,6 +294,28 @@ void completionCountTests(const Device& device, Recorder& recorder) { Require(memory[0] == 1 && memory[2] == (storeAlways ? 1u : 7u) && memory[16] == 1, "completion stores ran for the wrong labels (overlapped and not-imported ones store, the untouched GPU-stored one skips)"); } +void afterRecordedWorkTests(const Device& device, Recorder& recorder) { + alignas(64) static std::uint32_t memory[16]; + const auto base = reinterpret_cast(memory); + const std::array value{std::byte{1}, std::byte{0}, std::byte{0}, std::byte{0}}; + recorder.Sync(); + std::vector ran; + std::uint32_t seen = 0; + Require(recorder.Idle() && !recorder.AfterRecordedWork([&] { ran.push_back(0); }) && ran.empty() && Recorder::PendingCompletionLabels() == 0, "an idle recorder kept an action for recorded work"); + recorder.NotePendingWrite(0x52000, 0x100); + Require(!Recorder::PendingLabelSince().has_value(), "a write started the label flush deadline"); + Require(recorder.AfterRecordedWork([&] { ran.push_back(1); }) && Recorder::PendingCompletionLabels() == 1 && Recorder::PendingLabelSince().has_value(), "an action behind the open batch is not pending under the label flush deadline"); + recorder.AfterCompletions(base, value, 6, 0, false); + Require(recorder.AfterRecordedWork([&] { seen = memory[0]; ran.push_back(2); }) && Recorder::PendingCompletionLabels() == 3, "an action behind a completion label is not pending"); + recorder.Submit(); + Require(recorder.AfterRecordedWork([&] { ran.push_back(3); }) && Recorder::PendingCompletionLabels() == 4, "an action behind an in-flight batch is not pending"); + device.WaitQueue(); + Require(ran.empty(), "an action ran before its batch was reaped"); + recorder.Sync(); + Require(ran == std::vector{1, 2, 3} && Recorder::PendingCompletionLabels() == 0 && recorder.Idle(), "actions behind recorded work did not run once each, in order"); + Require(seen == 1, "an action ran before the completion label recorded ahead of it"); +} + void labelTests(Recorder& recorder) { const std::array value{std::byte{1}, std::byte{0}, std::byte{0}, std::byte{0}}; Require(!recorder.PendingLabelIn(0x60000, 0x100), "an empty table reports a label"); @@ -2278,6 +2300,7 @@ int main() { readTrackingTests(device, recorder); writeSettledTests(device, recorder); completionCountTests(device, recorder); + afterRecordedWorkTests(device, recorder); labelTests(recorder); lateLabelTests(recorder); unchangedSinceTests(); diff --git a/core/libs/prx/libSceAgcDriver/tests/Submit.cpp b/core/libs/prx/libSceAgcDriver/tests/Submit.cpp index cb1fc3fac..29b596b07 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Submit.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Submit.cpp @@ -222,6 +222,30 @@ std::vector commands(const std::array&... packe return words; } +std::array endOfPipeLabel(volatile std::uint32_t* address, std::uint32_t value) { + const auto target = reinterpret_cast(address); + return {0xc0064900, 0x514, (1u << 29u) | (2u << 24u), static_cast(target), static_cast(static_cast(target) >> 32u), value, 0, 0}; +} + +void testEndOfPipeLabelsWithoutWork() { + alignas(64) static volatile std::uint32_t first = 0, second = 0, done = 0; + KernelEqueue eq = 0; + check(sceKernelCreateEqueue(&eq, "AGC EOP labels") == 0, "event queue creation failed"); + auto owner = EqueuePin_nid_postfix(eq); + int tag = 0; + check(sceAgcDriverAddEqEvent(eq, 0, &tag) == 0, "graphics event registration failed"); + submit(0x20, commands(waitEqual(&second, 2), writeData(&done, 1))); + submit(0, commands(endOfPipeLabel(&first, 1), endOfPipeLabel(&second, 2))); + waitFor(&done, 1, "an end-of-pipe label with no work before it never landed"); + check(first == 1 && second == 2, "end-of-pipe labels with no work before them landed wrong"); + AgcDriverWaitIdle_nid_postfix(); + std::array events{}; + check(owner->GetTriggeredEvents(events.data(), 2) == 1 && events[0].udata == &tag && events[0].data == 2, "end-of-pipe labels with no work before them lost their interrupts"); + check(sceAgcDriverDeleteEqEvent(eq, 0) == 0, "graphics event deletion failed"); + owner.reset(); + check(sceKernelDeleteEqueue(eq) == 0, "event queue deletion failed"); +} + void testLabelStoredSinceSubmission() { alignas(64) static volatile std::uint32_t gate = 0, label = 0, done = 0, late = 0; submit(0x20, commands(waitEqual(&gate, 1), waitEqual(&label, 1), writeData(&done, 1))); @@ -297,6 +321,7 @@ int main() { testSubmissions(); testEndOfPipeInterrupts(); testLabelStoredSinceSubmission(); + testEndOfPipeLabelsWithoutWork(); testLabelHeldAtSubmission(); testWideLabelStoredSinceSubmission(); testWorkerFailure();