perf(agc): deliver end-of-pipe interrupts after their work without draining

A RELEASE_MEM with an interrupt select drained the device: the queue
worker submitted everything recorded and waited for the GPU before it
stored the label and delivered the interrupt. PPSA21564 issues about 300
of them per frame, so the worker and the GPU took turns.

The label is now recorded like any other (on the GPU behind the work, or
as a completion action behind pending write-backs), and the delivery
becomes a completion action of the batch holding the work before the
packet (Recorder::AfterRecordedWork, after the batch's earlier
completions). It is counted with the completion labels, so idle workers
and pollers reap it, and it starts the label flush deadline. With draws
or dispatches recorded since the last submit the batch is submitted at
once. A batch holding only label stores stays open and goes out with the
next work, at the label flush deadline, at a wait, at a flip or at the
end of the submission, like any other pending label. A label the GPU
cannot store, or one that does not decode, still drains.

The interrupt so never arrives before the work ahead of it completed and
its label landed; it can only arrive later, by at most the flush
deadline plus the work recorded into the same batch.

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp

@@ -207,7 +207,7 @@ private:
     void recordQueuedLabelsByTry(std::uint32_t queue, std::atomic<std::uint64_t>& counter);
     static void recordQueuedLabelsFromHook();
     void flushBetweenPackets(std::uint32_t queue, std::uint32_t header, bool labelPacket);
-    bool preparePacketMemory(const Submission& submission, QueueState& queue, std::span<const std::uint32_t> packet, std::uint32_t header, std::uint32_t opcode, bool& wroteOnGpu, bool& endOfPipeInterrupt, bool& drawPacket, bool& sampleDump);
+    bool preparePacketMemory(const Submission& submission, QueueState& queue, std::span<const std::uint32_t> packet, std::uint32_t header, std::uint32_t opcode, bool& wroteOnGpu, bool& endOfPipeInterrupt, bool& interruptDeferred, bool& drawPacket, bool& sampleDump);
     void dumpSampleCounters(std::uint64_t address);
     template <typename TWork>
     static void timed(double WorkerProfile::*bucket, TWork&& work);

core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp

@@ -71,6 +71,7 @@ public:
     // `reapFirst` retires finished batches before the checks; a caller recording a group of labels
     // under one lock passes it for the first label only.
     int WriteLabelOnGpu(std::uint64_t address, std::span<const std::byte> bytes, std::uint64_t stamp, std::uint32_t queue, bool reapFirst = true);
+    bool AfterRecordedWork(std::function<void()> action, bool reapFirst);
     // Pending-label table lookup and open-batch overlap test for WAIT_REG_MEM (see Recorder).
     std::optional<Graphics::Recorder::LabelHit> PendingLabel(std::uint64_t address, std::size_t bytes, std::uint64_t afterStamp, Graphics::Recorder::LabelRefusal* refusal = nullptr) const;
     bool OpenWriteOverlaps(std::uint64_t address, std::size_t bytes) const;

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp

@@ -152,8 +152,8 @@ void Driver::execute(const Submission& submission) {
         };
         if (profilePackets) packetProfile.flushMs += std::chrono::duration<double, std::milli>(packetTimer.start - flushStart).count();
 
-        bool wroteOnGpu = false, endOfPipeInterrupt = false, drawPacket = false, sampleDump = false;
-        const bool drains = preparePacketMemory(submission, queue, packet, header, opcode, wroteOnGpu, endOfPipeInterrupt, drawPacket, sampleDump);
+        bool wroteOnGpu = false, endOfPipeInterrupt = false, interruptDeferred = false, drawPacket = false, sampleDump = false;
+        const bool drains = preparePacketMemory(submission, queue, packet, header, opcode, wroteOnGpu, endOfPipeInterrupt, interruptDeferred, drawPacket, sampleDump);
         traceLabel(packet, submission.queue);
 
         const bool waitPacket = opcode == 0x3c || opcode == 0x93 || header == RenderingWaitPacketHeader;
@@ -279,7 +279,7 @@ void Driver::execute(const Submission& submission) {
                     if (const auto label = Pm4::DecodeLabelWrite(packet)) noteLabelStore(label->address, label->Bytes(), ++eventSerial);
                 }
             }
-            if (endOfPipeInterrupt) AgcDriverDeliverEopInterrupt(submission.queue);
+            if (endOfPipeInterrupt && !interruptDeferred) AgcDriverDeliverEopInterrupt(submission.queue);
         }
         if (drawPacket) Graphics::Recorder::CountRecordedWork();
         cursor += count;

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp

@@ -4,12 +4,13 @@
 #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/DeferredLabels.hpp"
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Pm4.hpp"
+#include "prx/libSceAgcDriver/Eq/include/Event.hpp"
 #include <cstdlib>
 #include <cstring>
 
 namespace AgcDriver::DriverDetail {
 
-bool Driver::preparePacketMemory(const Submission& submission, QueueState& queue, std::span<const std::uint32_t> packet, std::uint32_t header, std::uint32_t opcode, bool& wroteOnGpu, bool& endOfPipeInterrupt, bool& drawPacket, bool& sampleDump) {
+bool Driver::preparePacketMemory(const Submission& submission, QueueState& queue, std::span<const std::uint32_t> packet, std::uint32_t header, std::uint32_t opcode, bool& wroteOnGpu, bool& endOfPipeInterrupt, bool& interruptDeferred, bool& drawPacket, bool& sampleDump) {
     static const bool drainAll = std::getenv("APS5_DRAIN_ALL") != nullptr;
     static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr;
     wroteOnGpu = false;
@@ -17,6 +18,42 @@ bool Driver::preparePacketMemory(const Submission& submission, QueueState& queue
     bool orderedAlready = false;
 
     endOfPipeInterrupt = opcode == 0x49 && ((packet[2] >> 24u) & 7u) != 0;
+    interruptDeferred = false;
+    if (!drainAll && endOfPipeInterrupt) {
+        const auto label = Pm4::DecodeLabelWrite(packet);
+        const bool storesNothing = (packet[2] >> 29u) == 0 || (packet[3] | (static_cast<std::uint64_t>(packet[4]) << 32u)) == 0;
+        if (label.has_value() || storesNothing) {
+            bumpEpoch(&EpochBumps::drains);
+            GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Label);
+            std::lock_guard gpuLock(GuestMemory::GpuMutex());
+            const auto localDevice = device.Load();
+            recordDeferredLabels(localDevice.get(), submission.queue);
+            const bool workOpen = Graphics::Recorder::RecordedWorkSinceSubmit() != 0;
+            int reason = localDevice != nullptr ? 0 : 1;
+            if (label.has_value()) {
+                const auto bytes = label->Bytes();
+                const auto stamp = ++eventSerial;
+                reason = localDevice != nullptr ? localDevice->WriteLabelOnGpu(label->address, bytes, stamp, submission.queue) : 4;
+                if (reason == 1) GuestMemory::Write(label->address, bytes, 4);
+                if (reason == 0 || reason == 1 || reason == 5 || reason == 6) {
+                    noteLabelStore(label->address, bytes, stamp);
+                    Graphics::Recorder::CloseLabelGroup(GuestMemory::TrackerGeneration());
+                }
+                countLabelOutcome(reason);
+                ++immediateLabels;
+            } else {
+                ++noOpLabels;
+            }
+            if (reason == 0 || reason == 5 || reason == 6) {
+                const auto queueId = submission.queue;
+                interruptDeferred = localDevice->AfterRecordedWork([queueId] { AgcDriverDeliverEopInterrupt(queueId); }, submission.queue == 0);
+                if (interruptDeferred && workOpen) localDevice->SubmitRecorded(submission.queue == 0);
+                wroteOnGpu = true;
+            } else if (reason == 1) {
+                wroteOnGpu = true;
+            }
+        }
+    }
     if (!drainAll && !endOfPipeInterrupt && (opcode == 0x49 || opcode == 0x37)) {
         if (const auto label = Pm4::DecodeLabelWrite(packet)) {
             const auto bytes = label->Bytes();

core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp

@@ -1127,6 +1127,12 @@ int VulkanDevice::WriteLabelOnGpu(std::uint64_t address, std::span<const std::by
     return 0;
 }
 
+bool VulkanDevice::AfterRecordedWork(std::function<void()> action, bool reapFirst) {
+    if (!state->recorder) return false;
+    if (reapFirst && OpportunisticReap()) state->recorder->Reap();
+    return state->recorder->AfterRecordedWork(std::move(action));
+}
+
 bool VulkanDevice::FillBuffer(std::uint64_t address, std::size_t bytes, std::span<const std::uint32_t, 4> pattern) {
     if (!state->recorder || bytes == 0 || bytes % 16 != 0 || address % 16 != 0) return false;
     auto& recorder = *state->recorder;

core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp

@@ -316,6 +316,7 @@ public:
     // completion store would land on memory the game may have reused by then (a stale label value
     // over a fresh command buffer). Debug aid: APS5_LABEL_STORE_ALWAYS=1 stores unconditionally.
     void AfterCompletions(std::uint64_t address, std::span<const std::byte> bytes, std::uint64_t stamp, std::uint32_t queue, bool storedOnGpu);
+    bool AfterRecordedWork(std::function<void()> action);
     // A CPU store of GPU results into guest memory (GuestBufferMemory::WriteBack): recorded so a
     // completion label store can tell whether its bytes were overwritten. Under GuestMemory::
     // GpuMutex (every write-back runs inside finish() or a synchronous draw's wait): the first

core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp

@@ -2359,6 +2359,18 @@ void Recorder::AfterCompletions(std::uint64_t address, std::span<const std::byte
     }
 }
 
+bool Recorder::AfterRecordedWork(std::function<void()> action) {
+    if (Idle()) return false;
+    Batch& batch = open != nullptr ? *open : *inFlight.back();
+    batch.completions.push_back(std::move(action));
+    ++batch.completionLabelCount;
+    completionLabels.fetch_add(1, std::memory_order_acq_rel);
+    if (&batch == open.get() && activeRecorder == this && pendingLabelSince.load(std::memory_order_relaxed) == NoPendingLabel) {
+        pendingLabelSince.store(std::chrono::steady_clock::now().time_since_epoch().count(), std::memory_order_release);
+    }
+    return true;
+}
+
 void Recorder::NoteWrittenBack(std::uint64_t address, std::size_t bytes) {
     Recorder* recorder = activeRecorder;
     if (recorder == nullptr || bytes == 0) return;

core/libs/prx/libSceAgcDriver/tests/Recorder.cpp

@@ -294,6 +294,28 @@ void completionCountTests(const Device& device, Recorder& recorder) {
     Require(memory[0] == 1 && memory[2] == (storeAlways ? 1u : 7u) && memory[16] == 1, "completion stores ran for the wrong labels (overlapped and not-imported ones store, the untouched GPU-stored one skips)");
 }
 
+void afterRecordedWorkTests(const Device& device, Recorder& recorder) {
+    alignas(64) static std::uint32_t memory[16];
+    const auto base = reinterpret_cast<std::uint64_t>(memory);
+    const std::array<std::byte, 4> value{std::byte{1}, std::byte{0}, std::byte{0}, std::byte{0}};
+    recorder.Sync();
+    std::vector<int> ran;
+    std::uint32_t seen = 0;
+    Require(recorder.Idle() && !recorder.AfterRecordedWork([&] { ran.push_back(0); }) && ran.empty() && Recorder::PendingCompletionLabels() == 0, "an idle recorder kept an action for recorded work");
+    recorder.NotePendingWrite(0x52000, 0x100);
+    Require(!Recorder::PendingLabelSince().has_value(), "a write started the label flush deadline");
+    Require(recorder.AfterRecordedWork([&] { ran.push_back(1); }) && Recorder::PendingCompletionLabels() == 1 && Recorder::PendingLabelSince().has_value(), "an action behind the open batch is not pending under the label flush deadline");
+    recorder.AfterCompletions(base, value, 6, 0, false);
+    Require(recorder.AfterRecordedWork([&] { seen = memory[0]; ran.push_back(2); }) && Recorder::PendingCompletionLabels() == 3, "an action behind a completion label is not pending");
+    recorder.Submit();
+    Require(recorder.AfterRecordedWork([&] { ran.push_back(3); }) && Recorder::PendingCompletionLabels() == 4, "an action behind an in-flight batch is not pending");
+    device.WaitQueue();
+    Require(ran.empty(), "an action ran before its batch was reaped");
+    recorder.Sync();
+    Require(ran == std::vector<int>{1, 2, 3} && Recorder::PendingCompletionLabels() == 0 && recorder.Idle(), "actions behind recorded work did not run once each, in order");
+    Require(seen == 1, "an action ran before the completion label recorded ahead of it");
+}
+
 void labelTests(Recorder& recorder) {
     const std::array<std::byte, 4> value{std::byte{1}, std::byte{0}, std::byte{0}, std::byte{0}};
     Require(!recorder.PendingLabelIn(0x60000, 0x100), "an empty table reports a label");
@@ -2278,6 +2300,7 @@ int main() {
         readTrackingTests(device, recorder);
         writeSettledTests(device, recorder);
         completionCountTests(device, recorder);
+        afterRecordedWorkTests(device, recorder);
         labelTests(recorder);
         lateLabelTests(recorder);
         unchangedSinceTests();

core/libs/prx/libSceAgcDriver/tests/Submit.cpp

@@ -222,6 +222,30 @@ std::vector<std::uint32_t> commands(const std::array<std::uint32_t, N>&... packe
     return words;
 }
 
+std::array<std::uint32_t, 8> endOfPipeLabel(volatile std::uint32_t* address, std::uint32_t value) {
+    const auto target = reinterpret_cast<std::uintptr_t>(address);
+    return {0xc0064900, 0x514, (1u << 29u) | (2u << 24u), static_cast<std::uint32_t>(target), static_cast<std::uint32_t>(static_cast<std::uint64_t>(target) >> 32u), value, 0, 0};
+}
+
+void testEndOfPipeLabelsWithoutWork() {
+    alignas(64) static volatile std::uint32_t first = 0, second = 0, done = 0;
+    KernelEqueue eq = 0;
+    check(sceKernelCreateEqueue(&eq, "AGC EOP labels") == 0, "event queue creation failed");
+    auto owner = EqueuePin_nid_postfix(eq);
+    int tag = 0;
+    check(sceAgcDriverAddEqEvent(eq, 0, &tag) == 0, "graphics event registration failed");
+    submit(0x20, commands(waitEqual(&second, 2), writeData(&done, 1)));
+    submit(0, commands(endOfPipeLabel(&first, 1), endOfPipeLabel(&second, 2)));
+    waitFor(&done, 1, "an end-of-pipe label with no work before it never landed");
+    check(first == 1 && second == 2, "end-of-pipe labels with no work before them landed wrong");
+    AgcDriverWaitIdle_nid_postfix();
+    std::array<KernelEvent, 2> events{};
+    check(owner->GetTriggeredEvents(events.data(), 2) == 1 && events[0].udata == &tag && events[0].data == 2, "end-of-pipe labels with no work before them lost their interrupts");
+    check(sceAgcDriverDeleteEqEvent(eq, 0) == 0, "graphics event deletion failed");
+    owner.reset();
+    check(sceKernelDeleteEqueue(eq) == 0, "event queue deletion failed");
+}
+
 void testLabelStoredSinceSubmission() {
     alignas(64) static volatile std::uint32_t gate = 0, label = 0, done = 0, late = 0;
     submit(0x20, commands(waitEqual(&gate, 1), waitEqual(&label, 1), writeData(&done, 1)));
@@ -297,6 +321,7 @@ int main() {
         testSubmissions();
         testEndOfPipeInterrupts();
         testLabelStoredSinceSubmission();
+        testEndOfPipeLabelsWithoutWork();
         testLabelHeldAtSubmission();
         testWideLabelStoredSinceSubmission();
         testWorkerFailure();