diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp index b945e50cb..895548081 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp @@ -207,7 +207,7 @@ private: void recordQueuedLabelsByTry(std::uint32_t queue, std::atomic& counter); static void recordQueuedLabelsFromHook(); void flushBetweenPackets(std::uint32_t queue, std::uint32_t header, bool labelPacket); - bool preparePacketMemory(const Submission& submission, QueueState& queue, std::span packet, std::uint32_t header, std::uint32_t opcode, bool& wroteOnGpu, bool& endOfPipeInterrupt, bool& drawPacket, bool& sampleDump); + bool preparePacketMemory(const Submission& submission, QueueState& queue, std::span packet, std::uint32_t header, std::uint32_t opcode, bool& wroteOnGpu, bool& endOfPipeInterrupt, bool& interruptDeferred, bool& drawPacket, bool& sampleDump); void dumpSampleCounters(std::uint64_t address); template static void timed(double WorkerProfile::*bucket, TWork&& work); diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp index 02cd2611c..ba0a0a204 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp @@ -71,6 +71,7 @@ public: // `reapFirst` retires finished batches before the checks; a caller recording a group of labels // under one lock passes it for the first label only. int WriteLabelOnGpu(std::uint64_t address, std::span bytes, std::uint64_t stamp, std::uint32_t queue, bool reapFirst = true); + bool AfterRecordedWork(std::function action, bool reapFirst); // Pending-label table lookup and open-batch overlap test for WAIT_REG_MEM (see Recorder). std::optional PendingLabel(std::uint64_t address, std::size_t bytes, std::uint64_t afterStamp, Graphics::Recorder::LabelRefusal* refusal = nullptr) const; bool OpenWriteOverlaps(std::uint64_t address, std::size_t bytes) const; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp index 62e593360..dac8ff4ef 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp @@ -152,8 +152,8 @@ void Driver::execute(const Submission& submission) { }; if (profilePackets) packetProfile.flushMs += std::chrono::duration(packetTimer.start - flushStart).count(); - bool wroteOnGpu = false, endOfPipeInterrupt = false, drawPacket = false, sampleDump = false; - const bool drains = preparePacketMemory(submission, queue, packet, header, opcode, wroteOnGpu, endOfPipeInterrupt, drawPacket, sampleDump); + bool wroteOnGpu = false, endOfPipeInterrupt = false, interruptDeferred = false, drawPacket = false, sampleDump = false; + const bool drains = preparePacketMemory(submission, queue, packet, header, opcode, wroteOnGpu, endOfPipeInterrupt, interruptDeferred, drawPacket, sampleDump); traceLabel(packet, submission.queue); const bool waitPacket = opcode == 0x3c || opcode == 0x93 || header == RenderingWaitPacketHeader; @@ -279,7 +279,7 @@ void Driver::execute(const Submission& submission) { if (const auto label = Pm4::DecodeLabelWrite(packet)) noteLabelStore(label->address, label->Bytes(), ++eventSerial); } } - if (endOfPipeInterrupt) AgcDriverDeliverEopInterrupt(submission.queue); + if (endOfPipeInterrupt && !interruptDeferred) AgcDriverDeliverEopInterrupt(submission.queue); } if (drawPacket) Graphics::Recorder::CountRecordedWork(); cursor += count; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp index 4d514defe..e5301b851 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp @@ -4,12 +4,13 @@ #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/DeferredLabels.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libSceAgcDriver/Execution/include/Pm4.hpp" +#include "prx/libSceAgcDriver/Eq/include/Event.hpp" #include #include namespace AgcDriver::DriverDetail { -bool Driver::preparePacketMemory(const Submission& submission, QueueState& queue, std::span packet, std::uint32_t header, std::uint32_t opcode, bool& wroteOnGpu, bool& endOfPipeInterrupt, bool& drawPacket, bool& sampleDump) { +bool Driver::preparePacketMemory(const Submission& submission, QueueState& queue, std::span packet, std::uint32_t header, std::uint32_t opcode, bool& wroteOnGpu, bool& endOfPipeInterrupt, bool& interruptDeferred, bool& drawPacket, bool& sampleDump) { static const bool drainAll = std::getenv("APS5_DRAIN_ALL") != nullptr; static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; wroteOnGpu = false; @@ -17,6 +18,42 @@ bool Driver::preparePacketMemory(const Submission& submission, QueueState& queue bool orderedAlready = false; endOfPipeInterrupt = opcode == 0x49 && ((packet[2] >> 24u) & 7u) != 0; + interruptDeferred = false; + if (!drainAll && endOfPipeInterrupt) { + const auto label = Pm4::DecodeLabelWrite(packet); + const bool storesNothing = (packet[2] >> 29u) == 0 || (packet[3] | (static_cast(packet[4]) << 32u)) == 0; + if (label.has_value() || storesNothing) { + bumpEpoch(&EpochBumps::drains); + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Label); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + const auto localDevice = device.Load(); + recordDeferredLabels(localDevice.get(), submission.queue); + const bool workOpen = Graphics::Recorder::RecordedWorkSinceSubmit() != 0; + int reason = localDevice != nullptr ? 0 : 1; + if (label.has_value()) { + const auto bytes = label->Bytes(); + const auto stamp = ++eventSerial; + reason = localDevice != nullptr ? localDevice->WriteLabelOnGpu(label->address, bytes, stamp, submission.queue) : 4; + if (reason == 1) GuestMemory::Write(label->address, bytes, 4); + if (reason == 0 || reason == 1 || reason == 5 || reason == 6) { + noteLabelStore(label->address, bytes, stamp); + Graphics::Recorder::CloseLabelGroup(GuestMemory::TrackerGeneration()); + } + countLabelOutcome(reason); + ++immediateLabels; + } else { + ++noOpLabels; + } + if (reason == 0 || reason == 5 || reason == 6) { + const auto queueId = submission.queue; + interruptDeferred = localDevice->AfterRecordedWork([queueId] { AgcDriverDeliverEopInterrupt(queueId); }, submission.queue == 0); + if (interruptDeferred && workOpen) localDevice->SubmitRecorded(submission.queue == 0); + wroteOnGpu = true; + } else if (reason == 1) { + wroteOnGpu = true; + } + } + } if (!drainAll && !endOfPipeInterrupt && (opcode == 0x49 || opcode == 0x37)) { if (const auto label = Pm4::DecodeLabelWrite(packet)) { const auto bytes = label->Bytes(); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp index 2b859eb91..ce218b07d 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp @@ -1127,6 +1127,12 @@ int VulkanDevice::WriteLabelOnGpu(std::uint64_t address, std::span action, bool reapFirst) { + if (!state->recorder) return false; + if (reapFirst && OpportunisticReap()) state->recorder->Reap(); + return state->recorder->AfterRecordedWork(std::move(action)); +} + bool VulkanDevice::FillBuffer(std::uint64_t address, std::size_t bytes, std::span pattern) { if (!state->recorder || bytes == 0 || bytes % 16 != 0 || address % 16 != 0) return false; auto& recorder = *state->recorder; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp index 816f8844a..532d2f7e8 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp @@ -316,6 +316,7 @@ public: // completion store would land on memory the game may have reused by then (a stale label value // over a fresh command buffer). Debug aid: APS5_LABEL_STORE_ALWAYS=1 stores unconditionally. void AfterCompletions(std::uint64_t address, std::span bytes, std::uint64_t stamp, std::uint32_t queue, bool storedOnGpu); + bool AfterRecordedWork(std::function action); // A CPU store of GPU results into guest memory (GuestBufferMemory::WriteBack): recorded so a // completion label store can tell whether its bytes were overwritten. Under GuestMemory:: // GpuMutex (every write-back runs inside finish() or a synchronous draw's wait): the first diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp index 0c25fe4f6..ee7fb97b1 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp @@ -2359,6 +2359,18 @@ void Recorder::AfterCompletions(std::uint64_t address, std::span action) { + if (Idle()) return false; + Batch& batch = open != nullptr ? *open : *inFlight.back(); + batch.completions.push_back(std::move(action)); + ++batch.completionLabelCount; + completionLabels.fetch_add(1, std::memory_order_acq_rel); + if (&batch == open.get() && activeRecorder == this && pendingLabelSince.load(std::memory_order_relaxed) == NoPendingLabel) { + pendingLabelSince.store(std::chrono::steady_clock::now().time_since_epoch().count(), std::memory_order_release); + } + return true; +} + void Recorder::NoteWrittenBack(std::uint64_t address, std::size_t bytes) { Recorder* recorder = activeRecorder; if (recorder == nullptr || bytes == 0) return; diff --git a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp index 8ad28f8fe..cd4851a85 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp @@ -294,6 +294,28 @@ void completionCountTests(const Device& device, Recorder& recorder) { Require(memory[0] == 1 && memory[2] == (storeAlways ? 1u : 7u) && memory[16] == 1, "completion stores ran for the wrong labels (overlapped and not-imported ones store, the untouched GPU-stored one skips)"); } +void afterRecordedWorkTests(const Device& device, Recorder& recorder) { + alignas(64) static std::uint32_t memory[16]; + const auto base = reinterpret_cast(memory); + const std::array value{std::byte{1}, std::byte{0}, std::byte{0}, std::byte{0}}; + recorder.Sync(); + std::vector ran; + std::uint32_t seen = 0; + Require(recorder.Idle() && !recorder.AfterRecordedWork([&] { ran.push_back(0); }) && ran.empty() && Recorder::PendingCompletionLabels() == 0, "an idle recorder kept an action for recorded work"); + recorder.NotePendingWrite(0x52000, 0x100); + Require(!Recorder::PendingLabelSince().has_value(), "a write started the label flush deadline"); + Require(recorder.AfterRecordedWork([&] { ran.push_back(1); }) && Recorder::PendingCompletionLabels() == 1 && Recorder::PendingLabelSince().has_value(), "an action behind the open batch is not pending under the label flush deadline"); + recorder.AfterCompletions(base, value, 6, 0, false); + Require(recorder.AfterRecordedWork([&] { seen = memory[0]; ran.push_back(2); }) && Recorder::PendingCompletionLabels() == 3, "an action behind a completion label is not pending"); + recorder.Submit(); + Require(recorder.AfterRecordedWork([&] { ran.push_back(3); }) && Recorder::PendingCompletionLabels() == 4, "an action behind an in-flight batch is not pending"); + device.WaitQueue(); + Require(ran.empty(), "an action ran before its batch was reaped"); + recorder.Sync(); + Require(ran == std::vector{1, 2, 3} && Recorder::PendingCompletionLabels() == 0 && recorder.Idle(), "actions behind recorded work did not run once each, in order"); + Require(seen == 1, "an action ran before the completion label recorded ahead of it"); +} + void labelTests(Recorder& recorder) { const std::array value{std::byte{1}, std::byte{0}, std::byte{0}, std::byte{0}}; Require(!recorder.PendingLabelIn(0x60000, 0x100), "an empty table reports a label"); @@ -2278,6 +2300,7 @@ int main() { readTrackingTests(device, recorder); writeSettledTests(device, recorder); completionCountTests(device, recorder); + afterRecordedWorkTests(device, recorder); labelTests(recorder); lateLabelTests(recorder); unchangedSinceTests(); diff --git a/core/libs/prx/libSceAgcDriver/tests/Submit.cpp b/core/libs/prx/libSceAgcDriver/tests/Submit.cpp index cb1fc3fac..29b596b07 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Submit.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Submit.cpp @@ -222,6 +222,30 @@ std::vector commands(const std::array&... packe return words; } +std::array endOfPipeLabel(volatile std::uint32_t* address, std::uint32_t value) { + const auto target = reinterpret_cast(address); + return {0xc0064900, 0x514, (1u << 29u) | (2u << 24u), static_cast(target), static_cast(static_cast(target) >> 32u), value, 0, 0}; +} + +void testEndOfPipeLabelsWithoutWork() { + alignas(64) static volatile std::uint32_t first = 0, second = 0, done = 0; + KernelEqueue eq = 0; + check(sceKernelCreateEqueue(&eq, "AGC EOP labels") == 0, "event queue creation failed"); + auto owner = EqueuePin_nid_postfix(eq); + int tag = 0; + check(sceAgcDriverAddEqEvent(eq, 0, &tag) == 0, "graphics event registration failed"); + submit(0x20, commands(waitEqual(&second, 2), writeData(&done, 1))); + submit(0, commands(endOfPipeLabel(&first, 1), endOfPipeLabel(&second, 2))); + waitFor(&done, 1, "an end-of-pipe label with no work before it never landed"); + check(first == 1 && second == 2, "end-of-pipe labels with no work before them landed wrong"); + AgcDriverWaitIdle_nid_postfix(); + std::array events{}; + check(owner->GetTriggeredEvents(events.data(), 2) == 1 && events[0].udata == &tag && events[0].data == 2, "end-of-pipe labels with no work before them lost their interrupts"); + check(sceAgcDriverDeleteEqEvent(eq, 0) == 0, "graphics event deletion failed"); + owner.reset(); + check(sceKernelDeleteEqueue(eq) == 0, "event queue deletion failed"); +} + void testLabelStoredSinceSubmission() { alignas(64) static volatile std::uint32_t gate = 0, label = 0, done = 0, late = 0; submit(0x20, commands(waitEqual(&gate, 1), waitEqual(&label, 1), writeData(&done, 1))); @@ -297,6 +321,7 @@ int main() { testSubmissions(); testEndOfPipeInterrupts(); testLabelStoredSinceSubmission(); + testEndOfPipeLabelsWithoutWork(); testLabelHeldAtSubmission(); testWideLabelStoredSinceSubmission(); testWorkerFailure();