diff --git a/src/graphics/guest_gpu/command_processor/commandProcessor.h b/src/graphics/guest_gpu/command_processor/commandProcessor.h index 55fa3bad..7729fd16 100644 --- a/src/graphics/guest_gpu/command_processor/commandProcessor.h +++ b/src/graphics/guest_gpu/command_processor/commandProcessor.h @@ -32,11 +32,12 @@ private: struct BufferCursor { std::span commands; - uint32_t offset_dw = 0; - uint32_t deferred_advance_dw = 0; + uint32_t offset_dw = 0; }; std::vector m_buffer_stack; + std::span m_next_buffer; + bool m_chain = false; bool m_suspended = false; bool m_made_progress = false; }; @@ -129,7 +130,7 @@ public: [[nodiscard]] bool ShouldSkipPredicatedPackets() const { return m_predicate_skip; } Pm4ProcessResult Process(Pm4Execution& execution, std::span commands); - void ProcessIndirectBuffer(std::span commands); + void ProcessIndirectBuffer(std::span commands, bool chain); void SetFlip(const FlipInfo& flip) { m_flip = flip; } @@ -143,7 +144,7 @@ private: uint32_t cache_action, uint32_t event_index, uint32_t event_write_source, void* dst_gpu_addr, T value, uint32_t interrupt_selector, uint32_t interrupt_context_id); - void ProcessPm4(Pm4Execution& execution, size_t stop_depth); + void ProcessPm4(Pm4Execution& execution); void SuspendPm4(); CommandScheduler& GetScheduler() const { return m_renderer.GetCommandScheduler(); } CommandBuffer& CurrentBuffer() { return GetScheduler().Current(); } diff --git a/src/graphics/guest_gpu/command_processor/pm4Handlers.cpp b/src/graphics/guest_gpu/command_processor/pm4Handlers.cpp index fb0c2d1b..1bc4a71f 100644 --- a/src/graphics/guest_gpu/command_processor/pm4Handlers.cpp +++ b/src/graphics/guest_gpu/command_processor/pm4Handlers.cpp @@ -1495,10 +1495,10 @@ KYTY_CP_OP_PARSER(CpOpBranch) { reinterpret_cast(else_buffer), else_num_dw); if (take_then) { - cp.ProcessIndirectBuffer({then_buffer, then_num_dw}); + cp.ProcessIndirectBuffer({then_buffer, then_num_dw}, true); } else if (mode == 2 && else_num_dw != 0) { EXIT_NOT_IMPLEMENTED(else_buffer == nullptr); - cp.ProcessIndirectBuffer({else_buffer, else_num_dw}); + cp.ProcessIndirectBuffer({else_buffer, else_num_dw}, true); } return payload_dw; @@ -1959,7 +1959,7 @@ KYTY_CP_OP_PARSER(CpOpIndirectBuffer) { GraphicsDbgDumpDcb("ci", indirect_num_dw, indirect_buffer); - cp.ProcessIndirectBuffer({indirect_buffer, indirect_num_dw}); + cp.ProcessIndirectBuffer({indirect_buffer, indirect_num_dw}, (control & (1u << 20u)) != 0); return 3; } diff --git a/src/graphics/guest_gpu/graphicsRun.cpp b/src/graphics/guest_gpu/graphicsRun.cpp index 5992aa66..b291efeb 100644 --- a/src/graphics/guest_gpu/graphicsRun.cpp +++ b/src/graphics/guest_gpu/graphicsRun.cpp @@ -672,20 +672,16 @@ Pm4ProcessResult CommandProcessor::Process(Pm4Execution& execution, Pm4Execution* previous_execution; } execution_scope(*this, execution); - ProcessPm4(execution, 0); + ProcessPm4(execution); return execution.m_buffer_stack.empty() ? Pm4ProcessResult::Complete : Pm4ProcessResult::Blocked; } -void CommandProcessor::ProcessIndirectBuffer(std::span commands) { +void CommandProcessor::ProcessIndirectBuffer(std::span commands, bool chain) { EXIT_IF(g_current_execution == nullptr); - if (commands.empty()) { - return; - } - auto& execution = *g_current_execution; - const auto stop_depth = execution.m_buffer_stack.size(); - execution.m_buffer_stack.push_back({commands}); - ProcessPm4(execution, stop_depth); + EXIT_IF(!g_current_execution->m_next_buffer.empty()); + g_current_execution->m_next_buffer = commands; + g_current_execution->m_chain = chain; } void CommandProcessor::SuspendPm4() { @@ -693,21 +689,13 @@ void CommandProcessor::SuspendPm4() { g_current_execution->m_suspended = true; } -void CommandProcessor::ProcessPm4(Pm4Execution& execution, size_t stop_depth) { - while (execution.m_buffer_stack.size() > stop_depth) { +void CommandProcessor::ProcessPm4(Pm4Execution& execution) { + while (!execution.m_buffer_stack.empty()) { if (g_gpu_state != nullptr) { g_gpu_state->ProcessCommands(); } - const auto buffer_index = execution.m_buffer_stack.size() - 1; - auto& cursor = execution.m_buffer_stack[buffer_index]; + auto& cursor = execution.m_buffer_stack.back(); EXIT_IF(cursor.offset_dw > cursor.commands.size()); - if (cursor.deferred_advance_dw != 0) { - EXIT_IF(cursor.deferred_advance_dw > cursor.commands.size() - cursor.offset_dw); - cursor.offset_dw += cursor.deferred_advance_dw; - cursor.deferred_advance_dw = 0; - execution.m_made_progress = true; - continue; - } if (cursor.offset_dw == cursor.commands.size()) { execution.m_buffer_stack.pop_back(); continue; @@ -783,14 +771,19 @@ void CommandProcessor::ProcessPm4(Pm4Execution& execution, size_t stop_depth) { handler(*this, packet_header & ~1u, packet + 1, remaining_dw, total_dw) + 1; EXIT_IF(packet_dw > remaining_dw); if (execution.m_suspended) { - if (execution.m_buffer_stack.size() > buffer_index + 1) { - execution.m_buffer_stack[buffer_index].deferred_advance_dw = packet_dw; - } return; } - EXIT_IF(execution.m_buffer_stack.size() != buffer_index + 1); - execution.m_buffer_stack[buffer_index].offset_dw += packet_dw; + cursor.offset_dw += packet_dw; execution.m_made_progress = true; + if (!execution.m_next_buffer.empty()) { + // Chains and taken branches reuse the fetcher; only calls retain a return cursor. + if (execution.m_chain) { + cursor = {execution.m_next_buffer}; + } else { + execution.m_buffer_stack.push_back({execution.m_next_buffer}); + } + execution.m_next_buffer = {}; + } } } diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 8347e562..14bb78b9 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -30295,6 +30295,89 @@ void CheckPm4ContextStateOperations(RenderContext &renderer) { std::printf("[host] %-32s ok\n", "Pm4ContextState"); } +void CheckPm4IndirectControlFlow(RenderContext &renderer) { + constexpr const char *name = "Pm4IndirectControlFlow"; + GraphicsInitJmpTables(); + CommandProcessor processor(renderer, 0); + uint32_t selected = 0; + uint32_t returned = 0; + const auto address = [](const void *value) { + return reinterpret_cast(value); + }; + const auto write = [&](uint32_t *destination, uint32_t value) { + return std::array{ + KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0), 0, + static_cast(address(destination)), + static_cast(address(destination) >> 32u), value}; + }; + const auto indirect = [&](std::span target, bool chain) { + uint32_t control = 0x0f200000u | static_cast(target.size()); + if (chain) { + control |= 1u << 20u; + } + return std::array{ + KYTY_PM4(4, Pm4::IT_INDIRECT_BUFFER, 0), + static_cast(address(target.data())), + static_cast(address(target.data()) >> 32u), control}; + }; + const auto then_commands = write(&selected, 11); + const auto else_commands = write(&selected, 33); + const auto branch_suffix = write(&selected, 44); + const auto caller_suffix = write(&returned, 22); + alignas(8) uint64_t condition = 0; + std::array branch{ + KYTY_PM4(14, Pm4::IT_INDIRECT_BUFFER, 0), 0, + static_cast(address(&condition)), + static_cast(address(&condition) >> 32u), UINT32_MAX, + UINT32_MAX, 1, 0, + static_cast(address(then_commands.data())), + static_cast(address(then_commands.data()) >> 32u), + static_cast(then_commands.size()), + static_cast(address(else_commands.data())), + static_cast(address(else_commands.data()) >> 32u), + static_cast(else_commands.size())}; + std::copy(branch_suffix.begin(), branch_suffix.end(), branch.begin() + 14); + std::array caller{}; + const auto call_branch = indirect(branch, false); + std::copy(call_branch.begin(), call_branch.end(), caller.begin()); + std::copy(caller_suffix.begin(), caller_suffix.end(), caller.begin() + 4); + for (uint32_t mode : {1u, 2u}) { + branch[1] = mode | (3u << 8u); + for (uint64_t value : {0ull, 1ull}) { + condition = value; + selected = returned = 0; + uint32_t expected = 44; + if (value == 1) { + expected = 11; + } else if (mode == 2) { + expected = 33; + } + Pm4Execution execution; + Require(name, "conditional fetcher replacement", + processor.Process(execution, caller) == Pm4ProcessResult::Complete && + selected == expected && returned == 22, + "taken branch resumed its discarded suffix or lost the caller's return"); + } + } + + // This stream exceeds native recursion capacity, while chains need one fetcher cursor. + std::vector> links(65536); + std::span target = then_commands; + for (auto &link : links) { + link = indirect(target, true); + target = link; + } + const auto call_chain = indirect(target, false); + std::copy(call_chain.begin(), call_chain.end(), caller.begin()); + selected = returned = 0; + Pm4Execution execution; + Require(name, "long chain and call return", + processor.Process(execution, caller) == Pm4ProcessResult::Complete && + selected == 11 && returned == 22, + "long chain did not complete and return to the original caller"); + std::printf("[host] %-32s ok\n", name); +} + void CheckPm4WaitResume(RenderContext &renderer) { GraphicsInitJmpTables(); CommandProcessor processor(renderer, 0); @@ -30324,7 +30407,7 @@ void CheckPm4WaitResume(RenderContext &renderer) { nested[0] = KYTY_PM4(4, Pm4::IT_INDIRECT_BUFFER, 0); nested[1] = static_cast(address(child.data())); nested[2] = static_cast(address(child.data()) >> 32u); - nested[3] = 0x0f200000u | static_cast(child.size()); + nested[3] = 0x0f300000u | static_cast(child.size()); std::array commands{}; commands[0] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0); @@ -30347,6 +30430,10 @@ void CheckPm4WaitResume(RenderContext &renderer) { processor.Process(execution, commands) == Pm4ProcessResult::Blocked && prefix == 11 && child_observation == 0 && suffix == 0, "blocked indirect wait did not preserve its command position"); + Require("Pm4WaitResume", "still blocked", + processor.Process(execution, commands) == Pm4ProcessResult::Blocked && + !execution.MadeProgress() && suffix == 0, + "blocked wait advanced or replayed its caller"); label = 1; child[4] = 1; @@ -30609,6 +30696,10 @@ int main(int argc, char **argv) { if (argc == 2 && std::strcmp(argv[1], "--gpu-command-lane-only") == 0) { VulkanHarness vulkan; vulkan.CheckGpuCommandLane(); + CheckPm4IndirectControlFlow(vulkan.RuntimeRenderer()); + CheckPm4WaitResume(vulkan.RuntimeRenderer()); + CheckPm4RewindResume(vulkan.RuntimeRenderer()); + CheckPm4CeCompletion(vulkan.RuntimeRenderer()); return 0; } if (argc == 2 && std::strcmp(argv[1], "--alignbyte-only") == 0) { @@ -30878,6 +30969,7 @@ int main(int argc, char **argv) { CheckAgcWaitPackets(vulkan.RuntimeRenderer()); CheckAgcDrawIndirectMultiPacket(vulkan.RuntimeRenderer()); CheckPm4ContextStateOperations(vulkan.RuntimeRenderer()); + CheckPm4IndirectControlFlow(vulkan.RuntimeRenderer()); CheckPm4WaitResume(vulkan.RuntimeRenderer()); CheckPm4RewindResume(vulkan.RuntimeRenderer()); CheckPm4CeCompletion(vulkan.RuntimeRenderer());