From fcc5105aebcd62019a6d52644e426b557c44100b Mon Sep 17 00:00:00 2001 From: Celegans12 <53881228+Celegans12@users.noreply.github.com> Date: Fri, 9 Oct 2026 15:48:06 +0200 Subject: [PATCH] fix(shader): count folded draw offsets once in the vertex and instance indices When the embedded vertex fetch adds the base vertex or start instance user SGPR to the index VGPR, the driver folds that SGPR into the draw's first vertex and first instance. v5/v8 were gl_VertexIndex and gl_InstanceIndex, which already include it, so the translated add counted it twice, on direct draws and CPU-path indirect records alike. The entry of a vertex or local program now subtracts the folded SGPR's user data word from the builtin, which gives the zero-based indices the hardware has. A record drawn in place on the GPU keeps the CPU's word, so the fetch analyzer counts the SGPR as shared unless the fetch add is the only reader of v5/v8 and the SGPR (no earlier access or EXEC write, no carry, clamp, second add or other write of the SGPR); such records take the CPU path. The fold moves into Driver::FoldDrawOffsets, ClassifyIndirectDraw becomes public for the test, and the env switches that skipped the fold go. --- .../Execution/include/Driver/Driver.hpp | 3 +- .../Execution/src/Driver/Draw/Draw.cpp | 14 +- .../src/Driver/Draw/IndirectDraw.cpp | 11 +- .../tests/execution/DrawOffsetIndices.cpp | 500 ++++++++++++++++++ .../Translation/EmbeddedVertexFetch.hpp | 2 + .../Translation/src/EmbeddedVertexFetch.cpp | 168 ++++++ .../Translation/src/InstructionTranslator.cpp | 18 +- 7 files changed, 696 insertions(+), 20 deletions(-) create mode 100644 core/libs/prx/libSceAgcDriver/tests/execution/DrawOffsetIndices.cpp diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp index 2f12dae2e..ba6a2fc8a 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp @@ -60,6 +60,8 @@ public: void ResolveShaderAbi(const Shader* shader, std::span context, std::span primitive); void ResolveGraphicsStagesAbi(std::span stages, std::span context, std::span primitive); void ResolveGraphicsAbi(const Shader* vertex, const Shader* pixel, std::uint32_t primitiveType); + static void FoldDrawOffsets(const ShaderRecompiler::RecompileResult& result, const DrawProgram& program, Pm4::DrawParameters& parameters); + static std::optional ClassifyIndirectDraw(const ShaderRecompiler::RecompileResult& result, const Graphics::State& graphics, const DrawProgram& frontProgram, const std::shared_ptr& localDevice, Pm4::DrawParameters& drawParameters, bool traceIndirect); private: friend class SampledReadScope; @@ -120,7 +122,6 @@ private: static bool drawPrecheck(); std::optional precheckDraw(const QueueState& queue, const Submission& submission, std::span packet, const Pm4::DrawParameters& drawParameters, std::string& rejected, bool& traceIndirect); static std::uint32_t drawUserWord(const DrawProgram& program, std::int32_t sgpr); - std::optional classifyIndirectDraw(const ShaderRecompiler::RecompileResult& result, const Graphics::State& graphics, const DrawProgram& frontProgram, const std::shared_ptr& localDevice, Pm4::DrawParameters& drawParameters, bool traceIndirect); DrawVerdict draw(QueueState& queue, std::span packet, const Submission& submission, std::string& rejected); void addDriverPhases(DispatchClass which, const std::array& ms, bool hit, bool validated); static PendingDispatchPhases& pendingDispatchPhases(); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp index bf6ca03d1..3cee82d8e 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp @@ -179,17 +179,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span packe phaseTiming.Phase(DrawRowDecode); } - static const bool indxOffsetSkipFold = std::getenv("APS5_INDX_OFFSET_SKIP_FOLD") != nullptr; - static const bool indexedOffsetFold = std::getenv("APS5_NO_INDEXED_OFFSET_FOLD") == nullptr; - const auto fold = [&](const ShaderRecompiler::RecompileResult& main, Pm4::DrawParameters& parameters) { - if (parameters.indexed && !indexedOffsetFold) return; - if (main.vertexOffsetSgpr >= 0 && (parameters.firstVertex == 0 || !indxOffsetSkipFold)) { - const auto offset = drawUserWord(programs.front(), main.vertexOffsetSgpr); - require(offset <= std::numeric_limits::max() - parameters.firstVertex, "draw vertex offset overflow"); - parameters.firstVertex += offset; - } - if (main.instanceOffsetSgpr >= 0) parameters.firstInstance = drawUserWord(programs.front(), main.instanceOffsetSgpr); - }; + const auto fold = [&](const ShaderRecompiler::RecompileResult& main, Pm4::DrawParameters& parameters) { FoldDrawOffsets(main, programs.front(), parameters); }; std::optional indirectCpu; std::vector pushOffsets(programs.size(), 0); @@ -210,7 +200,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span packe } const auto& result = *programResults[i]; if (i == 0 && drawParameters.indirect) { - indirectCpu = classifyIndirectDraw(result, graphics, programs.front(), localDevice, drawParameters, traceIndirect); + indirectCpu = ClassifyIndirectDraw(result, graphics, programs.front(), localDevice, drawParameters, traceIndirect); } else if (i == 0) { fold(result, drawParameters); } diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/IndirectDraw.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/IndirectDraw.cpp index a9ebfa025..21d38fdea 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/IndirectDraw.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/IndirectDraw.cpp @@ -12,7 +12,16 @@ std::uint32_t Driver::drawUserWord(const DrawProgram& program, std::int32_t sgpr return program.userData[index]; } -std::optional Driver::classifyIndirectDraw(const ShaderRecompiler::RecompileResult& result, const Graphics::State& graphics, const DrawProgram& frontProgram, const std::shared_ptr& localDevice, Pm4::DrawParameters& drawParameters, bool traceIndirect) { +void Driver::FoldDrawOffsets(const ShaderRecompiler::RecompileResult& result, const DrawProgram& program, Pm4::DrawParameters& parameters) { + if (result.vertexOffsetSgpr >= 0) { + const auto offset = drawUserWord(program, result.vertexOffsetSgpr); + require(offset <= std::numeric_limits::max() - parameters.firstVertex, "draw vertex offset overflow"); + parameters.firstVertex += offset; + } + if (result.instanceOffsetSgpr >= 0) parameters.firstInstance = drawUserWord(program, result.instanceOffsetSgpr); +} + +std::optional Driver::ClassifyIndirectDraw(const ShaderRecompiler::RecompileResult& result, const Graphics::State& graphics, const DrawProgram& frontProgram, const std::shared_ptr& localDevice, Pm4::DrawParameters& drawParameters, bool traceIndirect) { std::optional indirectCpu; auto& indirect = *drawParameters.indirect; using Rule = Pm4::DrawParameters::IndirectDraw::Rule; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/DrawOffsetIndices.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/DrawOffsetIndices.cpp new file mode 100644 index 000000000..775c2150f --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/DrawOffsetIndices.cpp @@ -0,0 +1,500 @@ +#include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; +using Path = AgcDriver::Graphics::IndirectDrawPath; +using Rule = AgcDriver::Pm4::DrawParameters::IndirectDraw::Rule; + +constexpr std::uint32_t BaseVertexSgpr = 16; +constexpr std::uint32_t StartInstanceSgpr = 17; +constexpr std::uint32_t Vertices = 3; +constexpr std::uint32_t Instances = 2; +constexpr std::uint32_t Slots = Vertices * Instances; +constexpr std::uint32_t Unwritten = 0xdeadbeefu; +constexpr std::uint32_t NoLocation = 0x280u; + +alignas(256) constexpr std::array VertexCode{ + 0x7e020305, 0x7e040308, + 0xd70f6a05, 0x00020a10, + 0xd70f6a08, 0x00021011, + 0xf4080506, 0xfa000000, + 0xf4080606, 0xfa000010, + 0x02060b08, + 0xbf8cc07f, + 0xe0002000, 0x80050a03, + 0xe0002000, 0x80060b03, + 0xbf8c3f70, + 0x4a08170a, 0x34080884, + 0x7e180301, 0x7e1a0302, 0x7e1c0305, 0x7e1e0308, + 0xe0781000, 0x80020c04, + 0x7e200280, 0x7e2202f2, + 0xf80008cf, 0x11101010, + 0xbf810000, +}; + +alignas(256) constexpr std::array PostAddCode{ + 0x7e020280, 0x7e040280, + 0xd70f7d05, 0x00020a10, + 0xd70f7d08, 0x00021011, + 0xf4080506, 0xfa000000, + 0xf4080606, 0xfa000010, + 0x02060b08, + 0xbf8cc07f, + 0xe0002000, 0x80050a03, + 0xe0002000, 0x80060b03, + 0xbf8c3f70, + 0x4a08170a, 0x34080884, + 0x7e180301, 0x7e1a0302, 0x7e1c0305, 0x7e1e0308, + 0xe0781000, 0x80020c04, + 0x7e200280, 0x7e2202f2, + 0xf80008cf, 0x11101010, + 0xbf810000, +}; + +alignas(256) constexpr std::array CarryCode{ + 0x7e020280, 0x7e040280, + 0xd70f7d05, 0x00020a10, + 0xd70f6a08, 0x00021011, + 0xf4080506, 0xfa000000, + 0xf4080606, 0xfa000010, + 0x02060b08, + 0xbf8cc07f, + 0xe0002000, 0x80050a03, + 0xe0002000, 0x80060b03, + 0xbf8c3f70, + 0x4a08170a, 0x34080884, + 0x7e180301, 0x7e1a0302, 0x7e1c0305, 0x7e1e0308, + 0xe0781000, 0x80020c04, + 0x7e200280, 0x7e2202f2, + 0xf80008cf, 0x11101010, + 0xbf810000, +}; + +alignas(256) constexpr std::array ClampCode{ + 0x7e020280, 0x7e040280, + 0xd55d8005, 0x04150010, + 0xd70f7d08, 0x00021011, + 0xf4080506, 0xfa000000, + 0xf4080606, 0xfa000010, + 0x02060b08, + 0xbf8cc07f, + 0xe0002000, 0x80050a03, + 0xe0002000, 0x80060b03, + 0xbf8c3f70, + 0x4a08170a, 0x34080884, + 0x7e180301, 0x7e1a0302, 0x7e1c0305, 0x7e1e0308, + 0xe0781000, 0x80020c04, + 0x7e200280, 0x7e2202f2, + 0xf80008cf, 0x11101010, + 0xbf810000, +}; + +alignas(256) constexpr std::array TwiceCode{ + 0x7e020280, 0x7e040280, + 0xd70f7d05, 0x00020a10, + 0xd70f7d05, 0x00020a10, + 0xd70f7d08, 0x00021011, + 0xf4080506, 0xfa000000, + 0xf4080606, 0xfa000010, + 0x02060b08, + 0xbf8cc07f, + 0xe0002000, 0x80050a03, + 0xe0002000, 0x80060b03, + 0xbf8c3f70, + 0x4a08170a, 0x34080884, + 0x7e180301, 0x7e1a0302, 0x7e1c0305, 0x7e1e0308, + 0xe0781000, 0x80020c04, + 0x7e200280, 0x7e2202f2, + 0xf80008cf, 0x11101010, + 0xbf810000, +}; + +alignas(256) constexpr std::array ExecCode{ + 0xbefe03c1, 0x7e040280, + 0xd70f7d05, 0x00020a10, + 0xd70f7d08, 0x00021011, + 0xf4080506, 0xfa000000, + 0xf4080606, 0xfa000010, + 0x02060b08, + 0xbf8cc07f, + 0xe0002000, 0x80050a03, + 0xe0002000, 0x80060b03, + 0xbf8c3f70, + 0x4a08170a, 0x34080884, + 0x7e180301, 0x7e1a0302, 0x7e1c0305, 0x7e1e0308, + 0xe0781000, 0x80020c04, + 0x7e200280, 0x7e2202f2, + 0xf80008cf, 0x11101010, + 0xbf810000, +}; + +alignas(256) constexpr std::array CarryWriteCode{ + 0xd70f1102, 0x00020100, + 0xd70f7d05, 0x00020a10, + 0xd70f7d08, 0x00021011, + 0xf4080506, 0xfa000000, + 0xf4080606, 0xfa000010, + 0x02060b08, + 0xbf8cc07f, + 0xe0002000, 0x80050a03, + 0xe0002000, 0x80060b03, + 0xbf8c3f70, + 0x4a08170a, 0x34080884, + 0x7e180301, 0x7e1a0302, 0x7e1c0305, 0x7e1e0308, + 0xe0781000, 0x80020c04, + 0x7e200280, 0x7e2202f2, + 0xf80008cf, 0x11101010, + 0xbf810000, +}; + +alignas(256) constexpr std::array OtherVgprCode{ + 0x7e020280, 0x7e040280, + 0xd70f7d00, 0x00020010, + 0xd70f7d08, 0x00021011, + 0xf4080506, 0xfa000000, + 0xf4080606, 0xfa000010, + 0x02060b08, + 0xbf8cc07f, + 0xe0002000, 0x80050a03, + 0xe0002000, 0x80060b03, + 0xbf8c3f70, + 0x4a08170a, 0x34080884, + 0x7e180301, 0x7e1a0302, 0x7e1c0305, 0x7e1e0308, + 0xe0781000, 0x80020c04, + 0x7e200280, 0x7e2202f2, + 0xf80008cf, 0x11101010, + 0xbf810000, +}; + +alignas(256) constexpr std::array PixelCode{0x7e0002f2u, 0x7e020280u, 0xf800180fu, 0x00010100u, 0xbf810000u}; + +constexpr std::uint32_t Width = 64; +constexpr std::uint32_t Height = 32; +alignas(256) std::array Pixels{}; +alignas(256) std::array Output{}; +alignas(256) std::array VertexSlots{}; +alignas(256) std::array InstanceSlots{}; +alignas(256) std::array BufferTable{}; +alignas(256) constexpr std::array Indices{2, 0, 1, 0}; + +std::array SlotDescriptor(const void* data, std::uint32_t records) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), records, 0x01014facu}; +} + +std::array OutputDescriptor() { + const auto address = reinterpret_cast(Output.data()); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), static_cast(sizeof(Output)), 0x31016facu}; +} + +struct Expected { + std::uint32_t vertex; + std::uint32_t instance; + std::uint32_t vertexSum; + std::uint32_t instanceSum; +}; + +class Fixture { +public: + explicit Fixture(std::shared_ptr device) : device(std::move(device)) { + const auto vertex = SlotDescriptor(VertexSlots.data(), static_cast(VertexSlots.size())); + const auto instance = SlotDescriptor(InstanceSlots.data(), static_cast(InstanceSlots.size())); + std::copy(vertex.begin(), vertex.end(), BufferTable.begin()); + std::copy(instance.begin(), instance.end(), BufferTable.begin() + 4); + records = static_cast(::operator new(RecordBytes, std::align_val_t{65536})); + GuestAllocations::Mutation mutation; + mutation.Add(records, RecordBytes, true, true); + } + + std::vector UserData(std::uint32_t baseVertex, std::uint32_t startInstance) const { + std::vector userData(10, 0u); + const auto output = OutputDescriptor(); + std::copy(output.begin(), output.end(), userData.begin()); + const auto table = reinterpret_cast(BufferTable.data()); + userData[4] = static_cast(table); + userData[5] = static_cast(table >> 32u); + userData[BaseVertexSgpr - 8u] = baseVertex; + userData[StartInstanceSgpr - 8u] = startInstance; + return userData; + } + + ShaderRecompiler::RecompileResult Vertex(std::span code, const std::vector& userData) const { + ShaderRecompiler::ShaderVertexStageInfo info{}; + info.resourcesNum = 2; + info.resources[0].fields = {BufferTable[0], BufferTable[1], BufferTable[2], BufferTable[3]}; + info.resources[1].fields = {BufferTable[4], BufferTable[5], BufferTable[6], BufferTable[7]}; + info.resourcesDst[0] = {10, 1, 0, 0}; + info.resourcesDst[1] = {11, 1, 1, 1}; + info.fetchAttribReg = 6; + info.fetchBufferReg = 4; + info.fetchEmbedded = true; + const std::array memory{{ + {reinterpret_cast(code.data()), std::as_bytes(code)}, + {reinterpret_cast(BufferTable.data()), std::as_bytes(std::span(BufferTable))} + }}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Vertex, reinterpret_cast(code.data()), code, 0, {}}, + {32, 8, userData, std::nullopt, std::nullopt, info, memory}, + device->Target(), + {0, 0, 0, 64} + }; + request.useCache = false; + auto result = ShaderRecompiler::Recompile(request); + Require(result.vertexOffsetSgpr == static_cast(BaseVertexSgpr) && result.instanceOffsetSgpr == static_cast(StartInstanceSgpr), "the fetch adds of s16 and s17 were not folded into the draw (vertex " + std::to_string(result.vertexOffsetSgpr) + ", instance " + std::to_string(result.instanceOffsetSgpr) + ")"); + return result; + } + + AgcDriver::Graphics::State State() const { + AgcDriver::Graphics::State state{}; + state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, 32u, 32u, std::nullopt, std::nullopt}; + state.color = {reinterpret_cast(Pixels.data()), {Width, Height}, VK_FORMAT_R8G8B8A8_UNORM, Pixels.size(), 0xe4u}; + state.colors = {state.color}; + state.hasColorTarget = true; + state.renderExtent = {Width, Height}; + state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; + state.viewport = {0.0f, static_cast(Height), static_cast(Width), -static_cast(Height), 0.0f, 1.0f}; + state.scissor = {{0, 0}, {Width, Height}}; + state.cullMode = VK_CULL_MODE_NONE; + state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE; + state.blend.colorWriteMask = 15u; + state.blends = {state.blend}; + return state; + } + + void Draw(const ShaderRecompiler::RecompileResult& vertex, const AgcDriver::Pm4::DrawParameters& draw) { + const auto vertexPush = static_cast(vertex.pushConstants.size()); + ShaderRecompiler::ShaderPixelStageInfo pixelInfo{}; + pixelInfo.wave32 = true; + pixelInfo.targetOutputMode[0] = 9u; + pixelInfo.targetExportMapping.fill(0xe4u); + ShaderRecompiler::RecompileRequest pixelRequest{{ShaderStage::Fragment, reinterpret_cast(PixelCode.data()), PixelCode, 0, {}}, {32u, 0u, {}, std::nullopt, pixelInfo, std::nullopt, {}}, device->Target(), {0u, 0u, vertexPush, 128u - vertexPush}}; + pixelRequest.useCache = false; + const auto pixel = ShaderRecompiler::Recompile(pixelRequest); + const std::array shaders{{{ShaderStage::Vertex, &vertex, 0u}, {ShaderStage::Fragment, &pixel, vertexPush}}}; + Output.fill(Unwritten); + device->Draw(State(), draw, shaders); + device->WaitIdle(); + } + + void Place(std::uint32_t firstVertexFetch, std::span fetchedVertices, std::uint32_t firstInstanceFetch) { + VertexSlots.fill(Unwritten); + InstanceSlots.fill(Unwritten); + for (std::uint32_t vertex = 0; vertex < fetchedVertices.size(); ++vertex) VertexSlots[firstVertexFetch + fetchedVertices[vertex]] = vertex; + for (std::uint32_t instance = 0; instance < Instances; ++instance) InstanceSlots[firstInstanceFetch + instance] = instance * Vertices; + } + + std::string Check(const std::string& what, std::span rawVertices, std::uint32_t baseVertex, std::uint32_t startInstance, bool checkRaw) const { + std::string failures; + for (std::uint32_t instance = 0; instance < Instances; ++instance) { + for (std::uint32_t vertex = 0; vertex < Vertices; ++vertex) { + const auto slot = instance * Vertices + vertex; + const Expected expected{rawVertices[vertex], instance, rawVertices[vertex] + baseVertex, instance + startInstance}; + const auto* actual = Output.data() + slot * 4u; + const bool same = actual[2] == expected.vertexSum && actual[3] == expected.instanceSum && (!checkRaw || (actual[0] == expected.vertex && actual[1] == expected.instance)); + char line[320]; + if (checkRaw) std::snprintf(line, sizeof(line), "%s, vertex %u instance %u: v5 %u v8 %u, after the fetch adds %u %u; expected %u %u, %u %u\n", what.c_str(), vertex, instance, actual[0], actual[1], actual[2], actual[3], expected.vertex, expected.instance, expected.vertexSum, expected.instanceSum); + else std::snprintf(line, sizeof(line), "%s, vertex %u instance %u: after the fetch adds %u %u; expected %u %u\n", what.c_str(), vertex, instance, actual[2], actual[3], expected.vertexSum, expected.instanceSum); + std::cout << line; + if (!same) failures += line; + } + } + return failures; + } + + std::uint32_t* Records() const { + return records; + } + + std::shared_ptr device; + +private: + static constexpr std::size_t RecordBytes = 65536; + std::uint32_t* records = nullptr; +}; + +AgcDriver::DriverDetail::DrawProgram Program(const std::vector& userData) { + AgcDriver::DriverDetail::DrawProgram program{}; + program.firstUserSgpr = 8; + program.userData = userData; + return program; +} + +AgcDriver::Pm4::DrawParameters FoldedDraw(const ShaderRecompiler::RecompileResult& vertex, const std::vector& userData, AgcDriver::Pm4::DrawParameters draw) { + AgcDriver::DriverDetail::Driver::FoldDrawOffsets(vertex, Program(userData), draw); + return draw; +} + +std::string PathName(const std::optional& path) { + return path ? "the CPU path (" + std::string(AgcDriver::Graphics::IndirectDrawPathName(*path)) + ")" : std::string("the GPU"); +} + +std::string Classified(Fixture& fixture, const std::string& what, const ShaderRecompiler::RecompileResult& vertex, const std::vector& userData, AgcDriver::Pm4::DrawParameters& draw, std::optional expectedPath, Rule expectedRule) { + const auto path = AgcDriver::DriverDetail::Driver::ClassifyIndirectDraw(vertex, fixture.State(), Program(userData), fixture.device, draw, false); + const auto& indirect = *draw.indirect; + const bool rules = expectedPath || (indirect.vertexRule == expectedRule && indirect.instanceRule == expectedRule); + std::cout << what << ": classified to " << PathName(path) << '\n'; + if (path == expectedPath && rules) return {}; + return what + ": classified to " + PathName(path) + ", expected " + PathName(expectedPath) + (expectedPath ? "" : expectedRule == Rule::InPlace ? " with both dimensions in place" : " with both dimensions constant") + "\n"; +} + +std::string Analysis(Fixture& fixture) { + struct Case { + const char* what; + std::span code; + bool vertex; + bool instance; + }; + const std::array cases{{ + {"a program that copies v5 and v8 before its fetch adds", VertexCode, true, true}, + {"a program whose fetch adds are the only readers of v5, v8, s16 and s17", PostAddCode, false, false}, + {"a program that reads the instance add's carry", CarryCode, false, true}, + {"a program whose vertex add is a clamped v_sad_u32", ClampCode, true, false}, + {"a program that adds s16 twice", TwiceCode, true, false}, + {"a program that writes EXEC before its fetch adds", ExecCode, true, true}, + {"a program that writes another add's carry to s17 before its fetch adds", CarryWriteCode, false, true}, + {"a program whose s16 add writes v0, not v5", OtherVgprCode, true, false}, + }}; + const auto userData = fixture.UserData(0x51u, 0x62u); + std::string failures; + for (const auto& entry : cases) { + const auto result = fixture.Vertex(entry.code, userData); + char line[256]; + std::snprintf(line, sizeof(line), "%s: folded value observable outside the fetch add: vertex %d instance %d; expected %d %d\n", entry.what, result.vertexOffsetShared ? 1 : 0, result.instanceOffsetShared ? 1 : 0, entry.vertex ? 1 : 0, entry.instance ? 1 : 0); + std::cout << line; + if (result.vertexOffsetShared != entry.vertex || result.instanceOffsetShared != entry.instance) failures += line; + } + return failures; +} + +std::string Run(Fixture& fixture) { + std::string failures = Analysis(fixture); + const std::array autoIndices{0, 1, 2}; + const std::array bufferIndices{Indices[0], Indices[1], Indices[2]}; + const auto indexAddress = reinterpret_cast(Indices.data()); + + { + constexpr std::uint32_t indxOffset = 5; + constexpr std::uint32_t baseVertex = 1000; + constexpr std::uint32_t startInstance = 40; + const auto userData = fixture.UserData(baseVertex, startInstance); + const auto vertex = fixture.Vertex(VertexCode, userData); + const auto draw = FoldedDraw(vertex, userData, {0, Vertices, 0, Instances, 0, false, indxOffset, 0}); + fixture.Place(indxOffset + baseVertex, autoIndices, startInstance); + fixture.Draw(vertex, draw); + const std::array raw{indxOffset, indxOffset + 1, indxOffset + 2}; + failures += fixture.Check("direct auto draw, GE_INDX_OFFSET 5, s16 1000, s17 40", raw, baseVertex, startInstance, true); + } + { + constexpr std::uint32_t baseVertex = 1000; + constexpr std::uint32_t startInstance = 40; + const auto userData = fixture.UserData(baseVertex, startInstance); + const auto vertex = fixture.Vertex(VertexCode, userData); + const auto draw = FoldedDraw(vertex, userData, {indexAddress, Vertices, 2, Instances, 0, true, 0, 0}); + fixture.Place(baseVertex, bufferIndices, startInstance); + fixture.Draw(vertex, draw); + failures += fixture.Check("direct indexed draw, s16 1000, s17 40", bufferIndices, baseVertex, startInstance, true); + } + + const auto staleUserData = fixture.UserData(0x51u, 0x62u); + const std::array indexedRecord{Vertices, Instances, 0, 700, 59455}; + std::copy(indexedRecord.begin(), indexedRecord.end(), fixture.Records()); + AgcDriver::Pm4::DrawParameters::IndirectDraw indexedIndirect{reinterpret_cast(fixture.Records()), 0x25u, 20u, 20u, 1u, false, 0u, 0x10u, 0x11u, NoLocation, false, 0u}; + indexedIndirect.baseVertexSgpr = static_cast(BaseVertexSgpr); + indexedIndirect.startInstanceSgpr = static_cast(StartInstanceSgpr); + { + const std::string what = "indexed indirect record whose program copies v5/v8 before its fetch adds, vertex offset 700, start instance 59455"; + AgcDriver::Pm4::DrawParameters packet{indexAddress, static_cast(Indices.size()), 2, 0, 0, true, 0, 0}; + packet.indirect = indexedIndirect; + failures += Classified(fixture, what, fixture.Vertex(VertexCode, staleUserData), staleUserData, packet, Path::NotFolded, Rule::Constant); + const auto arguments = AgcDriver::Pm4::ReadDrawArguments(indexedIndirect, 0); + const auto userData = fixture.UserData(arguments.vertexOffset, arguments.firstInstance); + const auto vertex = fixture.Vertex(VertexCode, userData); + const auto draw = FoldedDraw(vertex, userData, {indexAddress + arguments.firstVertexOrIndex * 2u, arguments.count, 2, arguments.instances, 0, true, 0, 0}); + fixture.Place(arguments.vertexOffset, bufferIndices, arguments.firstInstance); + fixture.Draw(vertex, draw); + failures += fixture.Check(what + ", on the CPU path", bufferIndices, arguments.vertexOffset, arguments.firstInstance, true); + } + { + const std::string what = "auto indirect record whose program copies v5/v8 before its fetch adds, start vertex 300, start instance 7"; + const std::array record{Vertices, Instances, 300, 7}; + std::copy(record.begin(), record.end(), fixture.Records() + 16); + AgcDriver::Pm4::DrawParameters::IndirectDraw indirect{reinterpret_cast(fixture.Records() + 16), 0x24u, 16u, 16u, 1u, false, 0u, 0x10u, 0x11u, NoLocation, false, 0u}; + indirect.baseVertexSgpr = static_cast(BaseVertexSgpr); + indirect.startInstanceSgpr = static_cast(StartInstanceSgpr); + AgcDriver::Pm4::DrawParameters packet{0, 0, 0, 0, 0, false, 0, 0}; + packet.indirect = indirect; + failures += Classified(fixture, what, fixture.Vertex(VertexCode, staleUserData), staleUserData, packet, Path::NotFolded, Rule::Constant); + const auto arguments = AgcDriver::Pm4::ReadDrawArguments(indirect, 0); + const auto userData = fixture.UserData(arguments.firstVertexOrIndex, arguments.firstInstance); + const auto vertex = fixture.Vertex(VertexCode, userData); + const auto draw = FoldedDraw(vertex, userData, {0, arguments.count, 0, arguments.instances, 0, false, indirect.indxOffset, 0}); + fixture.Place(arguments.firstVertexOrIndex, autoIndices, arguments.firstInstance); + fixture.Draw(vertex, draw); + failures += fixture.Check(what + ", on the CPU path", autoIndices, arguments.firstVertexOrIndex, arguments.firstInstance, true); + } + if (fixture.device->DrawIndirectSupport().firstInstance) { + { + const std::string what = "indexed indirect record whose fetch adds are the only readers of v5/v8, user data s16 0x51, s17 0x62"; + const auto vertex = fixture.Vertex(PostAddCode, staleUserData); + AgcDriver::Pm4::DrawParameters draw{indexAddress, static_cast(Indices.size()), 2, 0, 0, true, 0, 0}; + draw.indirect = indexedIndirect; + failures += Classified(fixture, what, vertex, staleUserData, draw, std::nullopt, Rule::InPlace); + fixture.Place(indexedRecord[3], bufferIndices, indexedRecord[4]); + fixture.Draw(vertex, draw); + failures += fixture.Check(what + ", drawn in place on the GPU", bufferIndices, indexedRecord[3], indexedRecord[4], false); + } + { + const std::string what = "indexed indirect record whose packet names no register for its offsets, user data s16 0x51, s17 0x62"; + const auto vertex = fixture.Vertex(VertexCode, staleUserData); + AgcDriver::Pm4::DrawParameters draw{indexAddress, static_cast(Indices.size()), 2, 0, 0, true, 0, 0}; + draw.indirect = indexedIndirect; + draw.indirect->baseVertexLocation = NoLocation; + draw.indirect->startInstanceLocation = NoLocation; + draw.indirect->baseVertexSgpr = -1; + draw.indirect->startInstanceSgpr = -1; + failures += Classified(fixture, what, vertex, staleUserData, draw, std::nullopt, Rule::Constant); + fixture.Place(0x51u, bufferIndices, 0x62u); + fixture.Draw(vertex, draw); + failures += fixture.Check(what + ", drawn on the GPU", bufferIndices, 0x51u, 0x62u, true); + } + } + return failures; +} + +} + +int main() { + try { + std::shared_ptr device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Fixture fixture(device); + const auto failures = Run(fixture); + if (!failures.empty()) { + std::cerr << "vertex and instance indices differ from the guest's:\n" << failures; + return 1; + } + std::cout << "draw offset index tests passed\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/shader/recompiler/Translation/include/Translation/EmbeddedVertexFetch.hpp b/core/shader/recompiler/Translation/include/Translation/EmbeddedVertexFetch.hpp index 238056cd6..9729baeeb 100644 --- a/core/shader/recompiler/Translation/include/Translation/EmbeddedVertexFetch.hpp +++ b/core/shader/recompiler/Translation/include/Translation/EmbeddedVertexFetch.hpp @@ -24,6 +24,8 @@ struct EmbeddedFetchPlan { bool instanceOffsetShared = false; bool vertexOffsetConflict = false; bool instanceOffsetConflict = false; + bool vertexIndexObserved = false; + bool instanceIndexObserved = false; }; class EmbeddedVertexFetchAnalyzer { diff --git a/core/shader/recompiler/Translation/src/EmbeddedVertexFetch.cpp b/core/shader/recompiler/Translation/src/EmbeddedVertexFetch.cpp index 4fda5df13..105b110f6 100644 --- a/core/shader/recompiler/Translation/src/EmbeddedVertexFetch.cpp +++ b/core/shader/recompiler/Translation/src/EmbeddedVertexFetch.cpp @@ -208,6 +208,159 @@ bool touchesSgpr(const RdnaInstruction& inst, std::uint32_t sgpr) { return false; } +bool isVectorAlu(const RdnaInstruction& inst) { + switch (inst.family) { + case RdnaInstructionFamily::VOP1: + case RdnaInstructionFamily::VOP2: + case RdnaInstructionFamily::VOP3: + case RdnaInstructionFamily::VOP3P: + case RdnaInstructionFamily::VOPC: + return true; + default: + return false; + } +} + +bool isSequentialProgramControl(RdnaOpcode opcode) { + switch (opcode) { + case RdnaOpcode::SNop: + case RdnaOpcode::SWaitcnt: + case RdnaOpcode::SWaitcntDepctr: + case RdnaOpcode::SSetprio: + case RdnaOpcode::SSendmsg: + case RdnaOpcode::SBarrier: + case RdnaOpcode::SSleep: + case RdnaOpcode::SClause: + case RdnaOpcode::SInstPrefetch: + case RdnaOpcode::SRoundMode: + case RdnaOpcode::SDenormMode: + case RdnaOpcode::SIcacheInv: + case RdnaOpcode::STtracedata: + case RdnaOpcode::SIncperflevel: + case RdnaOpcode::SDecperflevel: + return true; + default: + return false; + } +} + +bool leavesStraightLine(const RdnaInstruction& inst) { + if (inst.op == RdnaOpcode::Unknown || inst.op == RdnaOpcode::SSetpcB64 || inst.op == RdnaOpcode::SRfeB64) return true; + return inst.family == RdnaInstructionFamily::SOPP && !isSequentialProgramControl(inst.op); +} + +bool readsSgprRange(const RdnaInstruction& inst, std::uint32_t first, std::uint32_t count) { + const std::array sources{&inst.source0, &inst.source1, &inst.source2, &inst.source3}; + for (std::uint32_t i = 0u; i < sources.size(); i++) { + if (!isScalarOperand(*sources[i])) continue; + const auto slot = scalarSlot(*sources[i]); + if (slot < first + count && first < slot + scalarOperandWidth(inst, i)) return true; + } + return false; +} + +bool readsVccImplicitly(const RdnaInstruction& inst) { + switch (inst.op) { + case RdnaOpcode::VCndmaskB32: + case RdnaOpcode::VAddcU32: + case RdnaOpcode::VSubCoCiU32: + case RdnaOpcode::VSubrevCoCiU32: + return !isScalarOperand(inst.source2); + case RdnaOpcode::VDivFmasF32: + case RdnaOpcode::VDivFmasF64: + return true; + default: + return false; + } +} + +bool mayReachVgpr(const RdnaInstruction& inst, std::uint32_t reg) { + switch (inst.family) { + case RdnaInstructionFamily::SOP1: + case RdnaInstructionFamily::SOP2: + case RdnaInstructionFamily::SOPK: + case RdnaInstructionFamily::SOPC: + case RdnaInstructionFamily::SMEM: + case RdnaInstructionFamily::SOPP: + return leavesStraightLine(inst); + default: + break; + } + if (!isVectorAlu(inst)) return true; + switch (inst.op) { + case RdnaOpcode::VMovrelsB32: + case RdnaOpcode::VMovreldB32: + case RdnaOpcode::VMovrelsdB32: + case RdnaOpcode::VMovrelsd2B32: + case RdnaOpcode::VSwaprelB32: + return true; + default: + break; + } + const std::array sources{&inst.source0, &inst.source1, &inst.source2, &inst.source3}; + for (std::uint32_t i = 0u; i < sources.size(); i++) { + const auto width = inst.op == RdnaOpcode::VMqsadU32U8 && i == 2u ? 4u : 2u; + if (isVectorOperand(*sources[i]) && reg >= sources[i]->reg && reg < sources[i]->reg + width) return true; + } + return isVectorOperand(inst.destination) && reg >= inst.destination.reg && reg < inst.destination.reg + std::max(embeddedFetchDstSize(inst), 2u); +} + +bool writesExec(const RdnaInstruction& inst) { + const auto exec = [](const RdnaOperand& operand) { return operand.kind == RdnaOperandKind::ExecLo || operand.kind == RdnaOperandKind::ExecHi; }; + if (exec(inst.destination) || exec(inst.destination2)) return true; + switch (inst.op) { + case RdnaOpcode::SAndSaveexecB32: + case RdnaOpcode::SOrSaveexecB32: + case RdnaOpcode::SXorSaveexecB32: + case RdnaOpcode::SAndn2SaveexecB32: + case RdnaOpcode::SOrn2SaveexecB32: + case RdnaOpcode::SNandSaveexecB32: + case RdnaOpcode::SNorSaveexecB32: + case RdnaOpcode::SXnorSaveexecB32: + case RdnaOpcode::SAndn1SaveexecB32: + case RdnaOpcode::SOrn1SaveexecB32: + case RdnaOpcode::SAndn1WrexecB32: + case RdnaOpcode::SAndn2WrexecB32: + case RdnaOpcode::SAndSaveexecB64: + case RdnaOpcode::SOrSaveexecB64: + case RdnaOpcode::SXorSaveexecB64: + case RdnaOpcode::SAndn2SaveexecB64: + case RdnaOpcode::SOrn2SaveexecB64: + case RdnaOpcode::SNandSaveexecB64: + case RdnaOpcode::SNorSaveexecB64: + case RdnaOpcode::SXnorSaveexecB64: + case RdnaOpcode::SAndn1SaveexecB64: + case RdnaOpcode::SOrn1SaveexecB64: + case RdnaOpcode::SAndn1WrexecB64: + case RdnaOpcode::SAndn2WrexecB64: + return true; + default: + return false; + } +} + +bool writesSgprMask(const RdnaInstruction& inst, std::uint32_t sgpr, std::uint32_t waveSize) { + const auto covers = [&](const RdnaOperand& operand, std::uint32_t width) { return isScalarOperand(operand) && sgpr >= scalarSlot(operand) && sgpr < scalarSlot(operand) + width; }; + return covers(inst.destination2, 2u) || (isVectorAlu(inst) && covers(inst.destination, waveSize == 64u ? 2u : 1u)); +} + +bool carryObserved(const RdnaProgram& program, std::size_t addIndex, std::uint32_t waveSize) { + const auto& add = program.instructions[addIndex]; + if (add.destination2.kind == RdnaOperandKind::None || add.destination2.kind == RdnaOperandKind::Null) return false; + if (!isScalarOperand(add.destination2)) return true; + const auto first = scalarSlot(add.destination2); + const std::uint32_t count = waveSize == 64u ? 2u : 1u; + for (std::size_t index = addIndex + 1u; index < program.instructions.size(); index++) { + const auto& inst = program.instructions[index]; + if (inst.op == RdnaOpcode::SEndpgm) return false; + if (leavesStraightLine(inst) || readsSgprRange(inst, first, count) || (first == kVccLoSlot && readsVccImplicitly(inst))) return true; + const bool laneMask = isVectorAlu(inst) && inst.op != RdnaOpcode::VReadlaneB32 && inst.op != RdnaOpcode::VReadfirstlaneB32; + const auto rewrites = [&](const RdnaOperand& operand) { return isScalarOperand(operand) && scalarSlot(operand) == first && (count == 1u || laneMask); }; + if (rewrites(inst.destination) || rewrites(inst.destination2)) return false; + } + return false; +} + } EmbeddedFetchPlan EmbeddedVertexFetchAnalyzer::Analyze(const RdnaProgram& program, std::uint32_t attributeTableRegister, std::uint32_t bufferTableRegister, std::uint32_t userDataBaseRegister, std::uint32_t userDataCount, std::uint32_t waveSize) const { @@ -447,6 +600,21 @@ EmbeddedFetchPlan EmbeddedVertexFetchAnalyzer::Analyze(const RdnaProgram& progra }; plan.vertexOffsetShared = shared(plan.vertexOffsetSgpr, vertexAddPcs); plan.instanceOffsetShared = shared(plan.instanceOffsetSgpr, instanceAddPcs); + const bool relativeScalarAccess = std::any_of(program.instructions.begin(), program.instructions.end(), [](const RdnaInstruction& inst) { + return inst.op == RdnaOpcode::SMovrelsB32 || inst.op == RdnaOpcode::SMovrelsB64 || inst.op == RdnaOpcode::SMovrelsd2B32 || inst.op == RdnaOpcode::SMovreldB32 || inst.op == RdnaOpcode::SMovreldB64; + }); + const auto observed = [&](std::int32_t sgpr, const std::vector& addPcs, std::uint32_t indexVgpr) { + if (sgpr < 0) return false; + if (addPcs.size() != 1u || relativeScalarAccess) return true; + const auto add = std::find_if(program.instructions.begin(), program.instructions.end(), [&](const RdnaInstruction& inst) { return inst.programCounter == addPcs.front(); }); + if (add == program.instructions.end() || add->destination.reg != indexVgpr || add->destination.clamp || add->clampResult) return true; + if (std::any_of(program.instructions.begin(), program.instructions.end(), [&](const RdnaInstruction& inst) { return inst.programCounter != add->programCounter && writesSgprMask(inst, static_cast(sgpr), waveSize); })) return true; + const auto reg = add->destination.reg; + if (std::any_of(program.instructions.begin(), add, [&](const RdnaInstruction& inst) { return writesExec(inst) || mayReachVgpr(inst, reg); })) return true; + return carryObserved(program, static_cast(add - program.instructions.begin()), waveSize); + }; + plan.vertexIndexObserved = observed(plan.vertexOffsetSgpr, vertexAddPcs, kVertexIndexVgpr); + plan.instanceIndexObserved = observed(plan.instanceOffsetSgpr, instanceAddPcs, kInstanceIndexVgpr); return plan; } diff --git a/core/shader/recompiler/Translation/src/InstructionTranslator.cpp b/core/shader/recompiler/Translation/src/InstructionTranslator.cpp index 2f5a7d6c5..d65af4c67 100644 --- a/core/shader/recompiler/Translation/src/InstructionTranslator.cpp +++ b/core/shader/recompiler/Translation/src/InstructionTranslator.cpp @@ -164,6 +164,12 @@ void emitEntryPrologue(IrProgram& program, IrBlock& entryBlock, const TranslateO const auto builtin = [&entryIr](StageInputKind kind, std::uint32_t component = 0u) -> IrValue& { return entryIr.Emit(IrOpcode::GetBuiltin, IrOpcodeType(IrOpcode::GetBuiltin), {&entryIr.Constant(static_cast(kind)), &entryIr.Constant(component)}); }; + const auto drawIndex = [&entryIr, &options, &builtin](StageInputKind kind) -> IrValue& { + const auto* fetch = options.embeddedFetch; + const std::int32_t folded = fetch == nullptr ? -1 : kind == StageInputKind::VertexIndex ? fetch->vertexOffsetSgpr : fetch->instanceOffsetSgpr; + IrValue& index = builtin(kind); + return folded < 0 ? index : entryIr.ISub(index, entryIr.GetUserData(static_cast(folded))); + }; for (std::uint32_t index = 0; index < options.userDataCount; index++) { const auto reg = static_cast(options.userDataBaseRegister + index); @@ -288,9 +294,9 @@ void emitEntryPrologue(IrProgram& program, IrBlock& entryBlock, const TranslateO entryIr.SetVectorReg(static_cast(8), entryIr.IAdd(draw(2u), builtin(StageInputKind::WorkgroupId, 1u))); } else if (options.stage == ShaderStageKind::Local) { entryIr.SetScalarReg(static_cast(3), entryIr.Constant(64u)); - entryIr.SetVectorReg(static_cast(2), builtin(StageInputKind::VertexIndex)); + entryIr.SetVectorReg(static_cast(2), drawIndex(StageInputKind::VertexIndex)); entryIr.SetVectorReg(static_cast(3), entryIr.Constant(0u)); - entryIr.SetVectorReg(static_cast(5), builtin(StageInputKind::InstanceIndex)); + entryIr.SetVectorReg(static_cast(5), drawIndex(StageInputKind::InstanceIndex)); } else if (options.stage == ShaderStageKind::TessellationControl) { const auto& tess = options.inputInfo.vertex->tess; entryIr.SetScalarReg(static_cast(2), entryIr.Emit(IrOpcode::TessellationBase, IrOpcodeType(IrOpcode::TessellationBase), {&entryIr.Constant(0u)})); @@ -341,8 +347,8 @@ void emitEntryPrologue(IrProgram& program, IrBlock& entryBlock, const TranslateO if (options.userDataBaseRegister >= 8u) { entryIr.SetScalarReg(static_cast(3), entryIr.Constant(options.waveSize | (options.waveSize << 8u))); } - entryIr.SetVectorReg(static_cast(5), builtin(StageInputKind::VertexIndex)); - entryIr.SetVectorReg(static_cast(8), builtin(StageInputKind::InstanceIndex)); + entryIr.SetVectorReg(static_cast(5), drawIndex(StageInputKind::VertexIndex)); + entryIr.SetVectorReg(static_cast(8), drawIndex(StageInputKind::InstanceIndex)); } } @@ -378,8 +384,8 @@ IrProgram InstructionTranslator::Translate(const RdnaProgram& decoded, const Con if (options.embeddedFetch != nullptr) { program.Info().vertexOffsetSgpr = options.embeddedFetch->vertexOffsetSgpr; program.Info().instanceOffsetSgpr = options.embeddedFetch->instanceOffsetSgpr; - program.Info().vertexOffsetShared = options.embeddedFetch->vertexOffsetShared; - program.Info().instanceOffsetShared = options.embeddedFetch->instanceOffsetShared; + program.Info().vertexOffsetShared = options.embeddedFetch->vertexOffsetShared || options.embeddedFetch->vertexIndexObserved; + program.Info().instanceOffsetShared = options.embeddedFetch->instanceOffsetShared || options.embeddedFetch->instanceIndexObserved; program.Info().vertexOffsetConflict = options.embeddedFetch->vertexOffsetConflict; program.Info().instanceOffsetConflict = options.embeddedFetch->instanceOffsetConflict; }