diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 20c42fc92..73a58caaf 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -1083,6 +1083,12 @@ if(BUILD_TESTING) endif() endforeach() endforeach() + add_test(NAME agc_vector_window_stage_inputs COMMAND agc_vector_window_tests --stage-inputs-only) + set_tests_properties(agc_vector_window_stage_inputs PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + set_tests_properties(agc_vector_window_stage_inputs agc_driver_relative_stage_inputs PROPERTIES TIMEOUT 30) + if(WIN32) + set_tests_properties(agc_vector_window_stage_inputs PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") + endif() add_test(NAME agc_driver_f64_denorm_preserve_spirv COMMAND agc_driver_f64_fma_subnormals_tests --verify-spv-denorm-mode) set_tests_properties(agc_driver_f64_denorm_preserve_spirv PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") if(WIN32) diff --git a/core/libs/prx/libSceAgcDriver/tests/VectorWindow.cpp b/core/libs/prx/libSceAgcDriver/tests/VectorWindow.cpp index 3e852184b..7f622c956 100644 --- a/core/libs/prx/libSceAgcDriver/tests/VectorWindow.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/VectorWindow.cpp @@ -1,9 +1,13 @@ #include "Recompiler.hpp" +#include "ControlFlow/GraphBuilder.hpp" +#include "RdnaDecoder/RdnaInstructionDecoder.hpp" +#include "Translation/InstructionTranslator.hpp" #include #include #include #include #include +#include #include #include #include @@ -15,11 +19,12 @@ namespace { using namespace ShaderRecompiler; constexpr std::uint32_t MoveM0Two = 0xbefc0382u; +constexpr std::uint32_t MoveM0GroupX = 0xbefc0304u; constexpr std::uint32_t MoveV4One = 0x7e0802f2u; constexpr std::uint32_t MoveV5Two = 0x7e0a02f4u; constexpr std::uint32_t MoveV6Four = 0x7e0c02f6u; constexpr std::uint32_t MovrelsV1V4 = 0x7e028704u; -constexpr std::uint32_t MovreldV4V1 = 0x7e088501u; +constexpr std::uint32_t MovreldV4V0 = 0x7e088500u; constexpr std::uint32_t MoveV1V5 = 0x7e020305u; constexpr std::uint32_t EndProgram = 0xbf810000u; @@ -36,14 +41,14 @@ void Require(bool condition, const std::string& message) { std::vector RecompileProgram(std::span code) { const std::array capabilities{spv::CapabilityShader, spv::CapabilityGroupNonUniform, spv::CapabilityGroupNonUniformBallot, spv::CapabilityGroupNonUniformVote, spv::CapabilityGroupNonUniformShuffle, spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses}; - const std::array userData{}; + const std::array userData{0x10000000u, 0u, 0x40u, 0x31016facu}; const std::array extensions{"SPV_KHR_physical_storage_buffer", "SPV_KHR_8bit_storage"}; RecompileRequest request{}; request.shader = {ShaderStage::Compute, 0x20000u, code, 0, {}}; request.context.waveSize = 32; request.context.userDataBaseRegister = 0; request.context.userData = userData; - request.context.compute = ShaderComputeStageInfo{{64u, 1u, 1u}, 4u, {false, false, false}, false, 1u}; + request.context.compute = ShaderComputeStageInfo{{64u, 1u, 1u}, 4u, {true, false, false}, false, 1u}; request.target.vulkanVersion = 0x00403000u; request.target.spirvVersion = 0x00010600u; request.target.subgroupSize = 32; @@ -65,6 +70,7 @@ Shape Scan(std::span words) { std::vector> constants; std::vector> arrays; std::vector> functionPointers; + std::vector windowVariables; for (std::size_t cursor = 5; cursor < words.size();) { const auto count = words[cursor] >> spv::WordCountShift; const auto op = static_cast(words[cursor] & spv::OpCodeMask); @@ -78,19 +84,162 @@ Shape Scan(std::span words) { for (const auto& [array, length] : arrays) { if (array != pointee) continue; ++shape.functionArrays; + windowVariables.push_back(words[cursor + 2]); for (const auto& [constant, value] : constants) { if (constant == length) shape.arrayLength = value; } } } } - if (op == spv::OpAccessChain) ++shape.accessChains; + if (op == spv::OpAccessChain) { + for (const auto variable : windowVariables) { + if (words[cursor + 3] == variable) ++shape.accessChains; + } + } if (op == spv::OpSelect) ++shape.selects; cursor += count; } return shape; } +IrProgram TranslateStage(ShaderStageKind stage, std::uint32_t waveSize, std::uint32_t base = 0u, std::uint32_t fetchStart = 0u, std::uint32_t fetchCount = 0u) { + ShaderComputeInputInfo compute{}; + compute.threadIdsNum = 3; + compute.threadsNum[0] = waveSize; + ShaderVertexInputInfo vertex{}; + vertex.mesh.threadsNum[0] = waveSize; + vertex.mesh.inputPrimitive = 1u; + vertex.mesh.primitivesPerGroup = 1u; + vertex.mesh.verticesPerGroup = 1u; + vertex.mesh.esgsItemSize = 1u; + ShaderPixelInputInfo pixel{}; + std::uint32_t next = 0u; + for (const auto input : {PixelInput::PerspectiveCenter, PixelInput::PerspectiveCentroid, PixelInput::LinearCenter, PixelInput::LinearCentroid, + PixelInput::PositionX, PixelInput::PositionY, PixelInput::PositionZ, PixelInput::PositionW, + PixelInput::FrontFace, PixelInput::Ancillary, PixelInput::LineStipple, PixelInput::PositionFixedPoint}) { + pixel.psInputVgpr[static_cast(input)] = next; + next += PixelInputVgprCount(input); + } + std::vector code{MoveM0Two}; + if (fetchCount != 0u) { + code.push_back(0xbe802102u); + code.push_back(0x7e008700u | base); + code.push_back(0x7e1002f2u); + vertex.resourcesNum = 1; + vertex.resourcesDst[0] = {static_cast(fetchStart), static_cast(fetchCount), 0, 0u}; + } + code.push_back(0x7e008700u | base); + code.push_back(EndProgram); + const auto decoded = RdnaInstructionDecoder{}.Decode(code); + if (fetchCount != 0u) { + const auto& call = decoded.instructions.at(1); + Require(call.op == RdnaOpcode::SSwappcB64 && call.family == RdnaInstructionFamily::SOP1 && call.opcodeId == 0x21u && + call.destination.kind == RdnaOperandKind::ScalarRegister && call.destination.reg == 0u && + call.source0.kind == RdnaOperandKind::ScalarRegister && call.source0.reg == 2u, + "external fetch fixture did not decode s_swappc_b64 s[0:1], s[2:3]"); + } + const SwappcInfo swappc{fetchCount != 0u, 0u, 8u}; + const auto graph = GraphBuilder{}.Build(decoded, &swappc); + Require(graph.hasFetchCall == (fetchCount != 0u), "stage fixture did not identify its external fetch call"); + TranslateOptions options{}; + options.stage = stage; + options.waveSize = waveSize; + options.userDataCount = 8u; + options.inputInfo = {&vertex, &pixel, &compute}; + return InstructionTranslator{}.Translate(decoded, graph, options); +} + +void RequireWindow(const IrProgram& program, std::uint32_t base, std::uint32_t limit, std::span initialized) { + Require(program.Metadata().vectorWindowBase == base && program.Metadata().vectorWindowSize == limit - base, "relative window does not include exactly the initialized stage registers"); + std::array stored{}; + for (const auto* instruction : program.BlockOrder().front()->Instructions()) { + if (instruction->Opcode() == IrOpcode::StoreVectorWindow) { + Require(instruction->Argument(0)->HasImmediate(), "stage input uses a dynamic window index"); + const auto index = base + instruction->Argument(0)->ImmediateU32(); + Require(index < limit, "stage input lies past the window"); + stored[index] = true; + } else if (instruction->Opcode() == IrOpcode::SetVectorRegister) { + const auto index = instruction->Argument(0)->Register().index; + Require(index < base, "stage input escaped the relative window into SSA"); + stored[index] = true; + } + } + for (const auto index : initialized) Require(stored[index], "stage input v" + std::to_string(index) + " was not initialized"); + bool bounded = false; + for (const auto& block : program.Blocks()) { + for (const auto* instruction : block->Instructions()) { + if (instruction->Opcode() == IrOpcode::ULessThan32 && instruction->Argument(1)->HasImmediate() && instruction->Argument(1)->ImmediateU32() == limit - base) bounded = true; + } + } + Require(bounded, "relative access did not use the final stage window bound"); +} + +template +bool RunStage(const char* name, TAction action) { + try { + action(); + std::printf("vector window stage inputs: %s passed\n", name); + return true; + } catch (const std::exception& error) { + std::fprintf(stderr, "vector window stage inputs: %s: %s\n", name, error.what()); + return false; + } +} + +bool StageInputs() { + bool passed = true; + const auto stage = [&](const char* name, ShaderStageKind kind, std::uint32_t limit, std::initializer_list initialized) { + passed &= RunStage(name, [&] { + for (const auto wave : {32u, 64u}) RequireWindow(TranslateStage(kind, wave), 0u, limit, {initialized.begin(), initialized.size()}); + }); + }; + stage("compute", ShaderStageKind::Compute, 3u, {0u, 1u, 2u}); + stage("vertex", ShaderStageKind::Vertex, 9u, {5u, 8u}); + stage("local", ShaderStageKind::Local, 6u, {2u, 3u, 5u}); + stage("tessellation control", ShaderStageKind::TessellationControl, 2u, {0u, 1u}); + stage("tessellation evaluation", ShaderStageKind::TessellationEvaluation, 9u, {0u, 5u, 6u, 7u, 8u}); + stage("mesh", ShaderStageKind::Mesh, 9u, {0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u}); + stage("pixel", ShaderStageKind::Pixel, 16u, {0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u, 14u, 15u}); + passed &= RunStage("SSA prefix", [] { + const std::array local{2u, 3u, 5u}; + const std::array vertex{5u, 8u}; + for (const auto wave : {32u, 64u}) { + RequireWindow(TranslateStage(ShaderStageKind::Local, wave, 4u), 4u, 6u, local); + RequireWindow(TranslateStage(ShaderStageKind::Vertex, wave, 16u), 16u, 17u, vertex); + } + }); + passed &= RunStage("external fetch", [] { + const std::array vertex{5u, 8u}; + for (const auto wave : {32u, 64u}) { + for (const auto start : {20u, 252u}) { + for (const auto count : {1u, 2u, 3u, 4u}) { + const auto program = TranslateStage(ShaderStageKind::Vertex, wave, 0u, start, count); + RequireWindow(program, 0u, start + count, vertex); + std::uint32_t fetched = 0u; + std::uint32_t boundedReads = 0u; + for (const auto& block : program.Blocks()) { + for (const auto* instruction : block->Instructions()) { + if (instruction->Opcode() == IrOpcode::StoreVectorWindow) { + const auto* value = instruction->Argument(1); + if (value->Opcode() == IrOpcode::SelectU32) value = value->Argument(1); + if (value->Opcode() == IrOpcode::GetAttribute) { + Require(instruction->Argument(0)->ImmediateU32() == start + fetched && value->Argument(1)->ImmediateU32() == fetched, + "external fetch did not initialize its entire window span"); + ++fetched; + } + } + if (instruction->Opcode() == IrOpcode::ULessThan32 && instruction->Argument(1)->HasImmediate() && instruction->Argument(1)->ImmediateU32() == start + count) ++boundedReads; + } + } + Require(fetched == count && boundedReads == 2u, "relative accesses to the external fetch disagree about the window"); + } + } + } + }); + if (passed) std::puts("Vector window stage input and fetch tests passed"); + return passed; +} + bool run(const char* name, bool (*test)()) { try { if (test()) return true; @@ -103,21 +252,25 @@ bool run(const char* name, bool (*test)()) { } -int main() { +int main(int argc, char** argv) { + if (argc == 2 && std::string_view(argv[1]) == "--stage-inputs-only") return StageInputs() ? 0 : 1; + if (argc != 1) return 2; bool passed = true; - passed &= run("a relative read and write index a function-scope window instead of selecting over the registers", [] { - const std::array code{MoveM0Two, MoveV4One, MoveV5Two, MoveV6Four, MovrelsV1V4, MovreldV4V1, MoveV1V5, EndProgram}; + passed &= run("observable relative reads and writes index a function-scope window", [] { + const std::array code{MoveM0GroupX, MoveV4One, MoveV5Two, MoveV6Four, MovrelsV1V4, + 0xe0700000u, 0x80000100u, MovreldV4V0, MoveV1V5, 0xe0700004u, 0x80000100u, EndProgram}; const auto shape = Scan(RecompileProgram(code)); - if (shape.functionArrays != 1 || shape.arrayLength != 3 || shape.accessChains < 2 || shape.selects > 4) std::fprintf(stderr, " window shape: arrays %zu length %u access chains %zu selects %zu\n", shape.functionArrays, shape.arrayLength, shape.accessChains, shape.selects); - return shape.functionArrays == 1 && shape.arrayLength == 3 && shape.accessChains >= 2 && shape.selects <= 4; + if (shape.functionArrays != 1 || shape.arrayLength != 3 || shape.accessChains < 2) std::fprintf(stderr, " window shape: arrays %zu length %u access chains %zu selects %zu\n", shape.functionArrays, shape.arrayLength, shape.accessChains, shape.selects); + return shape.functionArrays == 1 && shape.arrayLength == 3 && shape.accessChains >= 2; }); passed &= run("a program without relative addressing keeps its registers in SSA values", [] { - const std::array code{MoveV4One, MoveV5Two, MoveV6Four, MoveV1V5, EndProgram}; + const std::array code{MoveV4One, MoveV5Two, MoveV6Four, MoveV1V5, 0xe0700000u, 0x80000100u, EndProgram}; const auto shape = Scan(RecompileProgram(code)); return shape.functionArrays == 0 && shape.accessChains == 0; }); + if (passed) std::puts("Vector window shape and SSA control tests passed"); return passed ? 0 : 1; } diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/RelativeStageInputs.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/RelativeStageInputs.cpp new file mode 100644 index 000000000..ae310f425 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/RelativeStageInputs.cpp @@ -0,0 +1,148 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Extent = 64u; +alignas(256) std::array pixels{}; +alignas(256) std::array output{}; +alignas(256) constexpr std::array, 6> positions{{ + {0.0f, 0.0f, 0.0f, 1.0f}, {0.0f, 0.0f, 0.0f, 1.0f}, {0.0f, 0.0f, 0.0f, 1.0f}, + {-1.0f, -1.0f, 0.5f, 1.0f}, {3.0f, -1.0f, 0.5f, 1.0f}, {-1.0f, 3.0f, 0.5f, 1.0f} +}}; + +constexpr std::array relativeVertexCode{ + 0xbefc0385u, 0x7e028700u, 0xbefc0388u, 0x7e008700u, 0x4a000300u, + 0xe0382000u, 0x80000000u, 0xbf8c3f70u, 0xf80008cfu, 0x03020100u, 0xbf810000u +}; +constexpr std::array vertexCode{ + 0xe0382000u, 0x80000005u, 0xbf8c3f70u, 0xf80008cfu, 0x03020100u, 0xbf810000u +}; +constexpr std::array relativePixelCode{ + 0xbefc0383u, 0x7e008700u, 0xf800180fu, 0x00000000u, 0xbf810000u +}; +constexpr std::array pixelCode{ + 0x7e0002f2u, 0xf800180fu, 0x00000000u, 0xbf810000u +}; +constexpr std::array computeCode{ + 0xbefc0382u, 0x7e028700u, 0x34020283u, 0x4a000300u, 0x34000082u, + 0x7e028700u, 0xe0701000u, 0x80000100u, 0xbf810000u +}; + +std::array BufferDescriptor(const void* data, std::uint32_t stride, std::uint32_t count) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (stride << 16u), count, stride == 0u ? 0x31016facu : 0x01016facu}; +} + +void Draw(AgcDriver::VulkanDevice& device, bool relativeVertex) { + pixels.fill(std::byte{0x40}); + const auto target = device.Target(); + const auto userData = BufferDescriptor(positions.data(), 16u, positions.size()); + const auto code = relativeVertex ? std::span(relativeVertexCode) : std::span(vertexCode); + ShaderRecompiler::RecompileRequest vertex{ + {ShaderStage::Vertex, reinterpret_cast(code.data()), code, 0, {}}, + {32u, 0u, userData, std::nullopt, std::nullopt, ShaderRecompiler::ShaderVertexStageInfo{}, {}}, + target, {0u, 0u, 0u, 64u} + }; + vertex.useCache = false; + const auto vertexResult = ShaderRecompiler::Recompile(vertex); + const auto pushBytes = static_cast(vertexResult.pushConstants.size()); + ShaderRecompiler::ShaderPixelStageInfo pixel{}; + pixel.wave32 = true; + if (!relativeVertex) { + pixel.inputAddr = ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionX) + | ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionY) + | ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionZ) + | ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionW); + pixel.posX = pixel.posY = pixel.posZ = pixel.posW = true; + } + pixel.targetOutputMode[0] = 9u; + pixel.targetExportMapping.fill(0xe4u); + const auto fragmentCode = relativeVertex ? std::span(pixelCode) : std::span(relativePixelCode); + ShaderRecompiler::RecompileRequest fragment{ + {ShaderStage::Fragment, reinterpret_cast(fragmentCode.data()), fragmentCode, 0, {}}, + {32u, 0u, {}, std::nullopt, pixel, std::nullopt, {}}, target, PixelPushLayout(pushBytes, target) + }; + fragment.useCache = false; + const auto pixelResult = ShaderRecompiler::Recompile(fragment); + const std::array shaders{{ + {ShaderStage::Vertex, &vertexResult, 0u}, {ShaderStage::Fragment, &pixelResult, PixelPushOffset(pushBytes, target)} + }}; + AgcDriver::Graphics::State state{}; + state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, 32u, 32u, std::nullopt, std::nullopt}; + state.color = {reinterpret_cast(pixels.data()), {Extent, Extent}, VK_FORMAT_R8G8B8A8_UNORM, pixels.size(), 0xe4u}; + state.colors = {state.color}; + state.hasColorTarget = true; + state.renderExtent = {Extent, Extent}; + state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; + state.viewport = {0.0f, static_cast(Extent), static_cast(Extent), -static_cast(Extent), 0.0f, 1.0f}; + state.scissor = {{0, 0}, {Extent, Extent}}; + state.cullMode = VK_CULL_MODE_NONE; + state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE; + state.blend.colorWriteMask = 15u; + state.blends = {state.blend}; + AgcDriver::Pm4::DrawParameters draw{0u, 3u, 0u, 1u, 0u, false}; + draw.firstVertex = relativeVertex ? 1u : 3u; + draw.firstInstance = 2u; + device.Draw(state, draw, shaders); + device.WaitIdle(); + for (const auto value : pixels) Require(value == std::byte{255}, "relative stage input draw did not cover every pixel in white"); +} + +void Dispatch(AgcDriver::VulkanDevice& device) { + output.fill(0xdeadbeefu); + const auto userData = BufferDescriptor(output.data(), 0u, sizeof(output)); + const ShaderRecompiler::ShaderComputeStageInfo compute{{8u, 1u, 2u}, 0u, {false, false, false}, false, 3u}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(computeCode.data()), computeCode, 0, {}}, + {32u, 0u, userData, compute, std::nullopt, std::nullopt, {}}, device.Target(), {0u, 0u, 0u, 128u} + }; + request.useCache = false; + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, 1u, 1u, 1u); + device.WaitIdle(); + for (std::size_t index = 0; index < output.size(); ++index) { + Require(output[index] == index / 8u, "relative stage input dispatch did not preserve the implicit Z input at output " + std::to_string(index) + + ": got " + std::to_string(output[index]) + ", expected " + std::to_string(index / 8u)); + } +} + +template +bool Run(const char* name, TAction action) { + try { + action(); + std::cout << "relative stage inputs: " << name << " GPU readback passed\n"; + return true; + } catch (const std::exception& error) { + std::cerr << "relative stage inputs: " << name << ": " << error.what() << '\n'; + return false; + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + bool passed = Run("vertex", [&] { Draw(*device, true); }); + passed &= Run("pixel", [&] { Draw(*device, false); }); + passed &= Run("compute", [&] { Dispatch(*device); }); + if (!passed) return 1; + std::cout << "relative stage input tests passed\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/shader/recompiler/Translation/src/InstructionTranslator.cpp b/core/shader/recompiler/Translation/src/InstructionTranslator.cpp index f99b87af0..9d54fe93f 100644 --- a/core/shader/recompiler/Translation/src/InstructionTranslator.cpp +++ b/core/shader/recompiler/Translation/src/InstructionTranslator.cpp @@ -90,7 +90,8 @@ bool isCodeTableLoad(const ControlFlowGraph& cfg, std::uint32_t programCounter) return std::find(cfg.codeTableLoadProgramCounters.begin(), cfg.codeTableLoadProgramCounters.end(), programCounter) != cfg.codeTableLoadProgramCounters.end(); } -void translateExternalFetch(TranslationContext& context, const RdnaInstruction& call, const ShaderVertexInputInfo* input) { +template +void forEachExternalFetchLoad(const RdnaInstruction& call, const ShaderVertexInputInfo* input, TAction action) { if (input == nullptr || input->resourcesNum <= 0) { throw std::runtime_error("s_swappc_b64 fetch-shader call at program counter " + std::to_string(call.programCounter) + " has no parsed fetch-shader plan to inline"); } @@ -108,8 +109,9 @@ void translateExternalFetch(TranslationContext& context, const RdnaInstruction& fetch.programCounter = call.programCounter; fetch.destination.kind = RdnaOperandKind::VectorRegister; fetch.destination.reg = static_cast(destination.registerStart); + fetch.dataDwordCount = static_cast(destination.registersNum); fetch.formatted = true; - context.TranslateEmbeddedFetch(fetch, static_cast(index), static_cast(destination.registersNum)); + action(static_cast(index), fetch); } } @@ -161,6 +163,15 @@ void emitEntryPrologue(IrProgram& program, IrBlock& entryBlock, const TranslateO IrBuilder entryIr(program); entryIr.SetInsertionPoint(entryBlock); + const auto setVector = [&entryIr, &program](VectorReg reg, IrValue& value) { + auto& metadata = program.Metadata(); + const auto index = RegIndex(reg); + if (metadata.vectorWindowSize != 0u && index >= metadata.vectorWindowBase) { + metadata.vectorWindowSize = std::max(metadata.vectorWindowSize, index - metadata.vectorWindowBase + 1u); + } + entryIr.SetVectorReg(reg, value); + }; + const auto builtin = [&entryIr](StageInputKind kind, std::uint32_t component = 0u) -> IrValue& { return entryIr.Emit(IrOpcode::GetBuiltin, IrOpcodeType(IrOpcode::GetBuiltin), {&entryIr.Constant(static_cast(kind)), &entryIr.Constant(component)}); }; @@ -207,7 +218,7 @@ void emitEntryPrologue(IrProgram& program, IrBlock& entryBlock, const TranslateO const auto* cs = options.inputInfo.compute; const std::uint32_t threadIds = cs->threadIdsNum > 0 ? std::min(static_cast(cs->threadIdsNum), 3u) : 0u; for (std::uint32_t index = 0; index < threadIds; index++) { - entryIr.SetVectorReg(static_cast(index), builtin(StageInputKind::LocalInvocationId, index)); + setVector(static_cast(index), builtin(StageInputKind::LocalInvocationId, index)); } std::uint32_t regOffset = 0; for (std::uint32_t index = 0; index < 3u; index++) { @@ -279,11 +290,11 @@ void emitEntryPrologue(IrProgram& program, IrBlock& entryBlock, const TranslateO IrValue& first = fan ? entryIr.IMul(entryIr.IAdd(vertex, u32(1u)), item) : unlessRestarted(entryIr.IMul(entryIr.IAdd(vertex, parity), item)); IrValue& second = fan ? entryIr.IMul(entryIr.IAdd(vertex, u32(2u)), item) : size >= 2u ? unlessRestarted(entryIr.IMul(entryIr.ISub(entryIr.IAdd(vertex, u32(1u)), parity), item)) : u32(0u); IrValue& third = fan ? u32(0u) : size == 3u ? unlessRestarted(entryIr.IMul(entryIr.IAdd(vertex, u32(2u)), item)) : u32(0u); - entryIr.SetVectorReg(static_cast(0), entryIr.BitwiseOr(entryIr.BitwiseAnd(first, u32(0xffffu)), entryIr.ShiftLeftLogical(second, u32(16u)))); - entryIr.SetVectorReg(static_cast(1), entryIr.BitwiseAnd(third, u32(0xffffu))); - entryIr.SetVectorReg(static_cast(2), entryIr.IAdd(firstPrimitive, local)); - entryIr.SetVectorReg(static_cast(3), u32(0u)); - entryIr.SetVectorReg(static_cast(4), u32(0u)); + setVector(static_cast(0), entryIr.BitwiseOr(entryIr.BitwiseAnd(first, u32(0xffffu)), entryIr.ShiftLeftLogical(second, u32(16u)))); + setVector(static_cast(1), entryIr.BitwiseAnd(third, u32(0xffffu))); + setVector(static_cast(2), entryIr.IAdd(firstPrimitive, local)); + setVector(static_cast(3), u32(0u)); + setVector(static_cast(4), u32(0u)); if (options.userDataBaseRegister != 0u || options.userDataCount < 8u) { throw std::runtime_error("mesh shader translation requires the merged program's eight hidden user words"); } @@ -301,30 +312,30 @@ void emitEntryPrologue(IrProgram& program, IrBlock& entryBlock, const TranslateO } IrValue& restartIndex = entryIr.Select(entryIr.IEqual(indexBytes, u32(2u)), u32(0xffffu), u32(0xffffffffu)); IrValue& fetchedIndex = entryIr.Select(entryIr.LogicalAnd(restartTable, entryIr.IEqual(index, restartIndex)), u32(0u), index); - entryIr.SetVectorReg(static_cast(5), entryIr.IAdd(draw(1u), entryIr.Select(indexed, fetchedIndex, inputVertex))); - entryIr.SetVectorReg(static_cast(6), u32(0u)); - entryIr.SetVectorReg(static_cast(7), u32(0u)); - entryIr.SetVectorReg(static_cast(8), entryIr.IAdd(draw(2u), builtin(StageInputKind::WorkgroupId, 1u))); + setVector(static_cast(5), entryIr.IAdd(draw(1u), entryIr.Select(indexed, fetchedIndex, inputVertex))); + setVector(static_cast(6), u32(0u)); + setVector(static_cast(7), u32(0u)); + setVector(static_cast(8), entryIr.IAdd(draw(2u), builtin(StageInputKind::WorkgroupId, 1u))); } else if (options.stage == ShaderStageKind::Local) { entryIr.SetScalarReg(static_cast(3), entryIr.Constant(64u)); - entryIr.SetVectorReg(static_cast(2), drawIndex(StageInputKind::VertexIndex)); - entryIr.SetVectorReg(static_cast(3), tessellationBase(TessellationBaseKind::RelativeIndex)); - entryIr.SetVectorReg(static_cast(5), drawIndex(StageInputKind::InstanceIndex)); + setVector(static_cast(2), drawIndex(StageInputKind::VertexIndex)); + setVector(static_cast(3), tessellationBase(TessellationBaseKind::RelativeIndex)); + setVector(static_cast(5), drawIndex(StageInputKind::InstanceIndex)); } else if (options.stage == ShaderStageKind::TessellationControl) { const auto& tess = options.inputInfo.vertex->tess; entryIr.SetScalarReg(static_cast(2), tessellationBase(TessellationBaseKind::OffChip)); entryIr.SetScalarReg(static_cast(4), tessellationBase(TessellationBaseKind::Factor)); entryIr.SetScalarReg(static_cast(3), entryIr.Constant(0x81010000u | tess.inputControlPoints | (tess.outputControlPoints << 8u))); - entryIr.SetVectorReg(static_cast(0), builtin(StageInputKind::PrimitiveId)); - entryIr.SetVectorReg(static_cast(1), entryIr.BitwiseOr(entryIr.ShiftLeftLogical(builtin(StageInputKind::InvocationId), entryIr.Constant(8u)), tessellationBase(TessellationBaseKind::RelativeIndex))); + setVector(static_cast(0), builtin(StageInputKind::PrimitiveId)); + setVector(static_cast(1), entryIr.BitwiseOr(entryIr.ShiftLeftLogical(builtin(StageInputKind::InvocationId), entryIr.Constant(8u)), tessellationBase(TessellationBaseKind::RelativeIndex))); } else if (options.stage == ShaderStageKind::TessellationEvaluation) { entryIr.SetScalarReg(static_cast(3), entryIr.Constant(64u)); entryIr.SetScalarReg(static_cast(4), tessellationBase(TessellationBaseKind::OffChip)); - entryIr.SetVectorReg(static_cast(0), tessellationBase(TessellationBaseKind::PassthroughPrimitive)); - entryIr.SetVectorReg(static_cast(5), builtin(StageInputKind::TessCoord, 0u)); - entryIr.SetVectorReg(static_cast(6), builtin(StageInputKind::TessCoord, 1u)); - entryIr.SetVectorReg(static_cast(7), tessellationBase(TessellationBaseKind::RelativeIndex)); - entryIr.SetVectorReg(static_cast(8), builtin(StageInputKind::PrimitiveId)); + setVector(static_cast(0), tessellationBase(TessellationBaseKind::PassthroughPrimitive)); + setVector(static_cast(5), builtin(StageInputKind::TessCoord, 0u)); + setVector(static_cast(6), builtin(StageInputKind::TessCoord, 1u)); + setVector(static_cast(7), tessellationBase(TessellationBaseKind::RelativeIndex)); + setVector(static_cast(8), builtin(StageInputKind::PrimitiveId)); } else if (options.stage == ShaderStageKind::Pixel) { const auto* ps = options.inputInfo.pixel; const auto vgpr = [&](PixelInput input) { return ps->psInputVgpr[static_cast(input)]; }; @@ -332,44 +343,44 @@ void emitEntryPrologue(IrProgram& program, IrBlock& entryBlock, const TranslateO for (const auto [input, kind] : {std::pair{PixelInput::PerspectiveCenter, StageInputKind::BaryCoordSmooth}, std::pair{PixelInput::PerspectiveCentroid, StageInputKind::BaryCoordSmooth}, std::pair{PixelInput::LinearCenter, StageInputKind::BaryCoordNoPerspective}, std::pair{PixelInput::LinearCentroid, StageInputKind::BaryCoordNoPerspective}}) { if (!loaded(input)) continue; - entryIr.SetVectorReg(static_cast(vgpr(input)), builtin(kind, 0u)); - entryIr.SetVectorReg(static_cast(vgpr(input) + 1u), builtin(kind, 1u)); + setVector(static_cast(vgpr(input)), builtin(kind, 0u)); + setVector(static_cast(vgpr(input) + 1u), builtin(kind, 1u)); } if (loaded(PixelInput::PositionX)) { - entryIr.SetVectorReg(static_cast(vgpr(PixelInput::PositionX)), builtin(StageInputKind::FragCoord, 0u)); + setVector(static_cast(vgpr(PixelInput::PositionX)), builtin(StageInputKind::FragCoord, 0u)); } if (loaded(PixelInput::PositionY)) { - entryIr.SetVectorReg(static_cast(vgpr(PixelInput::PositionY)), builtin(StageInputKind::FragCoord, 1u)); + setVector(static_cast(vgpr(PixelInput::PositionY)), builtin(StageInputKind::FragCoord, 1u)); } if (loaded(PixelInput::PositionZ)) { - entryIr.SetVectorReg(static_cast(vgpr(PixelInput::PositionZ)), builtin(StageInputKind::FragCoord, 2u)); + setVector(static_cast(vgpr(PixelInput::PositionZ)), builtin(StageInputKind::FragCoord, 2u)); } if (loaded(PixelInput::PositionW)) { IrValue& reciprocalW = entryIr.BitCastF32(builtin(StageInputKind::FragCoord, 3u)); IrValue& w = entryIr.Emit(IrOpcode::FPRecip32, IrOpcodeType(IrOpcode::FPRecip32), {&reciprocalW}); - entryIr.SetVectorReg(static_cast(vgpr(PixelInput::PositionW)), entryIr.BitCastU32(w)); + setVector(static_cast(vgpr(PixelInput::PositionW)), entryIr.BitCastU32(w)); } if (loaded(PixelInput::FrontFace)) { - entryIr.SetVectorReg(static_cast(vgpr(PixelInput::FrontFace)), builtin(StageInputKind::FrontFacing)); + setVector(static_cast(vgpr(PixelInput::FrontFace)), builtin(StageInputKind::FrontFacing)); } if (loaded(PixelInput::Ancillary)) { - entryIr.SetVectorReg(static_cast(vgpr(PixelInput::Ancillary)), builtin(StageInputKind::PackedAncillary)); + setVector(static_cast(vgpr(PixelInput::Ancillary)), builtin(StageInputKind::PackedAncillary)); } if (loaded(PixelInput::LineStipple)) { - entryIr.SetVectorReg(static_cast(vgpr(PixelInput::LineStipple)), entryIr.Constant(0u)); + setVector(static_cast(vgpr(PixelInput::LineStipple)), entryIr.Constant(0u)); } if (loaded(PixelInput::PositionFixedPoint)) { IrValue& x = entryIr.Emit(IrOpcode::ConvertU32F32, IrType::U32, {&entryIr.BitCastF32(builtin(StageInputKind::FragCoord, 0u))}); IrValue& y = entryIr.Emit(IrOpcode::ConvertU32F32, IrType::U32, {&entryIr.BitCastF32(builtin(StageInputKind::FragCoord, 1u))}); - entryIr.SetVectorReg(static_cast(vgpr(PixelInput::PositionFixedPoint)), entryIr.BitwiseOr(entryIr.BitwiseAnd(x, entryIr.Constant(0xffffu)), entryIr.ShiftLeftLogical(y, entryIr.Constant(16u)))); + setVector(static_cast(vgpr(PixelInput::PositionFixedPoint)), entryIr.BitwiseOr(entryIr.BitwiseAnd(x, entryIr.Constant(0xffffu)), entryIr.ShiftLeftLogical(y, entryIr.Constant(16u)))); } } else if (options.stage == ShaderStageKind::Vertex) { if (options.userDataBaseRegister >= 8u) { const std::uint32_t lanes = options.waveSize == 64u && options.inputInfo.vertex->hostSubgroupSize == 32u ? 32u : options.waveSize; entryIr.SetScalarReg(static_cast(3), entryIr.Constant(lanes | (lanes << 8u))); } - entryIr.SetVectorReg(static_cast(5), drawIndex(StageInputKind::VertexIndex)); - entryIr.SetVectorReg(static_cast(8), drawIndex(StageInputKind::InstanceIndex)); + setVector(static_cast(5), drawIndex(StageInputKind::VertexIndex)); + setVector(static_cast(8), drawIndex(StageInputKind::InstanceIndex)); } } @@ -392,6 +403,11 @@ IrProgram InstructionTranslator::Translate(const RdnaProgram& decoded, const Con continue; } includeInstructionVectorRegisters(instruction, vectorLimit); + if (cfg.hasFetchCall && instruction.programCounter == cfg.fetchCallProgramCounter) { + forEachExternalFetchLoad(instruction, options.inputInfo.vertex, [&](std::uint32_t, const RdnaInstruction& fetch) { + vectorLimit = std::min(NumVectorRegs, std::max(vectorLimit, fetch.destination.reg + fetch.dataDwordCount)); + }); + } } } @@ -482,6 +498,9 @@ IrProgram InstructionTranslator::Translate(const RdnaProgram& decoded, const Con blocks.front()->AddBranch(blocks[blockIndices.at(cfg.blocks.front().id)]); emitEntryPrologue(program, *blocks.front(), options); + if (program.Metadata().vectorWindowSize != 0u) { + vectorLimit = program.Metadata().vectorWindowBase + program.Metadata().vectorWindowSize; + } for (const auto& cfgBlock : cfg.blocks) { const auto typedIndex = blockIndices.at(cfgBlock.id); @@ -511,7 +530,9 @@ IrProgram InstructionTranslator::Translate(const RdnaProgram& decoded, const Con } context.TranslateInstruction(instruction); if (cfg.hasFetchCall && instruction.programCounter == cfg.fetchCallProgramCounter) { - translateExternalFetch(context, instruction, options.inputInfo.vertex); + forEachExternalFetchLoad(instruction, options.inputInfo.vertex, [&](std::uint32_t attribute, const RdnaInstruction& fetch) { + context.TranslateEmbeddedFetch(fetch, attribute, fetch.dataDwordCount); + }); } } context.AddBranchCondition(cfgBlock, blockInfos[typedIndex]);