diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp index aae60dba2..8da0d00f9 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp @@ -2965,6 +2965,7 @@ ShaderRecompiler::SpirvTarget VulkanDevice::buildTarget() const { target.fixedPushSlots = state->graphicsPipelineLibrary; target.registrationModules = state->graphicsPipelineLibrary; target.srgbDecodeFormats = state->srgbDecodeFormats; + target.vertexSubgroups = (state->subgroup.supportedStages & VK_SHADER_STAGE_VERTEX_BIT) != 0; if (state->meshShader) { const auto& mesh = state->meshLimits; target.mesh = ShaderRecompiler::MeshTargetLimits{{mesh.maxMeshWorkGroupSize[0], mesh.maxMeshWorkGroupSize[1], mesh.maxMeshWorkGroupSize[2]}, mesh.maxMeshWorkGroupInvocations, std::min(mesh.maxMeshSharedMemorySize, mesh.maxMeshPayloadAndSharedMemorySize), mesh.maxMeshOutputVertices, mesh.maxMeshOutputPrimitives, mesh.maxMeshOutputComponents, std::min(mesh.maxMeshOutputMemorySize, mesh.maxMeshPayloadAndOutputMemorySize), mesh.meshOutputPerVertexGranularity, mesh.meshOutputPerPrimitiveGranularity}; diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp index f650a2b59..57596b5d6 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp @@ -1464,6 +1464,94 @@ void verifyNarrowVertexWave() { require(GetPreparedArtifact(*PrepareShader(vertex(wave32, 32u, 32u, 8u))).spirv == GetPreparedArtifact(*PrepareShader(vertex(wave32, 32u, 64u, 8u))).spirv, "narrow vertex wave: s3 of a wave32 vertex program depended on the subgroup width"); } +void verifySingleLaneVertexWave() { + using namespace ShaderRecompiler; + static constexpr std::array capabilities{spv::CapabilityGroupNonUniform, spv::CapabilityGroupNonUniformBallot, spv::CapabilityGroupNonUniformShuffle, spv::CapabilityGroupNonUniformArithmetic, spv::CapabilityClipDistance}; + const auto vertex = [](std::span code, std::uint32_t subgroupSize, bool vertexSubgroups) { + RecompileRequest request{}; + request.shader = {ShaderStage::Vertex, 0x58000u, code, 0, {}}; + request.context.waveSize = 64; + request.context.userDataBaseRegister = 8; + request.context.vertex = ShaderVertexStageInfo{}; + request.target.vulkanVersion = 0x00401000u; + request.target.spirvVersion = 0x00010300u; + request.target.subgroupSize = subgroupSize; + request.target.supportedCapabilities = capabilities; + request.target.vertexSubgroups = vertexSubgroups; + request.layout.pushConstantSizeBytes = 128; + request.useCache = false; + return request; + }; + const auto compiled = [&vertex](std::span code, std::uint32_t subgroupSize, bool vertexSubgroups) { + return Recompile(vertex(code, subgroupSize, vertexSubgroups)).spirv; + }; + const auto usesSubgroups = [](const std::vector& spirv) { + for (std::size_t cursor = 5; cursor < spirv.size() && (spirv[cursor] >> 16u) != 0u; cursor += spirv[cursor] >> 16u) { + const auto op = spirv[cursor] & 0xffffu; + if (op == spv::OpCapability && spirv[cursor + 1u] >= spv::CapabilityGroupNonUniform && spirv[cursor + 1u] <= spv::CapabilityGroupNonUniformQuad) return true; + if (op >= spv::OpGroupNonUniformElect && op <= spv::OpGroupNonUniformQuadSwap) return true; + if (op == spv::OpDecorate && spirv[cursor + 2u] == spv::DecorationBuiltIn && (spirv[cursor + 3u] == spv::BuiltInSubgroupLocalInvocationId || spirv[cursor + 3u] == spv::BuiltInSubgroupSize)) return true; + } + return false; + }; + const auto hasConstant = [](const std::vector& spirv, std::uint32_t value) { + for (std::size_t cursor = 5; cursor < spirv.size() && (spirv[cursor] >> 16u) != 0u; cursor += spirv[cursor] >> 16u) { + if ((spirv[cursor] & 0xffffu) == spv::OpConstant && (spirv[cursor] >> 16u) == 4u && spirv[cursor + 3u] == value) return true; + } + return false; + }; + static constexpr std::array waveInfo{0x7e020203u, 0xf80008cfu, 0x01010101u, 0xbf810000u}; + static constexpr std::array laneOps{ + 0xd7650015u, 0x000100c1u, 0xd7660015u, 0x00022ac1u, 0x7e0c0500u, 0xd7600007u, 0x00010100u, 0x4a022a06u, + 0x4a020207u, 0xf80008cfu, 0x01010101u, 0xbf810000u, + }; + static constexpr std::array waterfall{0xbe80047eu, 0xbe821400u, 0xd7600003u, 0x00000500u, 0xbe801c02u, 0xbf138000u, 0xbf85fffau, 0x7e020203u, 0xf80008cfu, 0x01010101u, 0xbf810000u}; + static constexpr std::array lane31{0xd7600006u, 0x00013f00u, 0x7e020206u, 0xf80008cfu, 0x01010101u, 0xbf810000u}; + + const auto single = compiled(waveInfo, 32u, false); + const auto wide = compiled(waveInfo, 64u, true); + require(hasConstant(single, 0x0101u) && !hasConstant(single, 0x4040u) && !hasConstant(single, 0x2020u), "single-lane vertex wave: s3 of a vertex program without vertex subgroups was not one vertex and one primitive"); + require(hasConstant(wide, 0x4040u) && !hasConstant(wide, 0x0101u), "single-lane vertex wave: s3 of a vertex program with vertex subgroups on 64 lanes changed"); + require(single == compiled(waveInfo, 4u, false), "single-lane vertex wave: the program without vertex subgroups depended on the device subgroup size"); + for (const auto code : {std::span(laneOps), std::span(waterfall)}) { + const auto withSubgroups = compiled(code, 32u, true); + const auto withoutSubgroups = compiled(code, 32u, false); + require(usesSubgroups(withSubgroups), "single-lane vertex wave: the program did not use subgroup operations with vertex subgroups"); + require(!withoutSubgroups.empty() && !usesSubgroups(withoutSubgroups), "single-lane vertex wave: a vertex program without vertex subgroups declared a subgroup capability, built-in or instruction"); + } + expectFailure([&] { static_cast(compiled(lane31, 32u, false)); }, "v_readlane_b32 of lane 31 is outside the 1-lane host subgroup", "single-lane vertex wave: a vertex program without vertex subgroups read lane 31"); + require(!compiled(lane31, 32u, true).empty(), "single-lane vertex wave: lane 31 of a 32-lane vertex subgroup did not build"); + + ShaderPixelStageInfo info{}; + info.inputAddr = PixelInputBit(PixelInput::PerspectiveCenter); + info.hasPerspectiveCenterVgpr = true; + info.targetOutputMode[0] = 9u; + info.targetExportMapping.fill(0xe4u); + static constexpr std::array pixelLane31{0xd7600006u, 0x00013f00u, 0x7e020206u, 0xf800180fu, 0x01010101u, 0xbf810000u}; + RecompileRequest pixel{}; + pixel.shader = {ShaderStage::Fragment, 0x30000u, pixelLane31, 0, {}}; + pixel.context.waveSize = 64; + pixel.context.pixel = info; + pixel.target.vulkanVersion = 0x00401000u; + pixel.target.spirvVersion = 0x00010300u; + pixel.target.subgroupSize = 32u; + pixel.target.vertexSubgroups = false; + pixel.layout.pushConstantSizeBytes = 128; + pixel.useCache = false; + require(usesSubgroups(Recompile(pixel).spirv), "single-lane vertex wave: a pixel program stopped using subgroups when only the vertex stage lacks them"); + + auto request = vertex(waveInfo, 32u, false); + const auto replay = RequestSerializer{}.Deserialize(RequestSerializer{}.Serialize(request)); + require(!replay.request.target.vertexSubgroups && replay.request.target.subgroupSize == 32u, "single-lane vertex wave: missing vertex subgroups were lost in serialization"); + require(Recompile(replay.request).spirv == single, "single-lane vertex wave: the replayed request built another program"); + std::vector withoutKey; + std::vector withKey; + RecompileCacheKey::Build(request, withoutKey); + request.target.vertexSubgroups = true; + RecompileCacheKey::Build(request, withKey); + require(withoutKey != withKey, "single-lane vertex wave: vertex subgroup support did not change the cache key"); +} + void verifyMeshConfiguration() { using namespace ShaderRecompiler; ShaderMeshInputInfo list; @@ -1637,12 +1725,12 @@ void verifyPixelRequestSerialization() { minimal.context.waveSize = 64; minimal.context.pixel = ShaderPixelStageInfo{}; const auto encoded = serializer.Serialize(minimal); - require(requestPrefix(encoded, 8u) == "NVNQQREAAAA=", "new requests did not use serialization version 17"); + require(requestPrefix(encoded, 8u) == "NVNQQRIAAAA=", "new requests did not use serialization version 18"); constexpr std::size_t mappingOffset = 8u + 37u + 18u + 163u; for (std::size_t bytes = 0; bytes < 19u; ++bytes) { expectFailure([&] { static_cast(serializer.Deserialize(requestPrefix(encoded, mappingOffset + bytes))); }, "truncated data", "a truncated pixel mapping, packing, dual-source, alpha-to-coverage or stencil reference export flag was accepted"); } - for (const auto unsupported : {"NVNQQQAAAAA=", "NVNQQRIAAAA="}) { + for (const auto unsupported : {"NVNQQQAAAAA=", "NVNQQRMAAAA="}) { expectFailure([&] { static_cast(serializer.Deserialize(unsupported)); }, "serialization version", "an unsupported request version was accepted"); } } @@ -3107,6 +3195,7 @@ int main(int argc, char** argv) { verifyLanesOutsideHostSubgroup(); verifyHalfWaveReduction(); verifyNarrowVertexWave(); + verifySingleLaneVertexWave(); verifyMeshConfiguration(); verifyPixelInputs(); verifyPixelRequestSerialization(); diff --git a/core/shader/recompiler/CacheKey.hpp b/core/shader/recompiler/CacheKey.hpp index 0d27b7ca8..08f5d79c5 100644 --- a/core/shader/recompiler/CacheKey.hpp +++ b/core/shader/recompiler/CacheKey.hpp @@ -243,6 +243,7 @@ private: append(key, value.float64DenormPreserve); append(key, value.registrationModules); append(key, value.registrationWordLimit); + append(key, value.vertexSubgroups); } }; diff --git a/core/shader/recompiler/ControlFlow/src/RequestSerializer.cpp b/core/shader/recompiler/ControlFlow/src/RequestSerializer.cpp index a0f2b44ed..84e55eeaf 100644 --- a/core/shader/recompiler/ControlFlow/src/RequestSerializer.cpp +++ b/core/shader/recompiler/ControlFlow/src/RequestSerializer.cpp @@ -736,7 +736,7 @@ std::string RequestSerializer::Serialize(const RecompileRequest& request) const std::string buffer; Writer writer(buffer); writer.WriteU32(0x41505335u); - writer.WriteU32(17u); + writer.WriteU32(18u); writeShaderBinary(writer, request.shader); writeGuestContext(writer, request.context); writeSpirvTarget(writer, request.target); @@ -758,6 +758,7 @@ std::string RequestSerializer::Serialize(const RecompileRequest& request) const writer.WriteBool(request.target.float64DenormPreserve); writer.WriteBool(request.target.registrationModules); writer.WriteU32(request.target.registrationWordLimit); + writer.WriteBool(request.target.vertexSubgroups); return base64Encode(buffer); } @@ -766,7 +767,7 @@ DeserializedRequest RequestSerializer::Deserialize(std::string_view text) const Reader reader(decoded); if (reader.ReadU32() != 0x41505335u) throw std::runtime_error("invalid recompile request signature"); const auto version = reader.ReadU32(); - if (version < 1u || version > 17u) throw std::runtime_error("unsupported recompile request serialization version"); + if (version < 1u || version > 18u) throw std::runtime_error("unsupported recompile request serialization version"); DeserializedRequest result{}; result.request.shader = readShaderBinary(reader, result.shaderCode, result.shaderHeader); result.request.context = readGuestContext(reader, result, version); @@ -789,6 +790,7 @@ DeserializedRequest RequestSerializer::Deserialize(std::string_view text) const if (version >= 16u) result.request.target.float64DenormPreserve = reader.ReadBool(); if (version >= 17u) result.request.target.registrationModules = reader.ReadBool(); if (version >= 17u) result.request.target.registrationWordLimit = reader.ReadU32(); + if (version >= 18u) result.request.target.vertexSubgroups = reader.ReadBool(); return result; } diff --git a/core/shader/recompiler/Recompiler.cpp b/core/shader/recompiler/Recompiler.cpp index 7b74fc9ce..df79d3e83 100644 --- a/core/shader/recompiler/Recompiler.cpp +++ b/core/shader/recompiler/Recompiler.cpp @@ -85,6 +85,8 @@ namespace { // The host subgroup width wave64 programs are laid out for. Debug aid: APS5_SINGLE_LANE= keeps the listed programs at one guest lane per invocation. std::uint32_t HostSubgroupSize(const RecompileRequest& request) { + const bool vertexStage = request.shader.stage == ShaderStage::Vertex || request.shader.stage == ShaderStage::Local; + if (vertexStage && !request.target.vertexSubgroups) return 1u; static const std::string list = [] { const char* text = std::getenv("APS5_SINGLE_LANE"); return text ? std::string(text) : std::string(); }(); if (!list.empty()) { if (list == "all") return 64u; @@ -424,7 +426,7 @@ CompiledVariant compileVariant(const RecompileRequest& request, IrProgram progra SpirvTargetOptions targetOptions {}; targetOptions.vulkanVersion = request.target.vulkanVersion; targetOptions.spirvVersion = request.target.spirvVersion; - targetOptions.subgroupSize = request.target.subgroupSize; + targetOptions.subgroupSize = HostSubgroupSize(request) == 1u ? 1u : request.target.subgroupSize; targetOptions.bdaAbiVersion = request.target.bdaAbiVersion; targetOptions.supportedCapabilities = request.target.supportedCapabilities; targetOptions.supportedExtensions = request.target.supportedExtensions; diff --git a/core/shader/recompiler/Recompiler.hpp b/core/shader/recompiler/Recompiler.hpp index 121ce4fce..eb84c86dd 100644 --- a/core/shader/recompiler/Recompiler.hpp +++ b/core/shader/recompiler/Recompiler.hpp @@ -232,6 +232,7 @@ struct SpirvTarget { bool float64DenormPreserve = false; bool registrationModules = false; std::uint32_t registrationWordLimit = 200000u; + bool vertexSubgroups = true; }; struct BindingLayout { diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvSubgroup.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvSubgroup.hpp index c7e4746b6..b38bd34c1 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvSubgroup.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvSubgroup.hpp @@ -6,6 +6,11 @@ namespace ShaderRecompiler { +bool HostSubgroupSingleLane(const SpirvEmitterState& state); +void EmitSubgroupBallot(SpirvEmitterState& state, std::uint32_t result, std::uint32_t predicate); +void EmitSubgroupBallotFindLsb(SpirvEmitterState& state, std::uint32_t result, std::uint32_t ballot); +void EmitSubgroupShuffle(SpirvEmitterState& state, std::uint32_t type, std::uint32_t result, std::uint32_t value, std::uint32_t lane); +void EmitSubgroupElect(SpirvEmitterState& state, std::uint32_t result); std::uint32_t EmitSubgroupLocalInvocationId(SpirvEmitterState& state); std::uint32_t EmitHostSubgroupLane(SpirvEmitterState& state, std::uint32_t lane); std::uint32_t EmitWaveBallot(SpirvEmitterState& state, std::uint32_t ballot); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp index 217c580b0..9936d3199 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp @@ -188,9 +188,9 @@ std::uint32_t EmitDsMaskedLaneRead(SpirvEmitterState& state, std::uint32_t sourc } const auto physicalLane = EmitHostSubgroupLane(state, lane); const auto shuffled = state.module.AllocateId(); - state.module.AddFunction(spv::OpGroupNonUniformShuffle, TypeU32(state), shuffled, ConstantU32(state, spv::ScopeSubgroup), source, physicalLane); + EmitSubgroupShuffle(state, TypeU32(state), shuffled, source, physicalLane); const auto sourceExec = state.module.AllocateId(); - state.module.AddFunction(spv::OpGroupNonUniformShuffle, TypeBool(state), sourceExec, ConstantU32(state, spv::ScopeSubgroup), exec, physicalLane); + EmitSubgroupShuffle(state, TypeBool(state), sourceExec, exec, physicalLane); const auto sourceActive = Binary(state, spv::OpLogicalAnd, TypeBool(state), sourceExec, EmitSubgroupLaneActiveBool(state, lane)); return Select(state, TypeU32(state), sourceActive, shuffled, ConstantU32(state, 0u)); } @@ -563,6 +563,7 @@ std::optional EmitHalfWaveReduction(SpirvValueEmitContext& ctx, c if (lane < 32u || state.hostSubgroupSize > 32u) return std::nullopt; return ConstantU32(state, scan->reduction->identity); } + if (HostSubgroupSingleLane(state)) return lane >= 32u ? ConstantU32(state, scan->reduction->identity) : ctx.Def(scan->source); const auto read = "v_readlane_b32 of lane " + std::to_string(lane) + " of a wave64 half-wave reduction scan"; const auto& capabilities = state.supportedCapabilities; if (std::find(capabilities.begin(), capabilities.end(), static_cast(spv::CapabilityGroupNonUniformArithmetic)) == capabilities.end()) { diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp index a3be534ad..3db76e6e7 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp @@ -118,7 +118,7 @@ std::uint32_t SpirvValueEmitContext::Ballot(const IrValue* predicate) { const auto ballotType = TypeU32Vector(state, 4u); const auto scope = ConstantU32(state, spv::ScopeSubgroup); const auto low = state.module.AllocateId(); - state.module.AddFunction(spv::OpGroupNonUniformBallot, ballotType, low, scope, otherHalf == nullptr || half == 0u ? Def(predicate) : otherHalf->Def(predicate)); + EmitSubgroupBallot(state, low, otherHalf == nullptr || half == 0u ? Def(predicate) : otherHalf->Def(predicate)); if (otherHalf == nullptr) { return EmitWaveBallot(state, low); } @@ -136,7 +136,7 @@ std::uint32_t SpirvValueEmitContext::Ballot(const IrValue* predicate) { std::uint32_t SpirvValueEmitContext::FirstLane(std::uint32_t ballot) { if (otherHalf == nullptr) { const auto result = state.module.AllocateId(); - state.module.AddFunction(spv::OpGroupNonUniformBallotFindLSB, TypeU32(state), result, ConstantU32(state, spv::ScopeSubgroup), ballot); + EmitSubgroupBallotFindLsb(state, result, ballot); return result; } const auto low = state.module.AllocateId(); @@ -159,7 +159,7 @@ std::uint32_t SpirvValueEmitContext::Shuffle(const IrValue& inst, std::size_t in const auto scope = ConstantU32(state, spv::ScopeSubgroup); const auto low = state.module.AllocateId(); if (otherHalf == nullptr) { - state.module.AddFunction(spv::OpGroupNonUniformShuffle, type, low, scope, Arg(inst, index), EmitHostSubgroupLane(state, lane)); + EmitSubgroupShuffle(state, type, low, Arg(inst, index), EmitHostSubgroupLane(state, lane)); return low; } const auto physicalLane = EmitBinaryU32(state, spv::OpBitwiseAnd, lane, ConstantU32(state, 31u)); @@ -253,6 +253,11 @@ std::vector SpirvEmitter::Emit(const IrProgram& program, const Sh state.registrationModules = target.registrationModules; state.fragmentDescriptorSet = target.fragmentDescriptorSet; state.hostSubgroupSize = target.subgroupSize; + if (HostSubgroupSingleLane(state)) { + state.requirements.subgroupBallot = false; + state.requirements.subgroupShuffle = false; + state.requirements.subgroupLocalInvocationId = false; + } state.splitSubgroup = program.WaveSize() == 32u && target.subgroupSize > 32u; if (state.splitSubgroup && (state.requirements.subgroupBallot || state.requirements.subgroupShuffle)) state.requirements.subgroupLocalInvocationId = true; const auto* workgroup = ShaderWorkgroupInputFor(state); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp index 5401643be..1870ca187 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp @@ -132,7 +132,7 @@ bool IsContinueTarget(const IrProgram& program, std::uint32_t block) { std::uint32_t EmitWaveAny(SpirvEmitterState& state, std::uint32_t predicate) { const auto ballot = state.module.AllocateId(); - state.module.AddFunction(spv::OpGroupNonUniformBallot, TypeU32Vector(state, 4u), ballot, ConstantU32(state, spv::ScopeSubgroup), predicate); + EmitSubgroupBallot(state, ballot, predicate); const auto wave = EmitWaveBallot(state, ballot); const auto low = state.module.AllocateId(); const auto high = state.module.AllocateId(); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp index 70296b7f3..64ae0cd78 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp @@ -1181,7 +1181,7 @@ std::uint32_t AppendConsume(SpirvValueEmitContext& ctx, const IrValue& inst, boo return value; }); const auto result = state.module.AllocateId(); - state.module.AddFunction(spv::OpGroupNonUniformShuffle, TypeU32(state), result, ConstantU32(state, spv::ScopeSubgroup), atomic, EmitHostSubgroupLane(state, sourceLane)); + EmitSubgroupShuffle(state, TypeU32(state), result, atomic, EmitHostSubgroupLane(state, sourceLane)); return result; } @@ -1214,7 +1214,7 @@ std::uint32_t LockedLdsUpdate(SpirvEmitterState& state, std::uint32_t type, std: state.module.AddFunction(spv::OpBranchConditional, done, pendingMerge, pending); EmitLabel(state, pending); const auto elected = state.module.AllocateId(); - state.module.AddFunction(spv::OpGroupNonUniformElect, TypeBool(state), elected, ConstantU32(state, spv::ScopeSubgroup)); + EmitSubgroupElect(state, elected); state.module.AddFunction(spv::OpSelectionMerge, servedMerge, spv::SelectionControlMaskNone); state.module.AddFunction(spv::OpBranchConditional, elected, attempt, servedMerge); EmitLabel(state, attempt); @@ -1244,7 +1244,7 @@ std::uint32_t LockedLdsUpdate(SpirvEmitterState& state, std::uint32_t type, std: state.module.AddFunction(spv::OpPhi, TypeBool(state), doneNext, served, servedMerge, done, body); state.module.AddFunction(spv::OpPhi, type, resultNext, servedResult, servedMerge, result, body); const auto ballot = state.module.AllocateId(); - state.module.AddFunction(spv::OpGroupNonUniformBallot, TypeU32Vector(state, 4u), ballot, ConstantU32(state, spv::ScopeSubgroup), Unary(state, spv::OpLogicalNot, TypeBool(state), doneNext)); + EmitSubgroupBallot(state, ballot, Unary(state, spv::OpLogicalNot, TypeBool(state), doneNext)); std::uint32_t remaining = ConstantU32(state, 0u); for (std::uint32_t component = 0; component < 4u; ++component) { const auto word = state.module.AllocateId(); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvSubgroup.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvSubgroup.cpp index cde24c0ba..6efac49c4 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvSubgroup.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvSubgroup.cpp @@ -1,5 +1,7 @@ #include "SpirvBackend/SpirvEmitter.hpp" #include "SpirvBackend/SpirvEmitterHelpers.hpp" +#include "SpirvBackend/SpirvMemory/SpirvSubgroup.hpp" +#include #include #include #include @@ -15,6 +17,7 @@ namespace ShaderRecompiler namespace { std::uint32_t HostInvocationId(SpirvEmitterState& state) { + if (HostSubgroupSingleLane(state)) return ConstantU32(state, 0u); if (state.subgroupLocalInvocationIdVariable == 0) { FailEmit("SubgroupLocalInvocationId was not declared before function emission"); } @@ -25,6 +28,46 @@ namespace ShaderRecompiler } + bool HostSubgroupSingleLane(const SpirvEmitterState& state) { + return state.hostSubgroupSize == 1u; + } + + void EmitSubgroupBallot(SpirvEmitterState& state, std::uint32_t result, std::uint32_t predicate) { + if (HostSubgroupSingleLane(state)) { + const auto zero = ConstantU32(state, 0u); + const auto own = Select(state, TypeU32(state), predicate, ConstantU32(state, 1u), zero); + state.module.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(state, 4u), result, own, zero, zero, zero); + return; + } + state.module.AddFunction(spv::OpGroupNonUniformBallot, TypeU32Vector(state, 4u), result, ConstantU32(state, spv::ScopeSubgroup), predicate); + } + + void EmitSubgroupBallotFindLsb(SpirvEmitterState& state, std::uint32_t result, std::uint32_t ballot) { + if (HostSubgroupSingleLane(state)) { + const auto own = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), own, ballot, 0u); + state.module.AddFunction(spv::OpExtInst, TypeU32(state), result, GlslStd450(state), GLSLstd450FindILsb, own); + return; + } + state.module.AddFunction(spv::OpGroupNonUniformBallotFindLSB, TypeU32(state), result, ConstantU32(state, spv::ScopeSubgroup), ballot); + } + + void EmitSubgroupShuffle(SpirvEmitterState& state, std::uint32_t type, std::uint32_t result, std::uint32_t value, std::uint32_t lane) { + if (HostSubgroupSingleLane(state)) { + state.module.AddFunction(spv::OpCopyObject, type, result, value); + return; + } + state.module.AddFunction(spv::OpGroupNonUniformShuffle, type, result, ConstantU32(state, spv::ScopeSubgroup), value, lane); + } + + void EmitSubgroupElect(SpirvEmitterState& state, std::uint32_t result) { + if (HostSubgroupSingleLane(state)) { + state.module.AddFunction(spv::OpCopyObject, TypeBool(state), result, ConstantBool(state, true)); + return; + } + state.module.AddFunction(spv::OpGroupNonUniformElect, TypeBool(state), result, ConstantU32(state, spv::ScopeSubgroup)); + } + std::uint32_t EmitSubgroupLocalInvocationId(SpirvEmitterState& state) { const auto value = HostInvocationId(state); if (state.splitSubgroup) return EmitBinaryU32(state, spv::OpBitwiseAnd, value, ConstantU32(state, 31u)); @@ -184,7 +227,7 @@ namespace ShaderRecompiler std::uint32_t EmitSubgroupLaneActiveBool(SpirvEmitterState& state, std::uint32_t lane) { const auto activeBallot = state.module.AllocateId(); - state.module.AddFunction(spv::OpGroupNonUniformBallot, TypeU32Vector(state, 4), activeBallot, ConstantU32(state, spv::ScopeSubgroup), ConstantBool(state, true)); + EmitSubgroupBallot(state, activeBallot, ConstantBool(state, true)); return EmitBallotLaneActiveBool(state, EmitWaveBallot(state, activeBallot), lane); } } diff --git a/core/shader/recompiler/Translation/src/InstructionTranslator.cpp b/core/shader/recompiler/Translation/src/InstructionTranslator.cpp index 04d50e26f..e3455f44e 100644 --- a/core/shader/recompiler/Translation/src/InstructionTranslator.cpp +++ b/core/shader/recompiler/Translation/src/InstructionTranslator.cpp @@ -376,7 +376,8 @@ void emitEntryPrologue(IrProgram& program, IrBlock& entryBlock, const TranslateO } } else if (options.stage == ShaderStageKind::Vertex) { if (options.userDataBaseRegister >= 8u) { - const std::uint32_t lanes = options.waveSize == 64u && options.inputInfo.vertex->hostSubgroupSize == 32u ? 32u : options.waveSize; + const auto hostLanes = options.inputInfo.vertex->hostSubgroupSize; + const std::uint32_t lanes = hostLanes == 1u ? 1u : options.waveSize == 64u && hostLanes == 32u ? 32u : options.waveSize; entryIr.SetScalarReg(static_cast(3), entryIr.Constant(lanes | (lanes << 8u))); } setVector(static_cast(5), drawIndex(StageInputKind::VertexIndex)); diff --git a/docs/dev/TechnicalDebt.md b/docs/dev/TechnicalDebt.md index 10d9d0c0a..10f2f99a6 100644 --- a/docs/dev/TechnicalDebt.md +++ b/docs/dev/TechnicalDebt.md @@ -179,7 +179,7 @@ Throughout the project, every function at every stage either **does exactly what - [User data registers](../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Diagnostics.cpp) (AGC driver) - a `SPI_SHADER_USER_DATA_*` or `COMPUTE_USER_DATA_*` register the title never wrote reads as 0 when a stage's RSRC2 count covers it, as Kyty (zeroed `UserSgprInfo`) and SharpEmu (missing registers read 0) do, and as AMD PAL and Mesa initialize them; the console's value is not confirmed - [SET_PREDICATION](../../core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp) (AGC driver) - boolean predication follows the AMD PM4 encoding [Mesa](https://gitlab.freedesktop.org/mesa/mesa/-/blob/main/src/amd/common/sid.h) uses: operation 3 reads 64 bits and 4 reads 32 bits, and bit 8 runs predicated packets when the value is non-zero (clear: when it is zero); the bit 12 hint is ignored. The address must have its low 4 bits clear, as `sceAgcDcbSetPredication` writes it. Query predication (Z-pass, primitive count), a predicated COND_EXEC, predicated command buffer chains and flips in predicated command buffers throw - [sceAgcGetGsOversubscription](../../core/libs/prx/libSceAgc/Shader/src/GsOversubscription.cpp) (libSceAgc) - signature and occupancy formula (base and expanded vertex/export capacities, budget shift, `GE_PC_ALLOC` / `SPI_SHADER_PGM_RSRC4_GS` encoding) follow KytyPS5 only, not confirmed on hardware or a title; a negative or NaN factor that gives a negative target, a missing GS context register or a zero `GE_MAX_OUTPUT_PER_SUBGROUP` throws -- [Vertex entry merged wave info](../../core/shader/recompiler/Translation/src/InstructionTranslator.cpp) (recompiler) - `s3` of every vertex-path shader is set to `waveSize | waveSize << 8` (vertex and primitive count of a full wave, wave index 0), or to `32 | 32 << 8` for a wave64 program on a 32-lane host subgroup, which runs it at one lane per invocation and holds no lanes 32-63. Layout taken from the prologue Mesa's ACO emits for NGG vertex shaders (`s_bfe_u64 exec` from `s3`), not measured on hardware. The driver gives every graphics program user data from `s8`, so the condition does not tell NGG from legacy vertex shaders: they just never read `s3`. A draw whose last wave is partial still reports a full wave, and so does a host wave with idle invocations (an RTX 5070 ran every vertex warp of a non-indexed triangle list with 30 of its 32 invocations). +- [Vertex entry merged wave info](../../core/shader/recompiler/Translation/src/InstructionTranslator.cpp) (recompiler) - `s3` of every vertex-path shader is set to `waveSize | waveSize << 8` (vertex and primitive count of a full wave, wave index 0), or to `32 | 32 << 8` for a wave64 program on a 32-lane host subgroup, which runs it at one lane per invocation and holds no lanes 32-63. On a device whose `subgroupSupportedStages` lacks the vertex stage (lavapipe, MoltenVK), vertex and local programs run as one-lane waves: `s3` is `1 | 1 << 8`, the lane id is 0, a ballot is the invocation's own bit, a shuffle or broadcast returns the invocation's value, an elect is true and a subgroup reduction is the value itself, so no `GroupNonUniform` capability is declared. Layout taken from the prologue Mesa's ACO emits for NGG vertex shaders (`s_bfe_u64 exec` from `s3`), not measured on hardware. The driver gives every graphics program user data from `s8`, so the condition does not tell NGG from legacy vertex shaders: they just never read `s3`. A draw whose last wave is partial still reports a full wave, and so does a host wave with idle invocations (an RTX 5070 ran every vertex warp of a non-indexed triangle list with 30 of its 32 invocations). - [NGG vertex programs on the mesh path](../../core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp) (AGC driver) - an unregistered vertex program with `PRIMGEN_EN` that uses LDS or `s_barrier` runs on the geometry (mesh) path. Each input primitive is assumed to yield at most one output primitive, the subgroup is bounded by `GE_MAX_OUTPUT_PER_SUBGROUP` as a vertex budget, and the GS/ES VGPR count checks are skipped, because the merged inputs keep their positions. Taken from the code Mesa's ACO emits for NGG culling and from PPSA59322 and PPSA99010 draws, not measured on hardware. Registered programs keep the vertex path - [s_barrier in vertex-path programs](../../core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp) (recompiler) - emits no instruction when the program has no LDS or GDS access and no memory write or atomic, so no wave of the subgroup can observe another; the `GS_ALLOC_REQ` such a barrier orders on NGG passthrough programs is already dropped on the vertex path. The Vertex execution model has no workgroup barrier. Programs with LDS, GDS or memory writes throw. Not measured on hardware - [Strided buffer overhang](../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) (libSceAgcDriver) - a strided buffer descriptor that ends less than one stride past the end of the readable registered allocation holding its base is bound only up to that allocation's end, even when another readable allocation starts right there. This assumes the bytes past the end are the rounded-up `num_records` of the last record (RADV rounds it up), which no fetch inside the buffer reads; a guest that really keeps a vertex split across two adjacent allocations would read zeros for the cut tail. Taken from the RetroArch Dolphin core case in #2912, not measured on hardware.