Merge pull request #4470 from overlordxrz-source/fix/vertex-one-lane-waves
fix(recompiler): run vertex programs as one-lane waves without vertex subgroups
This commit is contained in:
Dean
GitHub
commit
0c75934f5f
|
|
@ -2965,6 +2965,7 @@ ShaderRecompiler::SpirvTarget VulkanDevice::buildTarget() const {
|
|||
target.fixedPushSlots = state->graphicsPipelineLibrary;
|
||||
target.registrationModules = state->graphicsPipelineLibrary;
|
||||
target.srgbDecodeFormats = state->srgbDecodeFormats;
|
||||
target.vertexSubgroups = (state->subgroup.supportedStages & VK_SHADER_STAGE_VERTEX_BIT) != 0;
|
||||
if (state->meshShader) {
|
||||
const auto& mesh = state->meshLimits;
|
||||
target.mesh = ShaderRecompiler::MeshTargetLimits{{mesh.maxMeshWorkGroupSize[0], mesh.maxMeshWorkGroupSize[1], mesh.maxMeshWorkGroupSize[2]}, mesh.maxMeshWorkGroupInvocations, std::min(mesh.maxMeshSharedMemorySize, mesh.maxMeshPayloadAndSharedMemorySize), mesh.maxMeshOutputVertices, mesh.maxMeshOutputPrimitives, mesh.maxMeshOutputComponents, std::min(mesh.maxMeshOutputMemorySize, mesh.maxMeshPayloadAndOutputMemorySize), mesh.meshOutputPerVertexGranularity, mesh.meshOutputPerPrimitiveGranularity};
|
||||
|
|
|
|||
|
|
@ -1464,6 +1464,94 @@ void verifyNarrowVertexWave() {
|
|||
require(GetPreparedArtifact(*PrepareShader(vertex(wave32, 32u, 32u, 8u))).spirv == GetPreparedArtifact(*PrepareShader(vertex(wave32, 32u, 64u, 8u))).spirv, "narrow vertex wave: s3 of a wave32 vertex program depended on the subgroup width");
|
||||
}
|
||||
|
||||
void verifySingleLaneVertexWave() {
|
||||
using namespace ShaderRecompiler;
|
||||
static constexpr std::array<std::uint32_t, 5> capabilities{spv::CapabilityGroupNonUniform, spv::CapabilityGroupNonUniformBallot, spv::CapabilityGroupNonUniformShuffle, spv::CapabilityGroupNonUniformArithmetic, spv::CapabilityClipDistance};
|
||||
const auto vertex = [](std::span<const std::uint32_t> code, std::uint32_t subgroupSize, bool vertexSubgroups) {
|
||||
RecompileRequest request{};
|
||||
request.shader = {ShaderStage::Vertex, 0x58000u, code, 0, {}};
|
||||
request.context.waveSize = 64;
|
||||
request.context.userDataBaseRegister = 8;
|
||||
request.context.vertex = ShaderVertexStageInfo{};
|
||||
request.target.vulkanVersion = 0x00401000u;
|
||||
request.target.spirvVersion = 0x00010300u;
|
||||
request.target.subgroupSize = subgroupSize;
|
||||
request.target.supportedCapabilities = capabilities;
|
||||
request.target.vertexSubgroups = vertexSubgroups;
|
||||
request.layout.pushConstantSizeBytes = 128;
|
||||
request.useCache = false;
|
||||
return request;
|
||||
};
|
||||
const auto compiled = [&vertex](std::span<const std::uint32_t> code, std::uint32_t subgroupSize, bool vertexSubgroups) {
|
||||
return Recompile(vertex(code, subgroupSize, vertexSubgroups)).spirv;
|
||||
};
|
||||
const auto usesSubgroups = [](const std::vector<std::uint32_t>& spirv) {
|
||||
for (std::size_t cursor = 5; cursor < spirv.size() && (spirv[cursor] >> 16u) != 0u; cursor += spirv[cursor] >> 16u) {
|
||||
const auto op = spirv[cursor] & 0xffffu;
|
||||
if (op == spv::OpCapability && spirv[cursor + 1u] >= spv::CapabilityGroupNonUniform && spirv[cursor + 1u] <= spv::CapabilityGroupNonUniformQuad) return true;
|
||||
if (op >= spv::OpGroupNonUniformElect && op <= spv::OpGroupNonUniformQuadSwap) return true;
|
||||
if (op == spv::OpDecorate && spirv[cursor + 2u] == spv::DecorationBuiltIn && (spirv[cursor + 3u] == spv::BuiltInSubgroupLocalInvocationId || spirv[cursor + 3u] == spv::BuiltInSubgroupSize)) return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
const auto hasConstant = [](const std::vector<std::uint32_t>& spirv, std::uint32_t value) {
|
||||
for (std::size_t cursor = 5; cursor < spirv.size() && (spirv[cursor] >> 16u) != 0u; cursor += spirv[cursor] >> 16u) {
|
||||
if ((spirv[cursor] & 0xffffu) == spv::OpConstant && (spirv[cursor] >> 16u) == 4u && spirv[cursor + 3u] == value) return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
static constexpr std::array<std::uint32_t, 4> waveInfo{0x7e020203u, 0xf80008cfu, 0x01010101u, 0xbf810000u};
|
||||
static constexpr std::array<std::uint32_t, 12> laneOps{
|
||||
0xd7650015u, 0x000100c1u, 0xd7660015u, 0x00022ac1u, 0x7e0c0500u, 0xd7600007u, 0x00010100u, 0x4a022a06u,
|
||||
0x4a020207u, 0xf80008cfu, 0x01010101u, 0xbf810000u,
|
||||
};
|
||||
static constexpr std::array<std::uint32_t, 11> waterfall{0xbe80047eu, 0xbe821400u, 0xd7600003u, 0x00000500u, 0xbe801c02u, 0xbf138000u, 0xbf85fffau, 0x7e020203u, 0xf80008cfu, 0x01010101u, 0xbf810000u};
|
||||
static constexpr std::array<std::uint32_t, 6> lane31{0xd7600006u, 0x00013f00u, 0x7e020206u, 0xf80008cfu, 0x01010101u, 0xbf810000u};
|
||||
|
||||
const auto single = compiled(waveInfo, 32u, false);
|
||||
const auto wide = compiled(waveInfo, 64u, true);
|
||||
require(hasConstant(single, 0x0101u) && !hasConstant(single, 0x4040u) && !hasConstant(single, 0x2020u), "single-lane vertex wave: s3 of a vertex program without vertex subgroups was not one vertex and one primitive");
|
||||
require(hasConstant(wide, 0x4040u) && !hasConstant(wide, 0x0101u), "single-lane vertex wave: s3 of a vertex program with vertex subgroups on 64 lanes changed");
|
||||
require(single == compiled(waveInfo, 4u, false), "single-lane vertex wave: the program without vertex subgroups depended on the device subgroup size");
|
||||
for (const auto code : {std::span<const std::uint32_t>(laneOps), std::span<const std::uint32_t>(waterfall)}) {
|
||||
const auto withSubgroups = compiled(code, 32u, true);
|
||||
const auto withoutSubgroups = compiled(code, 32u, false);
|
||||
require(usesSubgroups(withSubgroups), "single-lane vertex wave: the program did not use subgroup operations with vertex subgroups");
|
||||
require(!withoutSubgroups.empty() && !usesSubgroups(withoutSubgroups), "single-lane vertex wave: a vertex program without vertex subgroups declared a subgroup capability, built-in or instruction");
|
||||
}
|
||||
expectFailure([&] { static_cast<void>(compiled(lane31, 32u, false)); }, "v_readlane_b32 of lane 31 is outside the 1-lane host subgroup", "single-lane vertex wave: a vertex program without vertex subgroups read lane 31");
|
||||
require(!compiled(lane31, 32u, true).empty(), "single-lane vertex wave: lane 31 of a 32-lane vertex subgroup did not build");
|
||||
|
||||
ShaderPixelStageInfo info{};
|
||||
info.inputAddr = PixelInputBit(PixelInput::PerspectiveCenter);
|
||||
info.hasPerspectiveCenterVgpr = true;
|
||||
info.targetOutputMode[0] = 9u;
|
||||
info.targetExportMapping.fill(0xe4u);
|
||||
static constexpr std::array<std::uint32_t, 6> pixelLane31{0xd7600006u, 0x00013f00u, 0x7e020206u, 0xf800180fu, 0x01010101u, 0xbf810000u};
|
||||
RecompileRequest pixel{};
|
||||
pixel.shader = {ShaderStage::Fragment, 0x30000u, pixelLane31, 0, {}};
|
||||
pixel.context.waveSize = 64;
|
||||
pixel.context.pixel = info;
|
||||
pixel.target.vulkanVersion = 0x00401000u;
|
||||
pixel.target.spirvVersion = 0x00010300u;
|
||||
pixel.target.subgroupSize = 32u;
|
||||
pixel.target.vertexSubgroups = false;
|
||||
pixel.layout.pushConstantSizeBytes = 128;
|
||||
pixel.useCache = false;
|
||||
require(usesSubgroups(Recompile(pixel).spirv), "single-lane vertex wave: a pixel program stopped using subgroups when only the vertex stage lacks them");
|
||||
|
||||
auto request = vertex(waveInfo, 32u, false);
|
||||
const auto replay = RequestSerializer{}.Deserialize(RequestSerializer{}.Serialize(request));
|
||||
require(!replay.request.target.vertexSubgroups && replay.request.target.subgroupSize == 32u, "single-lane vertex wave: missing vertex subgroups were lost in serialization");
|
||||
require(Recompile(replay.request).spirv == single, "single-lane vertex wave: the replayed request built another program");
|
||||
std::vector<std::uint64_t> withoutKey;
|
||||
std::vector<std::uint64_t> withKey;
|
||||
RecompileCacheKey::Build(request, withoutKey);
|
||||
request.target.vertexSubgroups = true;
|
||||
RecompileCacheKey::Build(request, withKey);
|
||||
require(withoutKey != withKey, "single-lane vertex wave: vertex subgroup support did not change the cache key");
|
||||
}
|
||||
|
||||
void verifyMeshConfiguration() {
|
||||
using namespace ShaderRecompiler;
|
||||
ShaderMeshInputInfo list;
|
||||
|
|
@ -1637,12 +1725,12 @@ void verifyPixelRequestSerialization() {
|
|||
minimal.context.waveSize = 64;
|
||||
minimal.context.pixel = ShaderPixelStageInfo{};
|
||||
const auto encoded = serializer.Serialize(minimal);
|
||||
require(requestPrefix(encoded, 8u) == "NVNQQREAAAA=", "new requests did not use serialization version 17");
|
||||
require(requestPrefix(encoded, 8u) == "NVNQQRIAAAA=", "new requests did not use serialization version 18");
|
||||
constexpr std::size_t mappingOffset = 8u + 37u + 18u + 163u;
|
||||
for (std::size_t bytes = 0; bytes < 19u; ++bytes) {
|
||||
expectFailure([&] { static_cast<void>(serializer.Deserialize(requestPrefix(encoded, mappingOffset + bytes))); }, "truncated data", "a truncated pixel mapping, packing, dual-source, alpha-to-coverage or stencil reference export flag was accepted");
|
||||
}
|
||||
for (const auto unsupported : {"NVNQQQAAAAA=", "NVNQQRIAAAA="}) {
|
||||
for (const auto unsupported : {"NVNQQQAAAAA=", "NVNQQRMAAAA="}) {
|
||||
expectFailure([&] { static_cast<void>(serializer.Deserialize(unsupported)); }, "serialization version", "an unsupported request version was accepted");
|
||||
}
|
||||
}
|
||||
|
|
@ -3107,6 +3195,7 @@ int main(int argc, char** argv) {
|
|||
verifyLanesOutsideHostSubgroup();
|
||||
verifyHalfWaveReduction();
|
||||
verifyNarrowVertexWave();
|
||||
verifySingleLaneVertexWave();
|
||||
verifyMeshConfiguration();
|
||||
verifyPixelInputs();
|
||||
verifyPixelRequestSerialization();
|
||||
|
|
|
|||
|
|
@ -243,6 +243,7 @@ private:
|
|||
append(key, value.float64DenormPreserve);
|
||||
append(key, value.registrationModules);
|
||||
append(key, value.registrationWordLimit);
|
||||
append(key, value.vertexSubgroups);
|
||||
}
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -736,7 +736,7 @@ std::string RequestSerializer::Serialize(const RecompileRequest& request) const
|
|||
std::string buffer;
|
||||
Writer writer(buffer);
|
||||
writer.WriteU32(0x41505335u);
|
||||
writer.WriteU32(17u);
|
||||
writer.WriteU32(18u);
|
||||
writeShaderBinary(writer, request.shader);
|
||||
writeGuestContext(writer, request.context);
|
||||
writeSpirvTarget(writer, request.target);
|
||||
|
|
@ -758,6 +758,7 @@ std::string RequestSerializer::Serialize(const RecompileRequest& request) const
|
|||
writer.WriteBool(request.target.float64DenormPreserve);
|
||||
writer.WriteBool(request.target.registrationModules);
|
||||
writer.WriteU32(request.target.registrationWordLimit);
|
||||
writer.WriteBool(request.target.vertexSubgroups);
|
||||
return base64Encode(buffer);
|
||||
}
|
||||
|
||||
|
|
@ -766,7 +767,7 @@ DeserializedRequest RequestSerializer::Deserialize(std::string_view text) const
|
|||
Reader reader(decoded);
|
||||
if (reader.ReadU32() != 0x41505335u) throw std::runtime_error("invalid recompile request signature");
|
||||
const auto version = reader.ReadU32();
|
||||
if (version < 1u || version > 17u) throw std::runtime_error("unsupported recompile request serialization version");
|
||||
if (version < 1u || version > 18u) throw std::runtime_error("unsupported recompile request serialization version");
|
||||
DeserializedRequest result{};
|
||||
result.request.shader = readShaderBinary(reader, result.shaderCode, result.shaderHeader);
|
||||
result.request.context = readGuestContext(reader, result, version);
|
||||
|
|
@ -789,6 +790,7 @@ DeserializedRequest RequestSerializer::Deserialize(std::string_view text) const
|
|||
if (version >= 16u) result.request.target.float64DenormPreserve = reader.ReadBool();
|
||||
if (version >= 17u) result.request.target.registrationModules = reader.ReadBool();
|
||||
if (version >= 17u) result.request.target.registrationWordLimit = reader.ReadU32();
|
||||
if (version >= 18u) result.request.target.vertexSubgroups = reader.ReadBool();
|
||||
return result;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -85,6 +85,8 @@ namespace {
|
|||
// The host subgroup width wave64 programs are laid out for. Debug aid: APS5_SINGLE_LANE=<hex code
|
||||
// addresses, comma separated, or "all"> keeps the listed programs at one guest lane per invocation.
|
||||
std::uint32_t HostSubgroupSize(const RecompileRequest& request) {
|
||||
const bool vertexStage = request.shader.stage == ShaderStage::Vertex || request.shader.stage == ShaderStage::Local;
|
||||
if (vertexStage && !request.target.vertexSubgroups) return 1u;
|
||||
static const std::string list = [] { const char* text = std::getenv("APS5_SINGLE_LANE"); return text ? std::string(text) : std::string(); }();
|
||||
if (!list.empty()) {
|
||||
if (list == "all") return 64u;
|
||||
|
|
@ -424,7 +426,7 @@ CompiledVariant compileVariant(const RecompileRequest& request, IrProgram progra
|
|||
SpirvTargetOptions targetOptions {};
|
||||
targetOptions.vulkanVersion = request.target.vulkanVersion;
|
||||
targetOptions.spirvVersion = request.target.spirvVersion;
|
||||
targetOptions.subgroupSize = request.target.subgroupSize;
|
||||
targetOptions.subgroupSize = HostSubgroupSize(request) == 1u ? 1u : request.target.subgroupSize;
|
||||
targetOptions.bdaAbiVersion = request.target.bdaAbiVersion;
|
||||
targetOptions.supportedCapabilities = request.target.supportedCapabilities;
|
||||
targetOptions.supportedExtensions = request.target.supportedExtensions;
|
||||
|
|
|
|||
|
|
@ -232,6 +232,7 @@ struct SpirvTarget {
|
|||
bool float64DenormPreserve = false;
|
||||
bool registrationModules = false;
|
||||
std::uint32_t registrationWordLimit = 200000u;
|
||||
bool vertexSubgroups = true;
|
||||
};
|
||||
|
||||
struct BindingLayout {
|
||||
|
|
|
|||
|
|
@ -6,6 +6,11 @@
|
|||
|
||||
namespace ShaderRecompiler {
|
||||
|
||||
bool HostSubgroupSingleLane(const SpirvEmitterState& state);
|
||||
void EmitSubgroupBallot(SpirvEmitterState& state, std::uint32_t result, std::uint32_t predicate);
|
||||
void EmitSubgroupBallotFindLsb(SpirvEmitterState& state, std::uint32_t result, std::uint32_t ballot);
|
||||
void EmitSubgroupShuffle(SpirvEmitterState& state, std::uint32_t type, std::uint32_t result, std::uint32_t value, std::uint32_t lane);
|
||||
void EmitSubgroupElect(SpirvEmitterState& state, std::uint32_t result);
|
||||
std::uint32_t EmitSubgroupLocalInvocationId(SpirvEmitterState& state);
|
||||
std::uint32_t EmitHostSubgroupLane(SpirvEmitterState& state, std::uint32_t lane);
|
||||
std::uint32_t EmitWaveBallot(SpirvEmitterState& state, std::uint32_t ballot);
|
||||
|
|
|
|||
|
|
@ -188,9 +188,9 @@ std::uint32_t EmitDsMaskedLaneRead(SpirvEmitterState& state, std::uint32_t sourc
|
|||
}
|
||||
const auto physicalLane = EmitHostSubgroupLane(state, lane);
|
||||
const auto shuffled = state.module.AllocateId();
|
||||
state.module.AddFunction(spv::OpGroupNonUniformShuffle, TypeU32(state), shuffled, ConstantU32(state, spv::ScopeSubgroup), source, physicalLane);
|
||||
EmitSubgroupShuffle(state, TypeU32(state), shuffled, source, physicalLane);
|
||||
const auto sourceExec = state.module.AllocateId();
|
||||
state.module.AddFunction(spv::OpGroupNonUniformShuffle, TypeBool(state), sourceExec, ConstantU32(state, spv::ScopeSubgroup), exec, physicalLane);
|
||||
EmitSubgroupShuffle(state, TypeBool(state), sourceExec, exec, physicalLane);
|
||||
const auto sourceActive = Binary(state, spv::OpLogicalAnd, TypeBool(state), sourceExec, EmitSubgroupLaneActiveBool(state, lane));
|
||||
return Select(state, TypeU32(state), sourceActive, shuffled, ConstantU32(state, 0u));
|
||||
}
|
||||
|
|
@ -563,6 +563,7 @@ std::optional<std::uint32_t> EmitHalfWaveReduction(SpirvValueEmitContext& ctx, c
|
|||
if (lane < 32u || state.hostSubgroupSize > 32u) return std::nullopt;
|
||||
return ConstantU32(state, scan->reduction->identity);
|
||||
}
|
||||
if (HostSubgroupSingleLane(state)) return lane >= 32u ? ConstantU32(state, scan->reduction->identity) : ctx.Def(scan->source);
|
||||
const auto read = "v_readlane_b32 of lane " + std::to_string(lane) + " of a wave64 half-wave reduction scan";
|
||||
const auto& capabilities = state.supportedCapabilities;
|
||||
if (std::find(capabilities.begin(), capabilities.end(), static_cast<std::uint32_t>(spv::CapabilityGroupNonUniformArithmetic)) == capabilities.end()) {
|
||||
|
|
|
|||
|
|
@ -118,7 +118,7 @@ std::uint32_t SpirvValueEmitContext::Ballot(const IrValue* predicate) {
|
|||
const auto ballotType = TypeU32Vector(state, 4u);
|
||||
const auto scope = ConstantU32(state, spv::ScopeSubgroup);
|
||||
const auto low = state.module.AllocateId();
|
||||
state.module.AddFunction(spv::OpGroupNonUniformBallot, ballotType, low, scope, otherHalf == nullptr || half == 0u ? Def(predicate) : otherHalf->Def(predicate));
|
||||
EmitSubgroupBallot(state, low, otherHalf == nullptr || half == 0u ? Def(predicate) : otherHalf->Def(predicate));
|
||||
if (otherHalf == nullptr) {
|
||||
return EmitWaveBallot(state, low);
|
||||
}
|
||||
|
|
@ -136,7 +136,7 @@ std::uint32_t SpirvValueEmitContext::Ballot(const IrValue* predicate) {
|
|||
std::uint32_t SpirvValueEmitContext::FirstLane(std::uint32_t ballot) {
|
||||
if (otherHalf == nullptr) {
|
||||
const auto result = state.module.AllocateId();
|
||||
state.module.AddFunction(spv::OpGroupNonUniformBallotFindLSB, TypeU32(state), result, ConstantU32(state, spv::ScopeSubgroup), ballot);
|
||||
EmitSubgroupBallotFindLsb(state, result, ballot);
|
||||
return result;
|
||||
}
|
||||
const auto low = state.module.AllocateId();
|
||||
|
|
@ -159,7 +159,7 @@ std::uint32_t SpirvValueEmitContext::Shuffle(const IrValue& inst, std::size_t in
|
|||
const auto scope = ConstantU32(state, spv::ScopeSubgroup);
|
||||
const auto low = state.module.AllocateId();
|
||||
if (otherHalf == nullptr) {
|
||||
state.module.AddFunction(spv::OpGroupNonUniformShuffle, type, low, scope, Arg(inst, index), EmitHostSubgroupLane(state, lane));
|
||||
EmitSubgroupShuffle(state, type, low, Arg(inst, index), EmitHostSubgroupLane(state, lane));
|
||||
return low;
|
||||
}
|
||||
const auto physicalLane = EmitBinaryU32(state, spv::OpBitwiseAnd, lane, ConstantU32(state, 31u));
|
||||
|
|
@ -253,6 +253,11 @@ std::vector<std::uint32_t> SpirvEmitter::Emit(const IrProgram& program, const Sh
|
|||
state.registrationModules = target.registrationModules;
|
||||
state.fragmentDescriptorSet = target.fragmentDescriptorSet;
|
||||
state.hostSubgroupSize = target.subgroupSize;
|
||||
if (HostSubgroupSingleLane(state)) {
|
||||
state.requirements.subgroupBallot = false;
|
||||
state.requirements.subgroupShuffle = false;
|
||||
state.requirements.subgroupLocalInvocationId = false;
|
||||
}
|
||||
state.splitSubgroup = program.WaveSize() == 32u && target.subgroupSize > 32u;
|
||||
if (state.splitSubgroup && (state.requirements.subgroupBallot || state.requirements.subgroupShuffle)) state.requirements.subgroupLocalInvocationId = true;
|
||||
const auto* workgroup = ShaderWorkgroupInputFor(state);
|
||||
|
|
|
|||
|
|
@ -132,7 +132,7 @@ bool IsContinueTarget(const IrProgram& program, std::uint32_t block) {
|
|||
|
||||
std::uint32_t EmitWaveAny(SpirvEmitterState& state, std::uint32_t predicate) {
|
||||
const auto ballot = state.module.AllocateId();
|
||||
state.module.AddFunction(spv::OpGroupNonUniformBallot, TypeU32Vector(state, 4u), ballot, ConstantU32(state, spv::ScopeSubgroup), predicate);
|
||||
EmitSubgroupBallot(state, ballot, predicate);
|
||||
const auto wave = EmitWaveBallot(state, ballot);
|
||||
const auto low = state.module.AllocateId();
|
||||
const auto high = state.module.AllocateId();
|
||||
|
|
|
|||
|
|
@ -1181,7 +1181,7 @@ std::uint32_t AppendConsume(SpirvValueEmitContext& ctx, const IrValue& inst, boo
|
|||
return value;
|
||||
});
|
||||
const auto result = state.module.AllocateId();
|
||||
state.module.AddFunction(spv::OpGroupNonUniformShuffle, TypeU32(state), result, ConstantU32(state, spv::ScopeSubgroup), atomic, EmitHostSubgroupLane(state, sourceLane));
|
||||
EmitSubgroupShuffle(state, TypeU32(state), result, atomic, EmitHostSubgroupLane(state, sourceLane));
|
||||
return result;
|
||||
}
|
||||
|
||||
|
|
@ -1214,7 +1214,7 @@ std::uint32_t LockedLdsUpdate(SpirvEmitterState& state, std::uint32_t type, std:
|
|||
state.module.AddFunction(spv::OpBranchConditional, done, pendingMerge, pending);
|
||||
EmitLabel(state, pending);
|
||||
const auto elected = state.module.AllocateId();
|
||||
state.module.AddFunction(spv::OpGroupNonUniformElect, TypeBool(state), elected, ConstantU32(state, spv::ScopeSubgroup));
|
||||
EmitSubgroupElect(state, elected);
|
||||
state.module.AddFunction(spv::OpSelectionMerge, servedMerge, spv::SelectionControlMaskNone);
|
||||
state.module.AddFunction(spv::OpBranchConditional, elected, attempt, servedMerge);
|
||||
EmitLabel(state, attempt);
|
||||
|
|
@ -1244,7 +1244,7 @@ std::uint32_t LockedLdsUpdate(SpirvEmitterState& state, std::uint32_t type, std:
|
|||
state.module.AddFunction(spv::OpPhi, TypeBool(state), doneNext, served, servedMerge, done, body);
|
||||
state.module.AddFunction(spv::OpPhi, type, resultNext, servedResult, servedMerge, result, body);
|
||||
const auto ballot = state.module.AllocateId();
|
||||
state.module.AddFunction(spv::OpGroupNonUniformBallot, TypeU32Vector(state, 4u), ballot, ConstantU32(state, spv::ScopeSubgroup), Unary(state, spv::OpLogicalNot, TypeBool(state), doneNext));
|
||||
EmitSubgroupBallot(state, ballot, Unary(state, spv::OpLogicalNot, TypeBool(state), doneNext));
|
||||
std::uint32_t remaining = ConstantU32(state, 0u);
|
||||
for (std::uint32_t component = 0; component < 4u; ++component) {
|
||||
const auto word = state.module.AllocateId();
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#include "SpirvBackend/SpirvEmitter.hpp"
|
||||
#include "SpirvBackend/SpirvEmitterHelpers.hpp"
|
||||
#include "SpirvBackend/SpirvMemory/SpirvSubgroup.hpp"
|
||||
#include <spirv/unified1/GLSL.std.450.h>
|
||||
#include <spirv/unified1/spirv.hpp>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
|
|
@ -15,6 +17,7 @@ namespace ShaderRecompiler
|
|||
namespace {
|
||||
|
||||
std::uint32_t HostInvocationId(SpirvEmitterState& state) {
|
||||
if (HostSubgroupSingleLane(state)) return ConstantU32(state, 0u);
|
||||
if (state.subgroupLocalInvocationIdVariable == 0) {
|
||||
FailEmit("SubgroupLocalInvocationId was not declared before function emission");
|
||||
}
|
||||
|
|
@ -25,6 +28,46 @@ namespace ShaderRecompiler
|
|||
|
||||
}
|
||||
|
||||
bool HostSubgroupSingleLane(const SpirvEmitterState& state) {
|
||||
return state.hostSubgroupSize == 1u;
|
||||
}
|
||||
|
||||
void EmitSubgroupBallot(SpirvEmitterState& state, std::uint32_t result, std::uint32_t predicate) {
|
||||
if (HostSubgroupSingleLane(state)) {
|
||||
const auto zero = ConstantU32(state, 0u);
|
||||
const auto own = Select(state, TypeU32(state), predicate, ConstantU32(state, 1u), zero);
|
||||
state.module.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(state, 4u), result, own, zero, zero, zero);
|
||||
return;
|
||||
}
|
||||
state.module.AddFunction(spv::OpGroupNonUniformBallot, TypeU32Vector(state, 4u), result, ConstantU32(state, spv::ScopeSubgroup), predicate);
|
||||
}
|
||||
|
||||
void EmitSubgroupBallotFindLsb(SpirvEmitterState& state, std::uint32_t result, std::uint32_t ballot) {
|
||||
if (HostSubgroupSingleLane(state)) {
|
||||
const auto own = state.module.AllocateId();
|
||||
state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), own, ballot, 0u);
|
||||
state.module.AddFunction(spv::OpExtInst, TypeU32(state), result, GlslStd450(state), GLSLstd450FindILsb, own);
|
||||
return;
|
||||
}
|
||||
state.module.AddFunction(spv::OpGroupNonUniformBallotFindLSB, TypeU32(state), result, ConstantU32(state, spv::ScopeSubgroup), ballot);
|
||||
}
|
||||
|
||||
void EmitSubgroupShuffle(SpirvEmitterState& state, std::uint32_t type, std::uint32_t result, std::uint32_t value, std::uint32_t lane) {
|
||||
if (HostSubgroupSingleLane(state)) {
|
||||
state.module.AddFunction(spv::OpCopyObject, type, result, value);
|
||||
return;
|
||||
}
|
||||
state.module.AddFunction(spv::OpGroupNonUniformShuffle, type, result, ConstantU32(state, spv::ScopeSubgroup), value, lane);
|
||||
}
|
||||
|
||||
void EmitSubgroupElect(SpirvEmitterState& state, std::uint32_t result) {
|
||||
if (HostSubgroupSingleLane(state)) {
|
||||
state.module.AddFunction(spv::OpCopyObject, TypeBool(state), result, ConstantBool(state, true));
|
||||
return;
|
||||
}
|
||||
state.module.AddFunction(spv::OpGroupNonUniformElect, TypeBool(state), result, ConstantU32(state, spv::ScopeSubgroup));
|
||||
}
|
||||
|
||||
std::uint32_t EmitSubgroupLocalInvocationId(SpirvEmitterState& state) {
|
||||
const auto value = HostInvocationId(state);
|
||||
if (state.splitSubgroup) return EmitBinaryU32(state, spv::OpBitwiseAnd, value, ConstantU32(state, 31u));
|
||||
|
|
@ -184,7 +227,7 @@ namespace ShaderRecompiler
|
|||
|
||||
std::uint32_t EmitSubgroupLaneActiveBool(SpirvEmitterState& state, std::uint32_t lane) {
|
||||
const auto activeBallot = state.module.AllocateId();
|
||||
state.module.AddFunction(spv::OpGroupNonUniformBallot, TypeU32Vector(state, 4), activeBallot, ConstantU32(state, spv::ScopeSubgroup), ConstantBool(state, true));
|
||||
EmitSubgroupBallot(state, activeBallot, ConstantBool(state, true));
|
||||
return EmitBallotLaneActiveBool(state, EmitWaveBallot(state, activeBallot), lane);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -376,7 +376,8 @@ void emitEntryPrologue(IrProgram& program, IrBlock& entryBlock, const TranslateO
|
|||
}
|
||||
} else if (options.stage == ShaderStageKind::Vertex) {
|
||||
if (options.userDataBaseRegister >= 8u) {
|
||||
const std::uint32_t lanes = options.waveSize == 64u && options.inputInfo.vertex->hostSubgroupSize == 32u ? 32u : options.waveSize;
|
||||
const auto hostLanes = options.inputInfo.vertex->hostSubgroupSize;
|
||||
const std::uint32_t lanes = hostLanes == 1u ? 1u : options.waveSize == 64u && hostLanes == 32u ? 32u : options.waveSize;
|
||||
entryIr.SetScalarReg(static_cast<ScalarReg>(3), entryIr.Constant(lanes | (lanes << 8u)));
|
||||
}
|
||||
setVector(static_cast<VectorReg>(5), drawIndex(StageInputKind::VertexIndex));
|
||||
|
|
|
|||
|
|
@ -179,7 +179,7 @@ Throughout the project, every function at every stage either **does exactly what
|
|||
- [User data registers](../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Diagnostics.cpp) (AGC driver) - a `SPI_SHADER_USER_DATA_*` or `COMPUTE_USER_DATA_*` register the title never wrote reads as 0 when a stage's RSRC2 count covers it, as Kyty (zeroed `UserSgprInfo`) and SharpEmu (missing registers read 0) do, and as AMD PAL and Mesa initialize them; the console's value is not confirmed
|
||||
- [SET_PREDICATION](../../core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp) (AGC driver) - boolean predication follows the AMD PM4 encoding [Mesa](https://gitlab.freedesktop.org/mesa/mesa/-/blob/main/src/amd/common/sid.h) uses: operation 3 reads 64 bits and 4 reads 32 bits, and bit 8 runs predicated packets when the value is non-zero (clear: when it is zero); the bit 12 hint is ignored. The address must have its low 4 bits clear, as `sceAgcDcbSetPredication` writes it. Query predication (Z-pass, primitive count), a predicated COND_EXEC, predicated command buffer chains and flips in predicated command buffers throw
|
||||
- [sceAgcGetGsOversubscription](../../core/libs/prx/libSceAgc/Shader/src/GsOversubscription.cpp) (libSceAgc) - signature and occupancy formula (base and expanded vertex/export capacities, budget shift, `GE_PC_ALLOC` / `SPI_SHADER_PGM_RSRC4_GS` encoding) follow KytyPS5 only, not confirmed on hardware or a title; a negative or NaN factor that gives a negative target, a missing GS context register or a zero `GE_MAX_OUTPUT_PER_SUBGROUP` throws
|
||||
- [Vertex entry merged wave info](../../core/shader/recompiler/Translation/src/InstructionTranslator.cpp) (recompiler) - `s3` of every vertex-path shader is set to `waveSize | waveSize << 8` (vertex and primitive count of a full wave, wave index 0), or to `32 | 32 << 8` for a wave64 program on a 32-lane host subgroup, which runs it at one lane per invocation and holds no lanes 32-63. Layout taken from the prologue Mesa's ACO emits for NGG vertex shaders (`s_bfe_u64 exec` from `s3`), not measured on hardware. The driver gives every graphics program user data from `s8`, so the condition does not tell NGG from legacy vertex shaders: they just never read `s3`. A draw whose last wave is partial still reports a full wave, and so does a host wave with idle invocations (an RTX 5070 ran every vertex warp of a non-indexed triangle list with 30 of its 32 invocations).
|
||||
- [Vertex entry merged wave info](../../core/shader/recompiler/Translation/src/InstructionTranslator.cpp) (recompiler) - `s3` of every vertex-path shader is set to `waveSize | waveSize << 8` (vertex and primitive count of a full wave, wave index 0), or to `32 | 32 << 8` for a wave64 program on a 32-lane host subgroup, which runs it at one lane per invocation and holds no lanes 32-63. On a device whose `subgroupSupportedStages` lacks the vertex stage (lavapipe, MoltenVK), vertex and local programs run as one-lane waves: `s3` is `1 | 1 << 8`, the lane id is 0, a ballot is the invocation's own bit, a shuffle or broadcast returns the invocation's value, an elect is true and a subgroup reduction is the value itself, so no `GroupNonUniform` capability is declared. Layout taken from the prologue Mesa's ACO emits for NGG vertex shaders (`s_bfe_u64 exec` from `s3`), not measured on hardware. The driver gives every graphics program user data from `s8`, so the condition does not tell NGG from legacy vertex shaders: they just never read `s3`. A draw whose last wave is partial still reports a full wave, and so does a host wave with idle invocations (an RTX 5070 ran every vertex warp of a non-indexed triangle list with 30 of its 32 invocations).
|
||||
- [NGG vertex programs on the mesh path](../../core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp) (AGC driver) - an unregistered vertex program with `PRIMGEN_EN` that uses LDS or `s_barrier` runs on the geometry (mesh) path. Each input primitive is assumed to yield at most one output primitive, the subgroup is bounded by `GE_MAX_OUTPUT_PER_SUBGROUP` as a vertex budget, and the GS/ES VGPR count checks are skipped, because the merged inputs keep their positions. Taken from the code Mesa's ACO emits for NGG culling and from PPSA59322 and PPSA99010 draws, not measured on hardware. Registered programs keep the vertex path
|
||||
- [s_barrier in vertex-path programs](../../core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp) (recompiler) - emits no instruction when the program has no LDS or GDS access and no memory write or atomic, so no wave of the subgroup can observe another; the `GS_ALLOC_REQ` such a barrier orders on NGG passthrough programs is already dropped on the vertex path. The Vertex execution model has no workgroup barrier. Programs with LDS, GDS or memory writes throw. Not measured on hardware
|
||||
- [Strided buffer overhang](../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) (libSceAgcDriver) - a strided buffer descriptor that ends less than one stride past the end of the readable registered allocation holding its base is bound only up to that allocation's end, even when another readable allocation starts right there. This assumes the bytes past the end are the rounded-up `num_records` of the last record (RADV rounds it up), which no fetch inside the buffer reads; a guest that really keeps a vertex split across two adjacent allocations would read zeros for the cut tail. Taken from the RetroArch Dolphin core case in #2912, not measured on hardware.
|
||||
|
|
|
|||
Loading…
Reference in New Issue