shifathosn
/
KytyPS5
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

shader: preserve wave-wide vector-store completion before publication

This commit is contained in:
nmzik 2026-10-04 00:17:28 +04:00
parent 42a315f0e4
commit f58014ff0e
16 changed files with 144 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp
View File

@ -263,6 +263,7 @@ Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program
}
}
switch (inst.GetOpcode()) {
case IR::ValueOpcode::StoreCompletion: requirements.subgroup_barrier = true; break;
case IR::ValueOpcode::BvhIntersect: requirements.bvh = true; break;
case IR::ValueOpcode::Ballot: requirements.subgroup_ballot = true; break;
case IR::ValueOpcode::DppMoveU32:

8
src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp
View File

@ -574,6 +574,14 @@ uint32_t EmitIdentity(ValueEmitContext&, uint32_t value) {
void EmitVoid(ValueEmitContext&) {}
void EmitStoreCompletion(EmitterState& state) {
state.builder.AddFunction(
spv::OpControlBarrier, ConstantU32(state, spv::ScopeSubgroup),
ConstantU32(state, spv::ScopeDevice),
ConstantU32(state, spv::MemorySemanticsReleaseMask | spv::MemorySemanticsUniformMemoryMask |
spv::MemorySemanticsImageMemoryMask));
}
void EmitBarrier(EmitterState& state) {
const auto tessellation = state.program.stage == ShaderType::TessellationControl;
if (!tessellation && ShaderWorkgroupInput(state.program.stage, state.input_info) == nullptr) {

2
src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h
View File

@ -188,11 +188,11 @@ void EmitVoid(ValueEmitContext&);
inline constexpr auto EmitReference = EmitVoid;
inline constexpr auto EmitReferenceU32 = EmitVoid;
inline constexpr auto EmitControlNop = EmitVoid;
inline constexpr auto EmitWaitcnt = EmitVoid;
inline constexpr auto EmitSendmsg = EmitVoid;
inline constexpr auto EmitTtraceData = EmitVoid;
inline constexpr auto EmitInstPrefetch = EmitVoid;
void EmitBarrier(EmitterState& state);
void EmitStoreCompletion(EmitterState& state);
void EmitMeshAllocate(ValueEmitContext& ctx, const IR::Inst& inst);
uint32_t EmitMeshDrawParameter(ValueEmitContext& ctx, const IR::Inst& inst);
uint32_t EmitGetTessellationAttribute(ValueEmitContext& ctx, const IR::Inst& inst);

1
src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h
View File

@ -61,6 +61,7 @@ const ImageDimensionInfo& ImageDimensionInfoFor(ImageDimension dimension);
struct SpirvRequirements {
bool bvh = false;
bool subgroup_ballot = false;
bool subgroup_barrier = false;
bool subgroup_shuffle = false;
bool subgroup_local_invocation_id = false;
bool compute_derivatives = false;

5
src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp
View File

@ -701,8 +701,9 @@ void DefineModule(EmitterState& state) {
if (state.requirements.image_gather_extended) {
state.builder.RequireCapability(spv::CapabilityImageGatherExtended);
}
if (state.lane_count == 2 || state.requirements.subgroup_ballot ||
state.requirements.subgroup_shuffle || state.requirements.subgroup_local_invocation_id) {
if (state.lane_count == 2 || state.requirements.subgroup_barrier ||
state.requirements.subgroup_ballot || state.requirements.subgroup_shuffle ||
state.requirements.subgroup_local_invocation_id) {
state.builder.RequireCapability(spv::CapabilityGroupNonUniform);
}
if (state.lane_count == 2 || state.requirements.subgroup_ballot) {

1
src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp
View File

@ -315,6 +315,7 @@ void EmitBlock(ValueEmitContext& ctx, const IR::Block* block, EmitInstruction&&
auto& lane = half == 0 ? ctx : *ctx.other_half;
ctx.state.lane_half = half;
if (half == 0 || (inst.GetOpcode() != IR::ValueOpcode::Barrier &&
inst.GetOpcode() != IR::ValueOpcode::StoreCompletion &&
inst.GetOpcode() != IR::ValueOpcode::MeshAllocate)) {
emit_instruction(lane, inst);
}

10
src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp
View File

@ -86,7 +86,7 @@ constexpr OpcodeMap SOPK_OPCODE_LIST[] = {
{0x08u, Opcode::S_CMP_LE_I32}, {0x09u, Opcode::S_CMP_EQ_U32}, {0x0au, Opcode::S_CMP_LG_U32},
{0x0bu, Opcode::S_CMP_GT_U32}, {0x0cu, Opcode::S_CMP_GE_U32}, {0x0du, Opcode::S_CMP_LT_U32},
{0x0eu, Opcode::S_CMP_LE_U32}, {0x0fu, Opcode::S_ADD_I32}, {0x10u, Opcode::S_MULK_I32},
{0x13u, Opcode::S_SETREG_B32}, {0x17u, Opcode::S_WAITCNT}, {0x18u, Opcode::S_WAITCNT},
{0x13u, Opcode::S_SETREG_B32}, {0x17u, Opcode::S_WAITCNT_VSCNT}, {0x18u, Opcode::S_WAITCNT},
{0x19u, Opcode::S_WAITCNT}, {0x1au, Opcode::S_WAITCNT},
{0x1bu, Opcode::S_SUBVECTOR_LOOP_BEGIN}, {0x1cu, Opcode::S_SUBVECTOR_LOOP_END},
};
@ -234,6 +234,14 @@ void DecodeSopk(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
inst.src_count = 1;
return;
}
case Opcode::S_WAITCNT_VSCNT:
inst.dst.kind = OperandKind::Null;
inst.src1 = inst.src0;
inst.src1.value = word & 0xffffu;
inst.src1.signed_val = static_cast<int32_t>(inst.src1.value);
DecodeScalarSource(sdst, pc, inst.src0);
inst.src_count = 2;
return;
case Opcode::S_SETREG_B32:
inst.dst.kind = OperandKind::Null;
inst.src1.kind = OperandKind::LiteralConstant;

5
src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp
View File

@ -535,6 +535,11 @@ std::string InstructionToString(const Instruction& inst) {
return WithUnsupportedReason(inst, fmt::format("0x{:08x}: s_setreg_b32 {}, {}", inst.pc,
OperandToString(inst.src0).c_str(),
OperandToString(inst.src1).c_str()));
case Opcode::S_WAITCNT_VSCNT:
return WithUnsupportedReason(inst, fmt::format("0x{:08x}: {} {}, {}", inst.pc,
magic_enum::enum_name(inst.opcode),
OperandToString(inst.src0),
OperandToString(inst.src1)));
case Opcode::S_NOP:
case Opcode::S_WAITCNT:
case Opcode::S_WAITCNT_DEPCTR:

1
src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h
View File

@ -625,6 +625,7 @@ enum class Opcode {
S_NOP,
S_WAITCNT,
S_WAITCNT_VSCNT,
S_WAITCNT_DEPCTR,
S_BARRIER,
S_BRANCH,

8
src/graphics/shader/recompiler/frontend/translate/Control.cpp
View File

@ -181,8 +181,12 @@ void Translator::EmitControlNop() {
ir.Emit(IR::ValueOpcode::ControlNop);
}
void Translator::EmitWaitcnt() {
ir.Emit(IR::ValueOpcode::Waitcnt);
void Translator::S_WAITCNT_VSCNT(const Decoder::Instruction& inst) {
const auto count = inst.src1.value & 63u;
if (inst.src0.kind != Decoder::OperandKind::Null || (count != 0u && count != 63u)) {
EXIT("unsupported partial or register-based S_WAITCNT_VSCNT at 0x%08x", inst.pc);
}
if (count == 0u) ir.Emit(IR::ValueOpcode::StoreCompletion);
}
void Translator::S_BARRIER() {

5
src/graphics/shader/recompiler/frontend/translate/Scalar.cpp
View File

@ -18,7 +18,8 @@ void Translator::EmitScalar(const Decoder::Instruction& inst) {
case O::S_CSELECT_B64: ScalarSelect64(inst, inst.src1); return;
case O::S_CMOV_B64: ScalarSelect64(inst, inst.dst); return;
case O::S_SETREG_B32: EmitControlNop(); return;
case O::S_WAITCNT: EmitWaitcnt(); return;
case O::S_WAITCNT_VSCNT: S_WAITCNT_VSCNT(inst); return;
case O::S_WAITCNT: return;
case O::S_AND_SAVEEXEC_B32:
S_SAVEEXEC(inst, IR::ValueOpcode::LogicalAnd, false, false, false);
@ -224,7 +225,7 @@ void Translator::EmitScalar(const Decoder::Instruction& inst) {
case O::S_SLEEP:
case O::S_SETPRIO:
case O::S_TRAP: EmitControlNop(); return;
case O::S_WAITCNT_DEPCTR: EmitWaitcnt(); return;
case O::S_WAITCNT_DEPCTR: return;
case O::S_BARRIER: S_BARRIER(); return;
case O::S_SENDMSG: S_SENDMSG(inst); return;
case O::S_TTRACEDATA: S_TTRACEDATA(); return;

2
src/graphics/shader/recompiler/frontend/translate/Translator.h
View File

@ -229,7 +229,7 @@ private:
void ScalarMinMax32(const Decoder::Instruction& inst, IR::ValueOpcode value_opcode,
IR::ValueOpcode compare_opcode);
void EmitControlNop();
void EmitWaitcnt();
void S_WAITCNT_VSCNT(const Decoder::Instruction& inst);
void S_BARRIER();
void S_SENDMSG(const Decoder::Instruction& inst);
void S_TTRACEDATA();

1
src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp
View File

@ -93,6 +93,7 @@ bool HasSideEffects(ValueOpcode opcode) {
case ValueOpcode::SetAttribute:
case ValueOpcode::SetTessellationAttribute:
case ValueOpcode::MeshAllocate:
case ValueOpcode::StoreCompletion:
case ValueOpcode::Barrier: return true;
default: return false;
}

2
src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc
View File

@ -316,7 +316,7 @@ VALUE_OPCODE(GetInterpolationParameter, U32, U32, U32, U32)
VALUE_OPCODE(SetAttribute, Void, U32x4, U1)
VALUE_OPCODE(ControlNop, Void)
VALUE_OPCODE(Waitcnt, Void)
VALUE_OPCODE(StoreCompletion, Void)
VALUE_OPCODE(Barrier, Void)
VALUE_OPCODE(Sendmsg, Void)
VALUE_OPCODE(TtraceData, Void)

44
tests/ShaderRecompilerComputeTests.cpp
View File

@ -29003,11 +29003,14 @@ TestCase Wave64AppendConsumeHighHalf() {
return test;
}
TestCase BufferWorkgroupPublication(u32 wave_size, bool dlc_only = false) {
TestCase BufferWorkgroupPublication(u32 wave_size, bool dlc_only = false,
bool store_completion = false) {
using O = ShaderOpcode;
constexpr u32 groups = 31;
TestCase test;
test.name = dlc_only ? "BufferWorkgroupPublicationDlc"
test.name = store_completion ? (wave_size == 64 ? "Wave64StoreCompletionPublication"
: "Wave32StoreCompletionPublication")
: dlc_only ? "BufferWorkgroupPublicationDlc"
: wave_size == 64 ? "BufferWorkgroupPublicationWave64"
: "BufferWorkgroupPublicationWave32";
auto &code = test.code;
@ -29022,15 +29025,21 @@ TestCase BufferWorkgroupPublication(u32 wave_size, bool dlc_only = false) {
AppendVMovU32(&code, 2, 0);
code.push_back(EncodeSMovB32(21, InlineU32(0)));
AppendSMovLiteral(&code, 22, 1000000);
if (store_completion) AppendSMovLiteral(&code, 26, groups * 4);
code.push_back(EncodeSopc(0x06, 20, InlineU32(0)));
const auto first_ticket = code.size();
code.push_back(0);
const auto poll = code.size();
if (store_completion) {
code.push_back(EncodeVop2(0x25, 4, 26, 20));
AppendVMovU32(&code, 3, 0);
AppendBufferStoreOpcode(&code, 0x32, 3, 4, true);
}
code.push_back(EncodeMubuf0(0x0c, 0, false, true, !dlc_only) | (1u << 15u));
code.push_back(EncodeMubuf1(2, 12, 20));
code.push_back(EncodeSopp(0x0c, 0)); // S_WAITCNT vmcnt(0).
code.push_back(EncodeVop1(0x02, 23, Vgpr(2)));
code.push_back(EncodeVop1(0x02, 23, Vgpr(store_completion ? 3 : 2)));
code.push_back(EncodeSopc(0x07, 23, InlineU32(0)));
const auto ready = code.size();
code.push_back(0);
@ -29045,15 +29054,33 @@ TestCase BufferWorkgroupPublication(u32 wave_size, bool dlc_only = false) {
code.push_back(EncodeVop2(0x25, 2, Vgpr(1), 2));
code.push_back(EncodeVop2(0x25, 2, InlineU32(1), 2));
code.push_back(EncodeVop2(0x25, 20, InlineU32(4), 20));
if (store_completion) {
// The last lane writes the payload; lane zero publishes readiness after the
// native wave-wide store completion, including both emulated wave64 halves.
code.push_back(EncodeVop1(0x02, 24, Vgpr(2)));
code.push_back(EncodeVop1(0x02, 25, Vgpr(20)));
code.push_back(EncodeSop1(0x04, 126, 193u));
code.push_back(EncodeVop1(0x01, 2, 24));
code.push_back(EncodeVop1(0x01, 20, 25));
code.push_back(EncodeSop2(0x1f, 126, InlineU32(1), InlineU32(wave_size - 1)));
}
// RDNA2 stores publish to L2 even without GLC/DLC on the producer.
AppendBufferStoreDword(&code, 2, 20);
if (store_completion) {
code.push_back(0xbbfd0000u); // Captured S_WAITCNT_VSCNT null, 0 at 5b10:0x2464.
code.push_back(EncodeSop1(0x04, 126, InlineU32(1)));
code.push_back(EncodeVop2(0x25, 4, 26, 20));
AppendVMovU32(&code, 3, 1);
AppendBufferStoreOpcode(&code, 0x32, 3, 4, true);
}
AppendEnd(&code);
test.initial.assign(groups + 1, 0);
test.initial.assign(groups * (store_completion ? 2 : 1) + 1, 0);
test.expected = {groups};
for (u32 ticket = 0; ticket < groups; ++ticket) {
test.expected.push_back((ticket + 1) * (ticket + 2) / 2);
}
if (store_completion) test.expected.resize(groups * 2 + 1, 1);
test.opcodes = {O::S_MOV_B64, O::S_MOV_B32, O::V_MOV_B32,
O::BUFFER_ATOMIC_ADD, O::V_READFIRSTLANE_B32,
O::V_LSHLREV_B32, O::S_CMP_EQ_U32, O::S_CMP_LG_U32,
@ -29061,6 +29088,11 @@ TestCase BufferWorkgroupPublication(u32 wave_size, bool dlc_only = false) {
O::S_WAITCNT, O::BUFFER_LOAD_DWORD, O::V_ADD_NC_U32,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"Coherent", "Volatile", "OpAtomicIAdd", "OpLoopMerge"};
if (store_completion) {
test.opcodes.push_back(O::S_WAITCNT_VSCNT);
test.opcodes.push_back(O::S_LSHL_B64);
test.required_spirv.push_back("OpControlBarrier %uint_3 %uint_1 %uint_2116");
}
test.compute_info.threads_num[0] = wave_size;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
@ -32085,6 +32117,8 @@ std::vector<TestCase> MakeCases() {
cases.push_back(BufferWorkgroupPublication(32));
cases.push_back(BufferWorkgroupPublication(64));
cases.push_back(BufferWorkgroupPublication(32, true));
cases.push_back(BufferWorkgroupPublication(32, false, true));
cases.push_back(BufferWorkgroupPublication(64, false, true));
AddCase(IntegerAddSubMul);
AddCase(BitwiseOps);
@ -37540,6 +37574,8 @@ int main(int argc, char **argv) {
RunCase(&vulkan, BufferWorkgroupPublication(32));
RunCase(&vulkan, BufferWorkgroupPublication(64));
RunCase(&vulkan, BufferWorkgroupPublication(32, true));
RunCase(&vulkan, BufferWorkgroupPublication(32, false, true));
RunCase(&vulkan, BufferWorkgroupPublication(64, false, true));
RunCase(&vulkan, BufferAtomicVariants());
return 0;
}

80
tests/shaderCfgTests.cpp
View File

@ -1817,28 +1817,71 @@ void TestNggVertexEntryState() {
}
}
void TestNewShaderRecompilerSopkWaitcntMarkers() {
const uint32_t shader[] = {
EncodeSopk(0x17, 125, 0xffff), // s_waitcnt_vscnt null, 0xffff
EncodeSopk(0x18, 125, 0), // s_waitcnt_vmcnt null, 0
EncodeSopk(0x19, 125, 0), // s_waitcnt_expcnt null, 0
EncodeSopk(0x1a, 125, 0), // s_waitcnt_lgkmcnt null, 0
EncodeSopp(0x01, 0), // s_endpgm
};
void TestNativeStoreCompletion() {
using namespace ShaderRecompiler;
const std::array register_wait = {EncodeSopk(0x17, 4, static_cast<int16_t>(0x9234)), EncodeSopp(0x01)};
Decoder::Program decoded;
Decoder::DecodeProgram(register_wait, decoded);
const auto &wait = decoded.instructions.front();
Check(wait.opcode == Decoder::Opcode::S_WAITCNT_VSCNT &&
wait.src0.kind == Decoder::OperandKind::Sgpr && wait.src0.reg == 4 &&
wait.src1.value == 0x9234u,
"store completion lost its counter kind, source, or unsigned immediate");
auto options = MakeCompileOptions(ShaderType::Compute);
options.dump_ir = true;
ShaderComputeInputInfo compute{};
compute.threads_num[1] = compute.threads_num[2] = 1;
options.input_info.compute = &compute;
for (const uint32_t wave_size : {32u, 64u}) {
compute.threads_num[0] = compute.wave_size = options.wave_size = wave_size;
const std::array barrier_only = {0xbbfd0000u, EncodeSopp(0x01)};
const auto minimal = RecompileForTest(barrier_only, options);
CheckSpirvBinaryValidates(minimal.spirv);
Check(SpirvContainsCapability(minimal.spirv, 61),
"standalone store completion lacks GroupNonUniform capability");
auto result = RecompileForTest(shader, options);
Check((result.decoded_dump.find("s_waitcnt 0") != std::string::npos),
"new decoder did not decode SOPK waitcnt marker");
Check((result.decoded_dump.find("s_waitcnt 65535") != std::string::npos),
"SOPK waitcnt marker immediate was not kept unsigned");
Check((result.ir_dump.find("Waitcnt null, 0x00000000") != std::string::npos),
"SOPK waitcnt did not lower to an IR marker");
Check((result.ir_dump.find("Waitcnt null, 0x0000ffff") != std::string::npos),
"SOPK waitcnt marker immediate was not translated as 16-bit unsigned");
CheckSpirvBinaryValidates(result.spirv);
// Captured 5b10:0x2464 drains node stores before the next parent atomic.
const std::array publication = {
EncodeVop1(0x01, 1, 129),
EncodeMubuf0(0x1c, 0, false), EncodeMubuf1(1, 0, 0),
0xbbfd0000u,
EncodeMubuf0(0x32, 4, false, true), EncodeMubuf1(1, 0, 0),
EncodeSopp(0x01),
};
const auto result = RecompileForTest(publication, options);
CheckSpirvBinaryValidates(result.spirv);
const auto store = result.ir_dump.find("StoreBufferU32");
const auto completion = result.ir_dump.find("StoreCompletion");
const auto atomic = result.ir_dump.find("BufferAtomicIAdd32");
Check(store < completion && completion < atomic && atomic != std::string::npos,
"store completion moved outside its native publication order");
const auto source = DisassembleSpirvBinary(result.spirv);
const auto barrier = source.find("OpControlBarrier %uint_3 %uint_1 %uint_2116");
Check(barrier != std::string::npos &&
source.find("OpStore") < barrier &&
barrier < source.find("OpAtomicIAdd") &&
source.find("OpControlBarrier", barrier + 1) == std::string::npos,
"store completion must synchronize the subgroup once after both wave halves");
}
const std::array no_store_wait = {
EncodeSopk(0x17, 125, 0xffff), // Effective VSCNT 63 waits for nothing.
EncodeSopk(0x18, 125, 0), EncodeSopk(0x19, 125, 0),
EncodeSopk(0x1a, 125, 0), EncodeSopp(0x01),
};
const auto no_wait = RecompileForTest(no_store_wait, options);
const auto source = DisassembleSpirvBinary(no_wait.spirv);
Check(source.find("OpControlBarrier") == std::string::npos &&
source.find("OpMemoryBarrier") == std::string::npos,
"other wait counters or VSCNT 63 became a store completion barrier");
#ifndef _WIN32
for (const uint32_t word : {EncodeSopk(0x17, 125, 1), register_wait[0]}) {
const std::array unsupported = {word, EncodeSopp(0x01)};
ExpectFatal([&] { (void)RecompileForTest(unsupported, options); },
"unsupported partial or register-based store wait was dropped");
}
#endif
}
void TestSopkCompareImmediateExtension() {
@ -14444,6 +14487,7 @@ int main() {
TestNewShaderDecoderArchitecture();
TestImageAddressOperands();
TestSopkCompareImmediateExtension();
TestNativeStoreCompletion();
TestNewShaderRecompilerCapturedVopcSdwaCmpxClass();
TestNewShaderRecompilerCapturedVopcSdwaCmpxLtU16();
TestNewShaderRecompilerIrLookupMissFailsExplicitly();