From f58014ff0e60c8e0352702c7ef2778c8ee18331d Mon Sep 17 00:00:00 2001 From: nmzik Date: Sun, 4 Oct 2026 00:17:28 +0400 Subject: [PATCH] shader: preserve wave-wide vector-store completion before publication --- .../recompiler/backend/spirv/SpirvEmitter.cpp | 1 + .../backend/spirv/spirvEmitterFlow.cpp | 8 ++ .../backend/spirv/spirvEmitterInstructions.h | 2 +- .../backend/spirv/spirvEmitterInternal.h | 1 + .../backend/spirv/spirvEmitterModule.cpp | 5 +- .../backend/spirv/spirvEmitterProgram.cpp | 1 + .../frontend/decode/ScalarAluOps.cpp | 10 ++- .../frontend/decode/ShaderDecoder.cpp | 5 ++ .../frontend/decode/ShaderDecoder.h | 1 + .../recompiler/frontend/translate/Control.cpp | 8 +- .../recompiler/frontend/translate/Scalar.cpp | 5 +- .../frontend/translate/Translator.h | 2 +- .../recompiler/ir/opcodes/ValueOpcodes.cpp | 1 + .../recompiler/ir/opcodes/ValueOpcodes.inc | 2 +- tests/ShaderRecompilerComputeTests.cpp | 44 +++++++++- tests/shaderCfgTests.cpp | 80 ++++++++++++++----- 16 files changed, 144 insertions(+), 32 deletions(-) diff --git a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp index 8b13bfbb..ab6e666e 100644 --- a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp @@ -263,6 +263,7 @@ Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program } } switch (inst.GetOpcode()) { + case IR::ValueOpcode::StoreCompletion: requirements.subgroup_barrier = true; break; case IR::ValueOpcode::BvhIntersect: requirements.bvh = true; break; case IR::ValueOpcode::Ballot: requirements.subgroup_ballot = true; break; case IR::ValueOpcode::DppMoveU32: diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp index d8bd9571..68d23aea 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp @@ -574,6 +574,14 @@ uint32_t EmitIdentity(ValueEmitContext&, uint32_t value) { void EmitVoid(ValueEmitContext&) {} +void EmitStoreCompletion(EmitterState& state) { + state.builder.AddFunction( + spv::OpControlBarrier, ConstantU32(state, spv::ScopeSubgroup), + ConstantU32(state, spv::ScopeDevice), + ConstantU32(state, spv::MemorySemanticsReleaseMask | spv::MemorySemanticsUniformMemoryMask | + spv::MemorySemanticsImageMemoryMask)); +} + void EmitBarrier(EmitterState& state) { const auto tessellation = state.program.stage == ShaderType::TessellationControl; if (!tessellation && ShaderWorkgroupInput(state.program.stage, state.input_info) == nullptr) { diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h index c6f3a4d9..35a7b781 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h @@ -188,11 +188,11 @@ void EmitVoid(ValueEmitContext&); inline constexpr auto EmitReference = EmitVoid; inline constexpr auto EmitReferenceU32 = EmitVoid; inline constexpr auto EmitControlNop = EmitVoid; -inline constexpr auto EmitWaitcnt = EmitVoid; inline constexpr auto EmitSendmsg = EmitVoid; inline constexpr auto EmitTtraceData = EmitVoid; inline constexpr auto EmitInstPrefetch = EmitVoid; void EmitBarrier(EmitterState& state); +void EmitStoreCompletion(EmitterState& state); void EmitMeshAllocate(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitMeshDrawParameter(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitGetTessellationAttribute(ValueEmitContext& ctx, const IR::Inst& inst); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h index 3b0dce75..0ea16b94 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h @@ -61,6 +61,7 @@ const ImageDimensionInfo& ImageDimensionInfoFor(ImageDimension dimension); struct SpirvRequirements { bool bvh = false; bool subgroup_ballot = false; + bool subgroup_barrier = false; bool subgroup_shuffle = false; bool subgroup_local_invocation_id = false; bool compute_derivatives = false; diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp index 771eccdd..cc5b1edc 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp @@ -701,8 +701,9 @@ void DefineModule(EmitterState& state) { if (state.requirements.image_gather_extended) { state.builder.RequireCapability(spv::CapabilityImageGatherExtended); } - if (state.lane_count == 2 || state.requirements.subgroup_ballot || - state.requirements.subgroup_shuffle || state.requirements.subgroup_local_invocation_id) { + if (state.lane_count == 2 || state.requirements.subgroup_barrier || + state.requirements.subgroup_ballot || state.requirements.subgroup_shuffle || + state.requirements.subgroup_local_invocation_id) { state.builder.RequireCapability(spv::CapabilityGroupNonUniform); } if (state.lane_count == 2 || state.requirements.subgroup_ballot) { diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp index 7aed831a..ac78acd7 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp @@ -315,6 +315,7 @@ void EmitBlock(ValueEmitContext& ctx, const IR::Block* block, EmitInstruction&& auto& lane = half == 0 ? ctx : *ctx.other_half; ctx.state.lane_half = half; if (half == 0 || (inst.GetOpcode() != IR::ValueOpcode::Barrier && + inst.GetOpcode() != IR::ValueOpcode::StoreCompletion && inst.GetOpcode() != IR::ValueOpcode::MeshAllocate)) { emit_instruction(lane, inst); } diff --git a/src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp b/src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp index 6c8b7bb8..3a6f084c 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp @@ -86,7 +86,7 @@ constexpr OpcodeMap SOPK_OPCODE_LIST[] = { {0x08u, Opcode::S_CMP_LE_I32}, {0x09u, Opcode::S_CMP_EQ_U32}, {0x0au, Opcode::S_CMP_LG_U32}, {0x0bu, Opcode::S_CMP_GT_U32}, {0x0cu, Opcode::S_CMP_GE_U32}, {0x0du, Opcode::S_CMP_LT_U32}, {0x0eu, Opcode::S_CMP_LE_U32}, {0x0fu, Opcode::S_ADD_I32}, {0x10u, Opcode::S_MULK_I32}, - {0x13u, Opcode::S_SETREG_B32}, {0x17u, Opcode::S_WAITCNT}, {0x18u, Opcode::S_WAITCNT}, + {0x13u, Opcode::S_SETREG_B32}, {0x17u, Opcode::S_WAITCNT_VSCNT}, {0x18u, Opcode::S_WAITCNT}, {0x19u, Opcode::S_WAITCNT}, {0x1au, Opcode::S_WAITCNT}, {0x1bu, Opcode::S_SUBVECTOR_LOOP_BEGIN}, {0x1cu, Opcode::S_SUBVECTOR_LOOP_END}, }; @@ -234,6 +234,14 @@ void DecodeSopk(uint32_t pc, std::span code, uint32_t word_index inst.src_count = 1; return; } + case Opcode::S_WAITCNT_VSCNT: + inst.dst.kind = OperandKind::Null; + inst.src1 = inst.src0; + inst.src1.value = word & 0xffffu; + inst.src1.signed_val = static_cast(inst.src1.value); + DecodeScalarSource(sdst, pc, inst.src0); + inst.src_count = 2; + return; case Opcode::S_SETREG_B32: inst.dst.kind = OperandKind::Null; inst.src1.kind = OperandKind::LiteralConstant; diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp index c933ae72..6a115a9d 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp @@ -535,6 +535,11 @@ std::string InstructionToString(const Instruction& inst) { return WithUnsupportedReason(inst, fmt::format("0x{:08x}: s_setreg_b32 {}, {}", inst.pc, OperandToString(inst.src0).c_str(), OperandToString(inst.src1).c_str())); + case Opcode::S_WAITCNT_VSCNT: + return WithUnsupportedReason(inst, fmt::format("0x{:08x}: {} {}, {}", inst.pc, + magic_enum::enum_name(inst.opcode), + OperandToString(inst.src0), + OperandToString(inst.src1))); case Opcode::S_NOP: case Opcode::S_WAITCNT: case Opcode::S_WAITCNT_DEPCTR: diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h index 43b65528..cade969e 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h @@ -625,6 +625,7 @@ enum class Opcode { S_NOP, S_WAITCNT, + S_WAITCNT_VSCNT, S_WAITCNT_DEPCTR, S_BARRIER, S_BRANCH, diff --git a/src/graphics/shader/recompiler/frontend/translate/Control.cpp b/src/graphics/shader/recompiler/frontend/translate/Control.cpp index b3ae0b66..0f4269e3 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Control.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Control.cpp @@ -181,8 +181,12 @@ void Translator::EmitControlNop() { ir.Emit(IR::ValueOpcode::ControlNop); } -void Translator::EmitWaitcnt() { - ir.Emit(IR::ValueOpcode::Waitcnt); +void Translator::S_WAITCNT_VSCNT(const Decoder::Instruction& inst) { + const auto count = inst.src1.value & 63u; + if (inst.src0.kind != Decoder::OperandKind::Null || (count != 0u && count != 63u)) { + EXIT("unsupported partial or register-based S_WAITCNT_VSCNT at 0x%08x", inst.pc); + } + if (count == 0u) ir.Emit(IR::ValueOpcode::StoreCompletion); } void Translator::S_BARRIER() { diff --git a/src/graphics/shader/recompiler/frontend/translate/Scalar.cpp b/src/graphics/shader/recompiler/frontend/translate/Scalar.cpp index ecd38384..f477ad9f 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Scalar.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Scalar.cpp @@ -18,7 +18,8 @@ void Translator::EmitScalar(const Decoder::Instruction& inst) { case O::S_CSELECT_B64: ScalarSelect64(inst, inst.src1); return; case O::S_CMOV_B64: ScalarSelect64(inst, inst.dst); return; case O::S_SETREG_B32: EmitControlNop(); return; - case O::S_WAITCNT: EmitWaitcnt(); return; + case O::S_WAITCNT_VSCNT: S_WAITCNT_VSCNT(inst); return; + case O::S_WAITCNT: return; case O::S_AND_SAVEEXEC_B32: S_SAVEEXEC(inst, IR::ValueOpcode::LogicalAnd, false, false, false); @@ -224,7 +225,7 @@ void Translator::EmitScalar(const Decoder::Instruction& inst) { case O::S_SLEEP: case O::S_SETPRIO: case O::S_TRAP: EmitControlNop(); return; - case O::S_WAITCNT_DEPCTR: EmitWaitcnt(); return; + case O::S_WAITCNT_DEPCTR: return; case O::S_BARRIER: S_BARRIER(); return; case O::S_SENDMSG: S_SENDMSG(inst); return; case O::S_TTRACEDATA: S_TTRACEDATA(); return; diff --git a/src/graphics/shader/recompiler/frontend/translate/Translator.h b/src/graphics/shader/recompiler/frontend/translate/Translator.h index 76a1ddc8..298c4c28 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Translator.h +++ b/src/graphics/shader/recompiler/frontend/translate/Translator.h @@ -229,7 +229,7 @@ private: void ScalarMinMax32(const Decoder::Instruction& inst, IR::ValueOpcode value_opcode, IR::ValueOpcode compare_opcode); void EmitControlNop(); - void EmitWaitcnt(); + void S_WAITCNT_VSCNT(const Decoder::Instruction& inst); void S_BARRIER(); void S_SENDMSG(const Decoder::Instruction& inst); void S_TTRACEDATA(); diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp index 767767b7..4e8a43a4 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp @@ -93,6 +93,7 @@ bool HasSideEffects(ValueOpcode opcode) { case ValueOpcode::SetAttribute: case ValueOpcode::SetTessellationAttribute: case ValueOpcode::MeshAllocate: + case ValueOpcode::StoreCompletion: case ValueOpcode::Barrier: return true; default: return false; } diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc index 986f8c93..a50d2695 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc @@ -316,7 +316,7 @@ VALUE_OPCODE(GetInterpolationParameter, U32, U32, U32, U32) VALUE_OPCODE(SetAttribute, Void, U32x4, U1) VALUE_OPCODE(ControlNop, Void) -VALUE_OPCODE(Waitcnt, Void) +VALUE_OPCODE(StoreCompletion, Void) VALUE_OPCODE(Barrier, Void) VALUE_OPCODE(Sendmsg, Void) VALUE_OPCODE(TtraceData, Void) diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 87e9ceb0..7ae5d77a 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -29003,11 +29003,14 @@ TestCase Wave64AppendConsumeHighHalf() { return test; } -TestCase BufferWorkgroupPublication(u32 wave_size, bool dlc_only = false) { +TestCase BufferWorkgroupPublication(u32 wave_size, bool dlc_only = false, + bool store_completion = false) { using O = ShaderOpcode; constexpr u32 groups = 31; TestCase test; - test.name = dlc_only ? "BufferWorkgroupPublicationDlc" + test.name = store_completion ? (wave_size == 64 ? "Wave64StoreCompletionPublication" + : "Wave32StoreCompletionPublication") + : dlc_only ? "BufferWorkgroupPublicationDlc" : wave_size == 64 ? "BufferWorkgroupPublicationWave64" : "BufferWorkgroupPublicationWave32"; auto &code = test.code; @@ -29022,15 +29025,21 @@ TestCase BufferWorkgroupPublication(u32 wave_size, bool dlc_only = false) { AppendVMovU32(&code, 2, 0); code.push_back(EncodeSMovB32(21, InlineU32(0))); AppendSMovLiteral(&code, 22, 1000000); + if (store_completion) AppendSMovLiteral(&code, 26, groups * 4); code.push_back(EncodeSopc(0x06, 20, InlineU32(0))); const auto first_ticket = code.size(); code.push_back(0); const auto poll = code.size(); + if (store_completion) { + code.push_back(EncodeVop2(0x25, 4, 26, 20)); + AppendVMovU32(&code, 3, 0); + AppendBufferStoreOpcode(&code, 0x32, 3, 4, true); + } code.push_back(EncodeMubuf0(0x0c, 0, false, true, !dlc_only) | (1u << 15u)); code.push_back(EncodeMubuf1(2, 12, 20)); code.push_back(EncodeSopp(0x0c, 0)); // S_WAITCNT vmcnt(0). - code.push_back(EncodeVop1(0x02, 23, Vgpr(2))); + code.push_back(EncodeVop1(0x02, 23, Vgpr(store_completion ? 3 : 2))); code.push_back(EncodeSopc(0x07, 23, InlineU32(0))); const auto ready = code.size(); code.push_back(0); @@ -29045,15 +29054,33 @@ TestCase BufferWorkgroupPublication(u32 wave_size, bool dlc_only = false) { code.push_back(EncodeVop2(0x25, 2, Vgpr(1), 2)); code.push_back(EncodeVop2(0x25, 2, InlineU32(1), 2)); code.push_back(EncodeVop2(0x25, 20, InlineU32(4), 20)); + if (store_completion) { + // The last lane writes the payload; lane zero publishes readiness after the + // native wave-wide store completion, including both emulated wave64 halves. + code.push_back(EncodeVop1(0x02, 24, Vgpr(2))); + code.push_back(EncodeVop1(0x02, 25, Vgpr(20))); + code.push_back(EncodeSop1(0x04, 126, 193u)); + code.push_back(EncodeVop1(0x01, 2, 24)); + code.push_back(EncodeVop1(0x01, 20, 25)); + code.push_back(EncodeSop2(0x1f, 126, InlineU32(1), InlineU32(wave_size - 1))); + } // RDNA2 stores publish to L2 even without GLC/DLC on the producer. AppendBufferStoreDword(&code, 2, 20); + if (store_completion) { + code.push_back(0xbbfd0000u); // Captured S_WAITCNT_VSCNT null, 0 at 5b10:0x2464. + code.push_back(EncodeSop1(0x04, 126, InlineU32(1))); + code.push_back(EncodeVop2(0x25, 4, 26, 20)); + AppendVMovU32(&code, 3, 1); + AppendBufferStoreOpcode(&code, 0x32, 3, 4, true); + } AppendEnd(&code); - test.initial.assign(groups + 1, 0); + test.initial.assign(groups * (store_completion ? 2 : 1) + 1, 0); test.expected = {groups}; for (u32 ticket = 0; ticket < groups; ++ticket) { test.expected.push_back((ticket + 1) * (ticket + 2) / 2); } + if (store_completion) test.expected.resize(groups * 2 + 1, 1); test.opcodes = {O::S_MOV_B64, O::S_MOV_B32, O::V_MOV_B32, O::BUFFER_ATOMIC_ADD, O::V_READFIRSTLANE_B32, O::V_LSHLREV_B32, O::S_CMP_EQ_U32, O::S_CMP_LG_U32, @@ -29061,6 +29088,11 @@ TestCase BufferWorkgroupPublication(u32 wave_size, bool dlc_only = false) { O::S_WAITCNT, O::BUFFER_LOAD_DWORD, O::V_ADD_NC_U32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; test.required_spirv = {"Coherent", "Volatile", "OpAtomicIAdd", "OpLoopMerge"}; + if (store_completion) { + test.opcodes.push_back(O::S_WAITCNT_VSCNT); + test.opcodes.push_back(O::S_LSHL_B64); + test.required_spirv.push_back("OpControlBarrier %uint_3 %uint_1 %uint_2116"); + } test.compute_info.threads_num[0] = wave_size; test.compute_info.threads_num[1] = 1; test.compute_info.threads_num[2] = 1; @@ -32085,6 +32117,8 @@ std::vector MakeCases() { cases.push_back(BufferWorkgroupPublication(32)); cases.push_back(BufferWorkgroupPublication(64)); cases.push_back(BufferWorkgroupPublication(32, true)); + cases.push_back(BufferWorkgroupPublication(32, false, true)); + cases.push_back(BufferWorkgroupPublication(64, false, true)); AddCase(IntegerAddSubMul); AddCase(BitwiseOps); @@ -37540,6 +37574,8 @@ int main(int argc, char **argv) { RunCase(&vulkan, BufferWorkgroupPublication(32)); RunCase(&vulkan, BufferWorkgroupPublication(64)); RunCase(&vulkan, BufferWorkgroupPublication(32, true)); + RunCase(&vulkan, BufferWorkgroupPublication(32, false, true)); + RunCase(&vulkan, BufferWorkgroupPublication(64, false, true)); RunCase(&vulkan, BufferAtomicVariants()); return 0; } diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index 422ef515..581ca6da 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -1817,28 +1817,71 @@ void TestNggVertexEntryState() { } } -void TestNewShaderRecompilerSopkWaitcntMarkers() { - const uint32_t shader[] = { - EncodeSopk(0x17, 125, 0xffff), // s_waitcnt_vscnt null, 0xffff - EncodeSopk(0x18, 125, 0), // s_waitcnt_vmcnt null, 0 - EncodeSopk(0x19, 125, 0), // s_waitcnt_expcnt null, 0 - EncodeSopk(0x1a, 125, 0), // s_waitcnt_lgkmcnt null, 0 - EncodeSopp(0x01, 0), // s_endpgm - }; +void TestNativeStoreCompletion() { + using namespace ShaderRecompiler; + const std::array register_wait = {EncodeSopk(0x17, 4, static_cast(0x9234)), EncodeSopp(0x01)}; + Decoder::Program decoded; + Decoder::DecodeProgram(register_wait, decoded); + const auto &wait = decoded.instructions.front(); + Check(wait.opcode == Decoder::Opcode::S_WAITCNT_VSCNT && + wait.src0.kind == Decoder::OperandKind::Sgpr && wait.src0.reg == 4 && + wait.src1.value == 0x9234u, + "store completion lost its counter kind, source, or unsigned immediate"); auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; + ShaderComputeInputInfo compute{}; + compute.threads_num[1] = compute.threads_num[2] = 1; + options.input_info.compute = &compute; + for (const uint32_t wave_size : {32u, 64u}) { + compute.threads_num[0] = compute.wave_size = options.wave_size = wave_size; + const std::array barrier_only = {0xbbfd0000u, EncodeSopp(0x01)}; + const auto minimal = RecompileForTest(barrier_only, options); + CheckSpirvBinaryValidates(minimal.spirv); + Check(SpirvContainsCapability(minimal.spirv, 61), + "standalone store completion lacks GroupNonUniform capability"); - auto result = RecompileForTest(shader, options); - Check((result.decoded_dump.find("s_waitcnt 0") != std::string::npos), - "new decoder did not decode SOPK waitcnt marker"); - Check((result.decoded_dump.find("s_waitcnt 65535") != std::string::npos), - "SOPK waitcnt marker immediate was not kept unsigned"); - Check((result.ir_dump.find("Waitcnt null, 0x00000000") != std::string::npos), - "SOPK waitcnt did not lower to an IR marker"); - Check((result.ir_dump.find("Waitcnt null, 0x0000ffff") != std::string::npos), - "SOPK waitcnt marker immediate was not translated as 16-bit unsigned"); - CheckSpirvBinaryValidates(result.spirv); + // Captured 5b10:0x2464 drains node stores before the next parent atomic. + const std::array publication = { + EncodeVop1(0x01, 1, 129), + EncodeMubuf0(0x1c, 0, false), EncodeMubuf1(1, 0, 0), + 0xbbfd0000u, + EncodeMubuf0(0x32, 4, false, true), EncodeMubuf1(1, 0, 0), + EncodeSopp(0x01), + }; + const auto result = RecompileForTest(publication, options); + CheckSpirvBinaryValidates(result.spirv); + const auto store = result.ir_dump.find("StoreBufferU32"); + const auto completion = result.ir_dump.find("StoreCompletion"); + const auto atomic = result.ir_dump.find("BufferAtomicIAdd32"); + Check(store < completion && completion < atomic && atomic != std::string::npos, + "store completion moved outside its native publication order"); + const auto source = DisassembleSpirvBinary(result.spirv); + const auto barrier = source.find("OpControlBarrier %uint_3 %uint_1 %uint_2116"); + Check(barrier != std::string::npos && + source.find("OpStore") < barrier && + barrier < source.find("OpAtomicIAdd") && + source.find("OpControlBarrier", barrier + 1) == std::string::npos, + "store completion must synchronize the subgroup once after both wave halves"); + } + + const std::array no_store_wait = { + EncodeSopk(0x17, 125, 0xffff), // Effective VSCNT 63 waits for nothing. + EncodeSopk(0x18, 125, 0), EncodeSopk(0x19, 125, 0), + EncodeSopk(0x1a, 125, 0), EncodeSopp(0x01), + }; + const auto no_wait = RecompileForTest(no_store_wait, options); + const auto source = DisassembleSpirvBinary(no_wait.spirv); + Check(source.find("OpControlBarrier") == std::string::npos && + source.find("OpMemoryBarrier") == std::string::npos, + "other wait counters or VSCNT 63 became a store completion barrier"); +#ifndef _WIN32 + for (const uint32_t word : {EncodeSopk(0x17, 125, 1), register_wait[0]}) { + const std::array unsupported = {word, EncodeSopp(0x01)}; + ExpectFatal([&] { (void)RecompileForTest(unsupported, options); }, + "unsupported partial or register-based store wait was dropped"); + } +#endif } void TestSopkCompareImmediateExtension() { @@ -14444,6 +14487,7 @@ int main() { TestNewShaderDecoderArchitecture(); TestImageAddressOperands(); TestSopkCompareImmediateExtension(); + TestNativeStoreCompletion(); TestNewShaderRecompilerCapturedVopcSdwaCmpxClass(); TestNewShaderRecompilerCapturedVopcSdwaCmpxLtU16(); TestNewShaderRecompilerIrLookupMissFailsExplicitly();