From 9caec9f2c8056c80fa54329c7ec982e2af1e4f8d Mon Sep 17 00:00:00 2001 From: Claxten Date: Tue, 6 Oct 2026 03:28:30 +0400 Subject: [PATCH] shader: preserve RDNA2 DLC coherence for flat address loads --- .../recompiler/backend/spirv/SpirvEmitter.cpp | 6 ++- .../backend/spirv/spirvEmitterMemory.cpp | 35 ++++++++++++----- .../recompiler/frontend/decode/MemoryOps.cpp | 4 +- .../recompiler/frontend/translate/Memory.cpp | 5 ++- tests/ShaderRecompilerComputeTests.cpp | 39 +++++++++++++++++++ 5 files changed, 75 insertions(+), 14 deletions(-) diff --git a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp index 45d71531..4bdde55c 100644 --- a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp @@ -205,7 +205,8 @@ Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program if (memory_index >= program.memory_info.size()) { Fail(program, "address operation has invalid memory metadata"); } - const auto kind = program.memory_info[memory_index].kind; + const auto& memory = program.memory_info[memory_index]; + const auto kind = memory.kind; if (kind == IR::ResourceKind::Scratch || kind == IR::ResourceKind::FlatLocal) { if (program.scratch_dwords == 0) { Fail(program, "scratch operation has no per-thread storage"); @@ -217,6 +218,9 @@ Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program } } else if (address_access == IR::AddressAccess::Write) { Fail(program, "writable FLAT/GLOBAL addresses require GPU ownership tracking"); + } else { + // Descriptor stores can alias coherent physical-address loads. + requirements.coherent_buffers |= memory.coherent; } } if (IR::BufferAccessOf(inst.GetOpcode()) != IR::BufferAccess::None) { diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp index 8e0a3e5e..7e8ef221 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp @@ -180,29 +180,46 @@ void RecordBdaFault(EmitterState& state, uint32_t page) { Binary(state, spv::OpBitwiseOr, TypeU32(state), value, bit)); } -uint32_t LoadBdaDword(ValueEmitContext& ctx, uint32_t address) { +uint32_t LoadBdaDword(ValueEmitContext& ctx, uint32_t address, bool coherent = false) { auto& state = ctx.state; const auto bda = GetBdaPointer(state, address); const auto present = Binary(state, spv::OpINotEqual, TypeBool(state), bda, ConstantU64(state, 0)); return EmitValueOrZeroIfCondition(state, present, [&]() { - const auto pointer = state.builder.AllocateId(); - state.builder.AddFunction(spv::OpConvertUToPtr, TypePhysicalU32Pointer(state), pointer, - bda); + auto pointer = state.builder.AllocateId(); + if (coherent) { + const auto block = state.builder.DecoratedType( + spv::OpTypeStruct, {{spv::OpMemberDecorate, {0, spv::DecorationOffset, 0}}, + {spv::OpMemberDecorate, {0, spv::DecorationCoherent}}, + {spv::OpDecorate, {spv::DecorationBlock}}}, TypeU32(state)); + state.builder.AddFunction( + spv::OpConvertUToPtr, TypePointer(state, spv::StorageClassPhysicalStorageBuffer, block), + pointer, bda); + const auto base = pointer; + pointer = state.builder.AllocateId(); + state.builder.AddFunction(spv::OpAccessChain, TypePhysicalU32Pointer(state), pointer, + base, ConstantU32(state, 0)); + } else { + state.builder.AddFunction(spv::OpConvertUToPtr, TypePhysicalU32Pointer(state), pointer, + bda); + } const auto value = state.builder.AllocateId(); constexpr uint32_t alignment = sizeof(uint32_t); state.builder.AddFunction(spv::OpLoad, TypeU32(state), value, pointer, - spv::MemoryAccessAlignedMask, alignment); + spv::MemoryAccessAlignedMask | + (coherent ? spv::MemoryAccessVolatileMask : spv::MemoryAccessMaskNone), + alignment); return value; }); } -uint32_t LoadBda(ValueEmitContext& ctx, uint32_t address, uint32_t active, uint32_t bits) { +uint32_t LoadBda(ValueEmitContext& ctx, uint32_t address, uint32_t active, uint32_t bits, + bool coherent = false) { auto& state = ctx.state; return EmitValueOrZeroIfCondition(state, active, [&]() { const auto aligned = Binary(state, spv::OpBitwiseAnd, TypeU64(state), address, ConstantU64(state, ~uint64_t {3})); - const auto first = LoadBdaDword(ctx, aligned); + const auto first = LoadBdaDword(ctx, aligned, coherent); const auto byte = Binary(state, spv::OpBitwiseAnd, TypeU32(state), Unary(state, spv::OpUConvert, TypeU32(state), address), ConstantU32(state, 3)); @@ -212,7 +229,7 @@ uint32_t LoadBda(ValueEmitContext& ctx, uint32_t address, uint32_t active, uint3 TypeBool(state), byte, ConstantU32(state, bits == 16u ? 2u : 0u)); const auto second = EmitValueOrZeroIfCondition(state, crosses, [&]() { return LoadBdaDword(ctx, Binary(state, spv::OpIAdd, TypeU64(state), aligned, - ConstantU64(state, sizeof(uint32_t)))); + ConstantU64(state, sizeof(uint32_t))), coherent); }); const auto shift = Binary(state, spv::OpShiftLeftLogical, TypeU32(state), byte, ConstantU32(state, 3)); @@ -1359,7 +1376,7 @@ void EmitLoadMemory(ValueEmitContext& ctx, const IR::Inst& inst) { else if (address_info.access == IR::AddressAccess::Read && mem.kind != IR::ResourceKind::Scratch) value = LoadBda(ctx, GuestAddress(ctx, inst, mem), ctx.Arg(inst, inst.NumArgs() - 1), - address_info.data_bits); + address_info.data_bits, mem.coherent); else if (op == IR::ValueOpcode::LoadBufferU32 && mem.formatted) value = mem.kind == IR::ResourceKind::IndirectBuffer ? LoadIndirectFormattedX(ctx, inst) : FormattedLoad(ctx, inst, mem); diff --git a/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp b/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp index 6a1b7021..f963091e 100644 --- a/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp @@ -343,7 +343,6 @@ void DecodeFlat(uint32_t pc, std::span code, uint32_t word_index const uint32_t word0 = code[word_index]; const uint32_t word1 = code[word_index + 1u]; const uint32_t offset = word0 & 0xfffu; - const uint32_t dlc = (word0 >> 12u) & 1u; const uint32_t lds = (word0 >> 13u) & 1u; const uint32_t seg = (word0 >> 14u) & 0x3u; const uint32_t opcode = (word0 >> 18u) & 0x7fu; @@ -355,6 +354,7 @@ void DecodeFlat(uint32_t pc, std::span code, uint32_t word_index inst.pc = pc; inst.word_count = 2; inst.offset = seg == 0u ? (offset & 0x7ffu) : SignExtendU32(offset, 12u); + inst.dlc = ((word0 >> 12u) & 1u) != 0; inst.glc = ((word0 >> 16u) & 1u) != 0; inst.slc = ((word0 >> 17u) & 1u) != 0; inst.family = Family::FLAT; @@ -364,7 +364,7 @@ void DecodeFlat(uint32_t pc, std::span code, uint32_t word_index ApplyMemoryInfo(inst, info); SetRawWords(inst, code, word_index, 2); - if (dlc != 0 || lds != 0 || inst.glc || inst.slc || seg == 3u) { + if (lds != 0 || inst.glc || inst.slc || seg == 3u) { SetUnsupported(inst, Family::FLAT, opcode, "FLAT modifiers or segment are not implemented"); return; } diff --git a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp index 817b6903..9aaf8134 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp @@ -129,8 +129,9 @@ IR::MemoryInfo MemoryInfoFromDecoded(const Decoder::Instruction& decoded) { // Vector loads use GLC/DLC to bypass L0/GL1; atomics use GLC only to return data. const bool buffer_atomic = decoded.opcode >= Decoder::Opcode::BUFFER_ATOMIC_SWAP && decoded.opcode <= Decoder::Opcode::BUFFER_ATOMIC_FMAX; - memory.coherent = memory.kind == ResourceKind::Buffer && !buffer_atomic && - (decoded.glc || decoded.dlc); + memory.coherent = (memory.kind == ResourceKind::Buffer || memory.kind == ResourceKind::Flat || + memory.kind == ResourceKind::Global) && + !buffer_atomic && (decoded.glc || decoded.dlc); memory.resource = ResourceIndexFromOperand(decoded.src1); memory.sampler = ResourceIndexFromOperand(decoded.src2); if (memory.kind == ResourceKind::ScalarBuffer) { diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 5163449b..9583fb4c 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -29596,6 +29596,41 @@ TestCase TBufferStoreVariants() { O::S_ENDPGM}}; } +TestCase FlatLoadDlcCaptured(u32 wave_size) { + using O = ShaderOpcode; + TestCase test; + test.name = wave_size == 64 ? "FlatLoadDlcCapturedWave64" : "FlatLoadDlcCapturedWave32"; + auto &code = test.code; + code.push_back(EncodeSop1(0x04, 126, InlineU32(1))); + AppendVMovU32(&code, 4, 0); + AppendVMovU32(&code, 5, 31); + // PPSA04288 CS 05b58beb842bacbf:0x638, FLAT_LOAD_DWORDX2 v[0:1], v[4:5] DLC. + code.insert(code.end(), {0xdc341000u, 0x007d0004u}); + AppendStoreVgpr(&code, 0, 0); + AppendStoreVgpr(&code, 1, 1); + // The descriptor store aliases the physical address read by the next load. + AppendStoreVgpr(&code, 1, 8); + code.push_back(0xbbfd0000u); // S_WAITCNT_VSCNT null, 0. + code.insert(code.end(), {0xdc341000u, 0x007d0004u}); + AppendStoreVgpr(&code, 0, 2); + AppendStoreVgpr(&code, 1, 3); + AppendEnd(&code); + test.initial = {0, 0, 0, 0, 0, 0, 0, 0, 0x11223344u, 0x55667788u}; + test.expected = {0x11223344u, 0x55667788u, 0x55667788u, 0x55667788u, + 0, 0, 0, 0, 0x55667788u, 0x55667788u}; + test.bda_mappings = {{0x1f00000000ull, 32}}; + test.opcodes = {O::S_MOV_B64, O::V_MOV_B32, O::FLAT_LOAD_DWORDX2, + O::BUFFER_STORE_DWORD, O::S_WAITCNT_VSCNT, O::S_ENDPGM}; + test.required_spirv = {"OpDecorate %buffers Coherent", " 0 Coherent", "Volatile|Aligned 4"}; + test.decoded_counts = {{"dlc=1", 2}}; + test.compute_info.threads_num[0] = wave_size; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.wave_size = wave_size; + test.has_compute_info = true; + return test; +} + TestCase FlatLoadVariants() { using O = ShaderOpcode; @@ -35312,6 +35347,8 @@ std::vector MakeCases() { AddCase(TBufferStoreFormatXy88IntegerComponents); AddCase(TBufferLoadFormatXy88IntegerComponents); AddCase(TBufferStoreVariants); + cases.push_back(FlatLoadDlcCaptured(32)); + cases.push_back(FlatLoadDlcCaptured(64)); AddCase(FlatLoadVariants); AddCase(FlatSubdwordLoadsApplyByteOffset); cases.push_back(GlobalLoadShortD16Captured(32)); @@ -40678,6 +40715,8 @@ int main(int argc, char **argv) { } if (argc == 2 && std::strcmp(argv[1], "--flat-d16-only") == 0) { VulkanHarness vulkan; + RunCase(&vulkan, FlatLoadDlcCaptured(32)); + RunCase(&vulkan, FlatLoadDlcCaptured(64)); RunCase(&vulkan, GlobalLoadShortD16Captured(32)); RunCase(&vulkan, GlobalLoadShortD16Captured(64)); RunCase(&vulkan, FlatLoadShortD16AddressSegments());