From b3e419ff1101999525fa2d061ada1d102cf788b1 Mon Sep 17 00:00:00 2001 From: Louis <116039387+tototomate123@users.noreply.github.com> Date: Thu, 1 Oct 2026 23:15:53 +0200 Subject: [PATCH] shader: implement 32-bit RDNA2 IMAGE_ATOMIC_CMPSWAP (#952) * shader: implement 32-bit RDNA2 image atomic compare-and-swap * shader: share atomic emission and correct image barriers --------- Co-authored-by: nmzik --- .../backend/spirv/spirvEmitterImage.cpp | 26 +------ .../backend/spirv/spirvEmitterInstructions.h | 1 + .../backend/spirv/spirvEmitterInternal.h | 19 ++--- .../backend/spirv/spirvEmitterMemory.cpp | 62 +-------------- .../spirv/spirvEmitterMemoryHelpers.cpp | 78 +++++++++++++++++-- .../recompiler/frontend/decode/ImageOps.cpp | 5 ++ .../frontend/decode/ShaderDecoder.cpp | 1 + .../frontend/decode/ShaderDecoder.h | 1 + .../recompiler/frontend/translate/Memory.cpp | 17 +++- .../recompiler/ir/opcodes/ValueOpcodes.cpp | 1 + .../recompiler/ir/opcodes/ValueOpcodes.inc | 1 + tests/ShaderRecompilerComputeTests.cpp | 53 +++++++++++++ 12 files changed, 163 insertions(+), 102 deletions(-) diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp index d385d197..14dfae8f 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp @@ -539,21 +539,6 @@ uint32_t StoreTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t d return PackImageTexel(ctx, mem, texel); } -spv::Op ImageAtomicOpcode(IR::ValueOpcode opcode) { - switch (opcode) { - case IR::ValueOpcode::ImageAtomicSwap32: return spv::OpAtomicExchange; - case IR::ValueOpcode::ImageAtomicIAdd32: return spv::OpAtomicIAdd; - case IR::ValueOpcode::ImageAtomicSMin32: return spv::OpAtomicSMin; - case IR::ValueOpcode::ImageAtomicUMin32: return spv::OpAtomicUMin; - case IR::ValueOpcode::ImageAtomicSMax32: return spv::OpAtomicSMax; - case IR::ValueOpcode::ImageAtomicUMax32: return spv::OpAtomicUMax; - case IR::ValueOpcode::ImageAtomicAnd32: return spv::OpAtomicAnd; - case IR::ValueOpcode::ImageAtomicOr32: return spv::OpAtomicOr; - case IR::ValueOpcode::ImageAtomicXor32: return spv::OpAtomicXor; - default: return spv::OpNop; - } -} - } // namespace void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) { @@ -878,10 +863,9 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) { ctx.Define(inst, ResultVector(ctx, result, numeric_class, dref, mem)); return; } - const auto atomic_opcode = ImageAtomicOpcode(op); if (image_info.access == IR::ImageAccess::Atomic) { const auto dimension = image.dimension; - ctx.Define(inst, EmitValueOrZeroIfCondition(state, ctx.Arg(inst, 3), [&]() { + ctx.Define(inst, EmitValueOrZeroIfCondition(state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() { const auto pointer = state.builder.AllocateId(); const auto pointer_type = state.builder.Type( spv::OpTypePointer, spv::StorageClassImage, TypeU32(state)); @@ -898,12 +882,8 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) { op == IR::ValueOpcode::ImageAtomicFMax32); }); } - const auto old = state.builder.AllocateId(); - state.builder.AddFunction(atomic_opcode, TypeU32(state), old, pointer, - ConstantU32(state, spv::ScopeDevice), - ConstantU32(state, spv::MemorySemanticsMaskNone), - ctx.Arg(inst, 2)); - EmitDeviceAtomicMemoryBarrier(state); + const auto old = EmitAtomicOperation(ctx, inst, pointer, spv::ScopeDevice); + EmitAtomicMemoryBarrier(state, IR::ResourceKind::Image); return old; })); return; diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h index cbfb7344..645f4998 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h @@ -310,6 +310,7 @@ inline constexpr auto EmitImageRead = EmitImage; inline constexpr auto EmitImageWrite = EmitImage; inline constexpr auto EmitImageSampleRaw = EmitImage; inline constexpr auto EmitImageGatherRaw = EmitImage; +inline constexpr auto EmitImageAtomicCompareSwap32 = EmitImage; inline constexpr auto EmitImageAtomicSwap32 = EmitImage; inline constexpr auto EmitImageAtomicIAdd32 = EmitImage; inline constexpr auto EmitImageAtomicSMin32 = EmitImage; diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h index 12a6d6ee..98643eb1 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h @@ -436,7 +436,12 @@ uint32_t EmitUFloatToF32Bits(EmitterState& state, uint32_t raw, uint32_t bits); uint32_t NormalizeFormatComponent(EmitterState& state, const Format::BufferFormatInfo& info, uint32_t component, uint32_t raw); -void EmitDeviceAtomicMemoryBarrier(EmitterState& state); +spv::Op SpirvAtomicOpcode(IR::ValueOpcode opcode); + +uint32_t EmitAtomicOperation(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t pointer, + uint32_t scope); + +void EmitAtomicMemoryBarrier(EmitterState& state, IR::ResourceKind kind); uint32_t EmitFloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t source, bool max_value); @@ -548,14 +553,7 @@ uint32_t EmitValueOrZeroIfCondition(EmitterState& state, uint32_t condition, Fn& template uint32_t AtomicUpdate(EmitterState& state, uint32_t pointer, IR::ResourceKind kind, Fn&& desired) { - const auto scope = kind == IR::ResourceKind::Lds ? spv::ScopeWorkgroup : spv::ScopeDevice; - const auto memory = [&] { - switch (kind) { - case IR::ResourceKind::Lds: return spv::MemorySemanticsWorkgroupMemoryMask; - case IR::ResourceKind::Image: return spv::MemorySemanticsImageMemoryMask; - default: return spv::MemorySemanticsUniformMemoryMask; - } - }(); + const auto scope = kind == IR::ResourceKind::Lds ? spv::ScopeWorkgroup : spv::ScopeDevice; const auto preheader = state.builder.AllocateId(); const auto header = state.builder.AllocateId(); const auto cont = state.builder.AllocateId(); @@ -584,8 +582,7 @@ uint32_t AtomicUpdate(EmitterState& state, uint32_t pointer, IR::ResourceKind ki EmitLabel(state, cont); state.builder.AddFunction(spv::OpBranch, header); EmitLabel(state, merge); - state.builder.AddFunction(spv::OpMemoryBarrier, ConstantU32(state, scope), - ConstantU32(state, spv::MemorySemanticsAcquireReleaseMask | memory)); + EmitAtomicMemoryBarrier(state, kind); return observed; } diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp index 1a47bfc1..275b3d96 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp @@ -555,57 +555,6 @@ void StoreWord(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem) }); } -spv::Op SpirvAtomicOpcode(IR::ValueOpcode opcode) { - switch (opcode) { - case IR::ValueOpcode::BufferAtomicCmpSwap32: return spv::OpAtomicCompareExchange; - case IR::ValueOpcode::BufferAtomicSwap32: - case IR::ValueOpcode::BufferAtomicSwap64: - case IR::ValueOpcode::SharedAtomicSwap32: return spv::OpAtomicExchange; - case IR::ValueOpcode::BufferAtomicIAdd32: - case IR::ValueOpcode::SharedAtomicIAdd64: - case IR::ValueOpcode::SharedAtomicIAdd32: return spv::OpAtomicIAdd; - case IR::ValueOpcode::BufferAtomicISub32: - case IR::ValueOpcode::SharedAtomicISub32: return spv::OpAtomicISub; - case IR::ValueOpcode::BufferAtomicSMin32: - case IR::ValueOpcode::SharedAtomicSMin32: return spv::OpAtomicSMin; - case IR::ValueOpcode::BufferAtomicUMin32: - case IR::ValueOpcode::SharedAtomicUMin32: return spv::OpAtomicUMin; - case IR::ValueOpcode::BufferAtomicSMax32: - case IR::ValueOpcode::SharedAtomicSMax32: return spv::OpAtomicSMax; - case IR::ValueOpcode::BufferAtomicUMax32: - case IR::ValueOpcode::SharedAtomicUMax32: return spv::OpAtomicUMax; - case IR::ValueOpcode::BufferAtomicAnd32: - case IR::ValueOpcode::BufferAtomicAnd64: - case IR::ValueOpcode::SharedAtomicAnd32: return spv::OpAtomicAnd; - case IR::ValueOpcode::BufferAtomicOr32: - case IR::ValueOpcode::BufferAtomicOr64: - case IR::ValueOpcode::SharedAtomicOr64: - case IR::ValueOpcode::SharedAtomicOr32: return spv::OpAtomicOr; - case IR::ValueOpcode::BufferAtomicXor32: - case IR::ValueOpcode::SharedAtomicXor32: return spv::OpAtomicXor; - default: return spv::OpNop; - } -} - -uint32_t EmitAtomicOperation(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t pointer, - uint32_t scope) { - const auto old = ctx.state.builder.AllocateId(); - if (inst.GetOpcode() == IR::ValueOpcode::BufferAtomicCmpSwap32) { - const auto desired = ctx.Arg(inst, inst.NumArgs() - 3); - const auto comparator = ctx.Arg(inst, inst.NumArgs() - 2); - ctx.state.builder.AddFunction( - spv::OpAtomicCompareExchange, TypeU32(ctx.state), old, pointer, - ConstantU32(ctx.state, scope), ConstantU32(ctx.state, spv::MemorySemanticsMaskNone), - ConstantU32(ctx.state, spv::MemorySemanticsMaskNone), desired, comparator); - } else { - const auto value = ctx.Arg(inst, inst.NumArgs() - 2); - ctx.state.builder.AddFunction(SpirvAtomicOpcode(inst.GetOpcode()), TypeU32(ctx.state), old, - pointer, ConstantU32(ctx.state, scope), - ConstantU32(ctx.state, spv::MemorySemanticsMaskNone), value); - } - return old; -} - template uint32_t EmitAtomicAccess(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem, Fn&& operation) { @@ -1076,14 +1025,7 @@ uint32_t EmitAtomic32(ValueEmitContext& ctx, const IR::Inst& inst) { const auto scope = mem.kind == IR::ResourceKind::Lds ? spv::ScopeWorkgroup : spv::ScopeDevice; const auto old = EmitAtomicOperation(ctx, inst, pointer, scope); - if (mem.kind == IR::ResourceKind::Lds) { - const auto semantics = - spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsWorkgroupMemoryMask; - ctx.state.builder.AddFunction(spv::OpMemoryBarrier, ConstantU32(ctx.state, scope), - ConstantU32(ctx.state, semantics)); - } else { - EmitDeviceAtomicMemoryBarrier(ctx.state); - } + EmitAtomicMemoryBarrier(ctx.state, mem.kind); return old; }); } @@ -1111,7 +1053,7 @@ uint32_t EmitBufferAtomic64(ValueEmitContext& ctx, const IR::Inst& inst) { TypeStorageBufferU64ElementPointer(state)), ConstantU32(state, spv::ScopeDevice), ConstantU32(state, spv::MemorySemanticsMaskNone), value); - EmitDeviceAtomicMemoryBarrier(state); + EmitAtomicMemoryBarrier(state, mem.kind); return Unary(state, spv::OpBitcast, TypeU64(state), old); }); }); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp index 0ccad9a6..9ec11028 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp @@ -306,11 +306,79 @@ uint32_t NormalizeFormatComponent(EmitterState& state, const Format::BufferForma } } -void EmitDeviceAtomicMemoryBarrier(EmitterState& state) { - const auto semantics = - spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsUniformMemoryMask; - state.builder.AddFunction(spv::OpMemoryBarrier, ConstantU32(state, spv::ScopeDevice), - ConstantU32(state, semantics)); +spv::Op SpirvAtomicOpcode(IR::ValueOpcode opcode) { + switch (opcode) { + case IR::ValueOpcode::ImageAtomicCompareSwap32: + case IR::ValueOpcode::BufferAtomicCmpSwap32: return spv::OpAtomicCompareExchange; + case IR::ValueOpcode::ImageAtomicSwap32: + case IR::ValueOpcode::BufferAtomicSwap32: + case IR::ValueOpcode::BufferAtomicSwap64: + case IR::ValueOpcode::SharedAtomicSwap32: return spv::OpAtomicExchange; + case IR::ValueOpcode::ImageAtomicIAdd32: + case IR::ValueOpcode::BufferAtomicIAdd32: + case IR::ValueOpcode::SharedAtomicIAdd64: + case IR::ValueOpcode::SharedAtomicIAdd32: return spv::OpAtomicIAdd; + case IR::ValueOpcode::BufferAtomicISub32: + case IR::ValueOpcode::SharedAtomicISub32: return spv::OpAtomicISub; + case IR::ValueOpcode::ImageAtomicSMin32: + case IR::ValueOpcode::BufferAtomicSMin32: + case IR::ValueOpcode::SharedAtomicSMin32: return spv::OpAtomicSMin; + case IR::ValueOpcode::ImageAtomicUMin32: + case IR::ValueOpcode::BufferAtomicUMin32: + case IR::ValueOpcode::SharedAtomicUMin32: return spv::OpAtomicUMin; + case IR::ValueOpcode::ImageAtomicSMax32: + case IR::ValueOpcode::BufferAtomicSMax32: + case IR::ValueOpcode::SharedAtomicSMax32: return spv::OpAtomicSMax; + case IR::ValueOpcode::ImageAtomicUMax32: + case IR::ValueOpcode::BufferAtomicUMax32: + case IR::ValueOpcode::SharedAtomicUMax32: return spv::OpAtomicUMax; + case IR::ValueOpcode::ImageAtomicAnd32: + case IR::ValueOpcode::BufferAtomicAnd32: + case IR::ValueOpcode::BufferAtomicAnd64: + case IR::ValueOpcode::SharedAtomicAnd32: return spv::OpAtomicAnd; + case IR::ValueOpcode::ImageAtomicOr32: + case IR::ValueOpcode::BufferAtomicOr32: + case IR::ValueOpcode::BufferAtomicOr64: + case IR::ValueOpcode::SharedAtomicOr64: + case IR::ValueOpcode::SharedAtomicOr32: return spv::OpAtomicOr; + case IR::ValueOpcode::ImageAtomicXor32: + case IR::ValueOpcode::BufferAtomicXor32: + case IR::ValueOpcode::SharedAtomicXor32: return spv::OpAtomicXor; + default: return spv::OpNop; + } +} + +uint32_t EmitAtomicOperation(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t pointer, + uint32_t scope) { + const auto opcode = SpirvAtomicOpcode(inst.GetOpcode()); + const auto old = ctx.state.builder.AllocateId(); + if (opcode == spv::OpAtomicCompareExchange) { + const auto desired = ctx.Arg(inst, inst.NumArgs() - 3); + const auto comparator = ctx.Arg(inst, inst.NumArgs() - 2); + ctx.state.builder.AddFunction( + spv::OpAtomicCompareExchange, TypeU32(ctx.state), old, pointer, + ConstantU32(ctx.state, scope), ConstantU32(ctx.state, spv::MemorySemanticsMaskNone), + ConstantU32(ctx.state, spv::MemorySemanticsMaskNone), desired, comparator); + } else { + const auto value = ctx.Arg(inst, inst.NumArgs() - 2); + ctx.state.builder.AddFunction(opcode, TypeU32(ctx.state), old, pointer, + ConstantU32(ctx.state, scope), + ConstantU32(ctx.state, spv::MemorySemanticsMaskNone), value); + } + return old; +} + +void EmitAtomicMemoryBarrier(EmitterState& state, IR::ResourceKind kind) { + const auto scope = kind == IR::ResourceKind::Lds ? spv::ScopeWorkgroup : spv::ScopeDevice; + const auto memory = [&] { + switch (kind) { + case IR::ResourceKind::Lds: return spv::MemorySemanticsWorkgroupMemoryMask; + case IR::ResourceKind::Image: return spv::MemorySemanticsImageMemoryMask; + default: return spv::MemorySemanticsUniformMemoryMask; + } + }(); + state.builder.AddFunction(spv::OpMemoryBarrier, ConstantU32(state, scope), + ConstantU32(state, spv::MemorySemanticsAcquireReleaseMask | memory)); } uint32_t EmitFloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t source, diff --git a/src/graphics/shader/recompiler/frontend/decode/ImageOps.cpp b/src/graphics/shader/recompiler/frontend/decode/ImageOps.cpp index 851da347..554c0c8a 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ImageOps.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/ImageOps.cpp @@ -192,6 +192,7 @@ constexpr MimgGatherInfo MIMG_GATHER_OPCODE_LIST[] = { constexpr Detail::OpcodeMap MIMG_ATOMIC_OPCODE_LIST[] = { {0x0fu, Opcode::IMAGE_ATOMIC_SWAP}, + {0x10u, Opcode::IMAGE_ATOMIC_CMPSWAP}, {0x11u, Opcode::IMAGE_ATOMIC_ADD}, {0x14u, Opcode::IMAGE_ATOMIC_SMIN}, {0x15u, Opcode::IMAGE_ATOMIC_UMIN}, @@ -359,6 +360,10 @@ void DecodeMimg(uint32_t pc, std::span code, uint32_t word_index SetUnsupported(inst, Family::MIMG, opcode, "MIMG image gather requires exactly one dmask bit"); } + if (inst.opcode == Opcode::IMAGE_ATOMIC_CMPSWAP && inst.dmask != 0x3u) { + SetUnsupported(inst, Family::MIMG, opcode, + "MIMG image compare-and-swap requires 32-bit DMASK 0x3"); + } const bool supports_d16 = sample != nullptr || gather != nullptr || opcode == 0x00u || opcode == 0x01u || opcode == 0x08u || opcode == 0x09u; if (d16 && !supports_d16) { diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp index 676405bc..c933ae72 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp @@ -574,6 +574,7 @@ std::string InstructionToString(const Instruction& inst) { case Opcode::IMAGE_SAMPLE: case Opcode::IMAGE_STORE: case Opcode::IMAGE_STORE_MIP: + case Opcode::IMAGE_ATOMIC_CMPSWAP: case Opcode::IMAGE_ATOMIC_SWAP: case Opcode::IMAGE_ATOMIC_ADD: case Opcode::IMAGE_ATOMIC_SMIN: diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h index 79326c11..43b65528 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h @@ -599,6 +599,7 @@ enum class Opcode { IMAGE_STORE, IMAGE_STORE_MIP, IMAGE_ATOMIC_SWAP, + IMAGE_ATOMIC_CMPSWAP, IMAGE_ATOMIC_ADD, IMAGE_ATOMIC_SMIN, IMAGE_ATOMIC_UMIN, diff --git a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp index 9f2e2bf1..c535ffb3 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp @@ -558,9 +558,18 @@ void Translator::IMAGE_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode const auto memory = MemoryInfoFromDecoded(inst); const auto resource = GetImageResource(memory); const auto address = MakeImageAddress(inst, MemorySourceAt(inst, 1)); - const auto result = - ir.Emit(opcode, {resource, address, ReadU32(MemorySourceAt(inst, 0)), ir.GetExec()}, - AddMemoryInfo(memory, inst.pc)); + const auto data = MemorySourceAt(inst, 0); + const auto flags = AddMemoryInfo(memory, inst.pc); + IR::Value result; + if (opcode == IR::ValueOpcode::ImageAtomicCompareSwap32) { + // VDATA supplies the replacement; VDATA+1 supplies the comparison. + result = ir.Emit(opcode, + {resource, address, ReadU32(data), ReadU32(OffsetOperand(data, 1u)), + ir.GetExec()}, + flags); + } else { + result = ir.Emit(opcode, {resource, address, ReadU32(data), ir.GetExec()}, flags); + } if (inst.glc) { WriteOperand(inst.dst, result); } @@ -1030,6 +1039,8 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) { case Decoder::Opcode::DS_WRXCHG_RTN_B32: return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicSwap32, true); + case Decoder::Opcode::IMAGE_ATOMIC_CMPSWAP: + return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicCompareSwap32); case Decoder::Opcode::IMAGE_ATOMIC_SWAP: return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicSwap32); case Decoder::Opcode::IMAGE_ATOMIC_ADD: diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp index 2ba0dbd3..a6723372 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp @@ -220,6 +220,7 @@ ImageOpcodeInfo ImageOpcodeInfoOf(ValueOpcode opcode) { return {ImageAccess::Read, ImageResourceClass::Sampled, true}; case ValueOpcode::ImageWrite: return {ImageAccess::Write, ImageResourceClass::Storage, false}; + case ValueOpcode::ImageAtomicCompareSwap32: case ValueOpcode::ImageAtomicSwap32: case ValueOpcode::ImageAtomicIAdd32: case ValueOpcode::ImageAtomicSMin32: diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc index 3f5df76b..9623c86e 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc @@ -298,6 +298,7 @@ VALUE_OPCODE(ImageRead, U32x4, ImageResource, ImageAddress, U1) VALUE_OPCODE(ImageWrite, Void, ImageResource, ImageAddress, U32x4, U1) VALUE_OPCODE(ImageSampleRaw, U32x4, ImageResource, SamplerResource, ImageAddress) VALUE_OPCODE(ImageGatherRaw, U32x4, ImageResource, SamplerResource, ImageAddress) +VALUE_OPCODE(ImageAtomicCompareSwap32, U32, ImageResource, ImageAddress, U32, U32, U1) VALUE_OPCODE(ImageAtomicSwap32, U32, ImageResource, ImageAddress, U32, U1) VALUE_OPCODE(ImageAtomicIAdd32, U32, ImageResource, ImageAddress, U32, U1) VALUE_OPCODE(ImageAtomicSMin32, U32, ImageResource, ImageAddress, U32, U1) diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 61a63450..1be1349f 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -30512,6 +30512,51 @@ TestCase ImageStoreAndAtomicShareTypedBinding() { return test; } +// RDNA 2 MIMG opcode 0x10: replacement at VDATA, comparison at VDATA+1. +TestCase ImageAtomicCompareSwapGlcAndExec() { + TestCase test; + test.name = "ImageAtomicCompareSwapGlcAndExec"; + auto &code = test.code; + code.push_back(EncodeSop1(0x04, 12, 126)); // Preserve EXEC. + AppendVMovU32(&code, 20, 2); + AppendVMovU32(&code, 21, 1); + const auto exchange = [&](u32 replacement, u32 comparison, bool glc, u32 output) { + AppendVMovLiteral(&code, 0, replacement); + AppendVMovLiteral(&code, 1, comparison); + code.push_back(EncodeMimg0(0x10, 3, 0, glc)); + code.push_back(EncodeMimg1(0, 20)); + AppendStoreVgpr(&code, 0, output); + AppendStoreVgpr(&code, 1, output + 1); + }; + exchange(0x12345678u, 0xdeadbeefu, true, 0); // Match; return original bits. + exchange(0x87654321u, 0xdeadbeefu, true, 2); // Mismatch; preserve texel. + exchange(0xabcdef01u, 0x12345678u, false, 4); // GLC=0 preserves VDATA. + AppendVMovLiteral(&code, 0, 0x11111111u); + AppendVMovLiteral(&code, 1, 0xabcdef01u); + code.push_back(EncodeSop1(0x04, 126, InlineU32(0))); // Disable EXEC. + code.push_back(EncodeMimg0(0x10, 3, 0, true)); + code.push_back(EncodeMimg1(0, 20)); + code.push_back(EncodeSop1(0x04, 126, 12)); + AppendStoreVgpr(&code, 0, 6); + AppendStoreVgpr(&code, 1, 7); + AppendEnd(&code); + test.expected = {0xdeadbeefu, 0xdeadbeefu, 0x12345678u, 0xdeadbeefu, + 0xabcdef01u, 0x12345678u, 0x11111111u, 0xabcdef01u}; + test.opcodes = {ShaderOpcode::V_MOV_B32, ShaderOpcode::S_MOV_B64, + ShaderOpcode::IMAGE_ATOMIC_CMPSWAP, ShaderOpcode::BUFFER_STORE_DWORD, + ShaderOpcode::S_ENDPGM}; + test.user_data = MakeStorageTextureData(Prospero::BufferFormat::k32UInt); + test.has_user_data = true; + test.storage_image_r32ui = std::vector(16, 0); + test.storage_image_r32ui[6] = 0xdeadbeefu; + test.expected_storage_image_r32ui = test.storage_image_r32ui; + test.expected_storage_image_r32ui[6] = 0xabcdef01u; + // Device scope, AcquireRelease | ImageMemory (0x808). + test.required_spirv = {"OpAtomicCompareExchange", "OpImageTexelPointer", + "OpMemoryBarrier %uint_1 %uint_2056"}; + return test; +} + TestCase ImageAtomicSwapReturnsPreviousTexel() { using O = ShaderOpcode; @@ -31661,6 +31706,7 @@ std::vector MakeCases() { AddCase(ImageStorePackedUintHonorsSparseDmask); AddCase(ComputeTgSizeSgprUsesWaveMetadata); AddCase(ImageStoreAndAtomicShareTypedBinding); + AddCase(ImageAtomicCompareSwapGlcAndExec); AddCase(ImageAtomicSwapReturnsPreviousTexel); AddCase(ImageStoreAndAtomicUseSeparateBindings); AddCase(ImageAtomicVariants); @@ -36636,6 +36682,13 @@ int main(int argc, char **argv) { RunCase(&vulkan, BufferAtomicCmpSwapExactRaw()); return 0; } + if (argc == 2 && std::strcmp(argv[1], "--image-cmpswap-only") == 0) { + VulkanHarness vulkan; + RunCase(&vulkan, ImageAtomicCompareSwapGlcAndExec()); + RunCase(&vulkan, ImageAtomicSwapReturnsPreviousTexel()); + RunCase(&vulkan, ImageAtomicVariants()); + return 0; + } if (argc == 2 && std::strcmp(argv[1], "--ds-atomics-only") == 0) { VulkanHarness vulkan; for (const auto &test : MakeCases()) {