diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h index b6ca411a..c2671c9c 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h @@ -321,15 +321,21 @@ inline constexpr auto EmitImageSampleRaw = EmitImage; inline constexpr auto EmitImageGatherRaw = EmitImage; inline constexpr auto EmitImageAtomicCompareSwap32 = EmitImage; inline constexpr auto EmitImageAtomicSwap32 = EmitImage; +inline constexpr auto EmitImageAtomicSwap64 = EmitImage; inline constexpr auto EmitImageAtomicIAdd32 = EmitImage; +inline constexpr auto EmitImageAtomicIAdd64 = EmitImage; inline constexpr auto EmitImageAtomicSMin32 = EmitImage; inline constexpr auto EmitImageAtomicUMin32 = EmitImage; +inline constexpr auto EmitImageAtomicUMin64 = EmitImage; inline constexpr auto EmitImageAtomicSMax32 = EmitImage; inline constexpr auto EmitImageAtomicUMax32 = EmitImage; inline constexpr auto EmitImageAtomicUMax64 = EmitImage; inline constexpr auto EmitImageAtomicAnd32 = EmitImage; +inline constexpr auto EmitImageAtomicAnd64 = EmitImage; inline constexpr auto EmitImageAtomicOr32 = EmitImage; +inline constexpr auto EmitImageAtomicOr64 = EmitImage; inline constexpr auto EmitImageAtomicXor32 = EmitImage; +inline constexpr auto EmitImageAtomicXor64 = EmitImage; inline constexpr auto EmitImageAtomicFMin32 = EmitImage; inline constexpr auto EmitImageAtomicFMax32 = EmitImage; void EmitUnreachable(ValueEmitContext& ctx, const IR::Inst& inst); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp index 468c8c2c..daf448ec 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp @@ -361,10 +361,12 @@ spv::Op SpirvAtomicOpcode(IR::ValueOpcode opcode) { case IR::ValueOpcode::ImageAtomicCompareSwap32: case IR::ValueOpcode::BufferAtomicCmpSwap32: return spv::OpAtomicCompareExchange; case IR::ValueOpcode::ImageAtomicSwap32: + case IR::ValueOpcode::ImageAtomicSwap64: case IR::ValueOpcode::BufferAtomicSwap32: case IR::ValueOpcode::BufferAtomicSwap64: case IR::ValueOpcode::SharedAtomicSwap32: return spv::OpAtomicExchange; case IR::ValueOpcode::ImageAtomicIAdd32: + case IR::ValueOpcode::ImageAtomicIAdd64: case IR::ValueOpcode::BufferAtomicIAdd32: case IR::ValueOpcode::SharedAtomicIAdd64: case IR::ValueOpcode::SharedAtomicIAdd32: return spv::OpAtomicIAdd; @@ -374,6 +376,7 @@ spv::Op SpirvAtomicOpcode(IR::ValueOpcode opcode) { case IR::ValueOpcode::BufferAtomicSMin32: case IR::ValueOpcode::SharedAtomicSMin32: return spv::OpAtomicSMin; case IR::ValueOpcode::ImageAtomicUMin32: + case IR::ValueOpcode::ImageAtomicUMin64: case IR::ValueOpcode::BufferAtomicUMin32: case IR::ValueOpcode::SharedAtomicUMin32: return spv::OpAtomicUMin; case IR::ValueOpcode::ImageAtomicSMax32: @@ -384,15 +387,18 @@ spv::Op SpirvAtomicOpcode(IR::ValueOpcode opcode) { case IR::ValueOpcode::BufferAtomicUMax32: case IR::ValueOpcode::SharedAtomicUMax32: return spv::OpAtomicUMax; case IR::ValueOpcode::ImageAtomicAnd32: + case IR::ValueOpcode::ImageAtomicAnd64: case IR::ValueOpcode::BufferAtomicAnd32: case IR::ValueOpcode::BufferAtomicAnd64: case IR::ValueOpcode::SharedAtomicAnd32: return spv::OpAtomicAnd; case IR::ValueOpcode::ImageAtomicOr32: + case IR::ValueOpcode::ImageAtomicOr64: case IR::ValueOpcode::BufferAtomicOr32: case IR::ValueOpcode::BufferAtomicOr64: case IR::ValueOpcode::SharedAtomicOr64: case IR::ValueOpcode::SharedAtomicOr32: return spv::OpAtomicOr; case IR::ValueOpcode::ImageAtomicXor32: + case IR::ValueOpcode::ImageAtomicXor64: case IR::ValueOpcode::BufferAtomicXor32: case IR::ValueOpcode::SharedAtomicXor32: return spv::OpAtomicXor; default: return spv::OpNop; diff --git a/src/graphics/shader/recompiler/frontend/decode/ImageOps.cpp b/src/graphics/shader/recompiler/frontend/decode/ImageOps.cpp index 1c08d461..04313b2c 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ImageOps.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/ImageOps.cpp @@ -20,6 +20,12 @@ struct MimgGatherInfo { uint32_t flags = 0; }; +struct MimgAtomicInfo { + uint32_t encoding = 0; + Opcode decoded = Opcode::UNSUPPORTED; + bool supports_64 = false; +}; + constexpr ImageDimension DecodeImageDimension(uint32_t dim) { switch (dim) { case 0u: return ImageDimension::Dim1D; @@ -190,17 +196,17 @@ constexpr MimgGatherInfo MIMG_GATHER_OPCODE_LIST[] = { {0x61u, Opcode::IMAGE_GATHER4H, ImageSampleFlagGatherHorizontal}, }; -constexpr Detail::OpcodeMap MIMG_ATOMIC_OPCODE_LIST[] = { - {0x0fu, Opcode::IMAGE_ATOMIC_SWAP}, +constexpr MimgAtomicInfo MIMG_ATOMIC_OPCODE_LIST[] = { + {0x0fu, Opcode::IMAGE_ATOMIC_SWAP, true}, {0x10u, Opcode::IMAGE_ATOMIC_CMPSWAP}, - {0x11u, Opcode::IMAGE_ATOMIC_ADD}, + {0x11u, Opcode::IMAGE_ATOMIC_ADD, true}, {0x14u, Opcode::IMAGE_ATOMIC_SMIN}, - {0x15u, Opcode::IMAGE_ATOMIC_UMIN}, + {0x15u, Opcode::IMAGE_ATOMIC_UMIN, true}, {0x16u, Opcode::IMAGE_ATOMIC_SMAX}, - {0x17u, Opcode::IMAGE_ATOMIC_UMAX}, - {0x18u, Opcode::IMAGE_ATOMIC_AND}, - {0x19u, Opcode::IMAGE_ATOMIC_OR}, - {0x1au, Opcode::IMAGE_ATOMIC_XOR}, + {0x17u, Opcode::IMAGE_ATOMIC_UMAX, true}, + {0x18u, Opcode::IMAGE_ATOMIC_AND, true}, + {0x19u, Opcode::IMAGE_ATOMIC_OR, true}, + {0x1au, Opcode::IMAGE_ATOMIC_XOR, true}, {0x1eu, Opcode::IMAGE_ATOMIC_FMIN}, {0x1fu, Opcode::IMAGE_ATOMIC_FMAX}, }; @@ -210,7 +216,7 @@ constexpr auto MIMG_GATHER_OPS = Detail::MakeOpcodeTable<0x100>(MIMG_GATHER_OPCO constexpr auto MIMG_ATOMIC_OPS = Detail::MakeOpcodeTable<0x100>(MIMG_ATOMIC_OPCODE_LIST); Opcode DecodeMimgOpcode(uint32_t opcode, const MimgSampleInfo* sample, const MimgGatherInfo* gather, - const Detail::OpcodeMap* atomic) { + const MimgAtomicInfo* atomic) { if (sample != nullptr) { return Opcode::IMAGE_SAMPLE; } @@ -246,7 +252,7 @@ uint32_t DecodeMimgSampleFlags(const MimgSampleInfo* sample, const MimgGatherInf uint32_t DecodeMimgAddressComponents(uint32_t opcode, ImageDimension dimension, const MimgSampleInfo* sample, const MimgGatherInfo* gather, - const Detail::OpcodeMap* atomic) { + const MimgAtomicInfo* atomic) { if (sample != nullptr) { return ImageSampleAddressComponents(sample->flags, dimension); } @@ -370,7 +376,7 @@ void DecodeMimg(uint32_t pc, std::span code, uint32_t word_index SetUnsupported(inst, Family::MIMG, opcode, "MIMG image atomic has invalid DMASK"); } else if (inst.dmask == mask64) { inst.data_bits = 64u; - if (inst.opcode != Opcode::IMAGE_ATOMIC_UMAX) { + if (!atomic->supports_64) { SetUnsupported(inst, Family::MIMG, opcode, "MIMG 64-bit image atomic opcode is not implemented"); } diff --git a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp index 9e1c45b1..f1bd226f 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp @@ -546,7 +546,10 @@ void Translator::BUFFER_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode } } -void Translator::IMAGE_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode) { +void Translator::IMAGE_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode32, + IR::ValueOpcode opcode64) { + const auto opcode = inst.data_bits == 64u ? opcode64 : opcode32; + EXIT_IF(opcode == IR::ValueOpcode::Count); const auto memory = MemoryInfoFromDecoded(inst); const auto resource = GetImageResource(memory); const auto address = MakeImageAddress(inst, MemorySourceAt(inst, 1)); @@ -1041,24 +1044,30 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) { case Decoder::Opcode::IMAGE_ATOMIC_CMPSWAP: return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicCompareSwap32); case Decoder::Opcode::IMAGE_ATOMIC_SWAP: - return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicSwap32); + return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicSwap32, + IR::ValueOpcode::ImageAtomicSwap64); case Decoder::Opcode::IMAGE_ATOMIC_ADD: - return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicIAdd32); + return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicIAdd32, + IR::ValueOpcode::ImageAtomicIAdd64); case Decoder::Opcode::IMAGE_ATOMIC_SMIN: return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicSMin32); case Decoder::Opcode::IMAGE_ATOMIC_UMIN: - return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicUMin32); + return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicUMin32, + IR::ValueOpcode::ImageAtomicUMin64); case Decoder::Opcode::IMAGE_ATOMIC_SMAX: return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicSMax32); case Decoder::Opcode::IMAGE_ATOMIC_UMAX: - return IMAGE_ATOMIC(inst, inst.data_bits == 64u ? IR::ValueOpcode::ImageAtomicUMax64 - : IR::ValueOpcode::ImageAtomicUMax32); + return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicUMax32, + IR::ValueOpcode::ImageAtomicUMax64); case Decoder::Opcode::IMAGE_ATOMIC_AND: - return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicAnd32); + return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicAnd32, + IR::ValueOpcode::ImageAtomicAnd64); case Decoder::Opcode::IMAGE_ATOMIC_OR: - return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicOr32); + return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicOr32, + IR::ValueOpcode::ImageAtomicOr64); case Decoder::Opcode::IMAGE_ATOMIC_XOR: - return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicXor32); + return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicXor32, + IR::ValueOpcode::ImageAtomicXor64); case Decoder::Opcode::IMAGE_ATOMIC_FMIN: return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicFMin32); case Decoder::Opcode::IMAGE_ATOMIC_FMAX: diff --git a/src/graphics/shader/recompiler/frontend/translate/Translator.h b/src/graphics/shader/recompiler/frontend/translate/Translator.h index eb00ad83..ab6ca393 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Translator.h +++ b/src/graphics/shader/recompiler/frontend/translate/Translator.h @@ -86,7 +86,8 @@ private: void BUFFER_LOAD(const Decoder::Instruction& inst); void BUFFER_STORE(const Decoder::Instruction& inst); void BUFFER_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode); - void IMAGE_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode); + void IMAGE_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode32, + IR::ValueOpcode opcode64 = IR::ValueOpcode::Count); void DS_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool returns_value); void FLAT_LOAD(const Decoder::Instruction& inst); void FLAT_STORE(const Decoder::Instruction& inst); diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp index 656d905b..9cad01fd 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp @@ -224,15 +224,21 @@ ImageOpcodeInfo ImageOpcodeInfoOf(ValueOpcode opcode) { return {ImageAccess::Write, ImageResourceClass::Storage, false}; case ValueOpcode::ImageAtomicCompareSwap32: case ValueOpcode::ImageAtomicSwap32: + case ValueOpcode::ImageAtomicSwap64: case ValueOpcode::ImageAtomicIAdd32: + case ValueOpcode::ImageAtomicIAdd64: case ValueOpcode::ImageAtomicSMin32: case ValueOpcode::ImageAtomicUMin32: + case ValueOpcode::ImageAtomicUMin64: case ValueOpcode::ImageAtomicSMax32: case ValueOpcode::ImageAtomicUMax32: case ValueOpcode::ImageAtomicUMax64: case ValueOpcode::ImageAtomicAnd32: + case ValueOpcode::ImageAtomicAnd64: case ValueOpcode::ImageAtomicOr32: + case ValueOpcode::ImageAtomicOr64: case ValueOpcode::ImageAtomicXor32: + case ValueOpcode::ImageAtomicXor64: case ValueOpcode::ImageAtomicFMin32: case ValueOpcode::ImageAtomicFMax32: return {ImageAccess::Atomic, ImageResourceClass::Storage, false}; diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc index b73b1563..67c310ee 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc @@ -310,15 +310,21 @@ VALUE_OPCODE(ImageSampleRaw, U32x4, ImageResource, SamplerResource, ImageAddress VALUE_OPCODE(ImageGatherRaw, U32x4, ImageResource, SamplerResource, ImageAddress, U32, U32, U32, U32) VALUE_OPCODE(ImageAtomicCompareSwap32, U32, ImageResource, ImageAddress, U32, U32, U1) VALUE_OPCODE(ImageAtomicSwap32, U32, ImageResource, ImageAddress, U32, U1) +VALUE_OPCODE(ImageAtomicSwap64, U64, ImageResource, ImageAddress, U64, U1) VALUE_OPCODE(ImageAtomicIAdd32, U32, ImageResource, ImageAddress, U32, U1) +VALUE_OPCODE(ImageAtomicIAdd64, U64, ImageResource, ImageAddress, U64, U1) VALUE_OPCODE(ImageAtomicSMin32, U32, ImageResource, ImageAddress, U32, U1) VALUE_OPCODE(ImageAtomicUMin32, U32, ImageResource, ImageAddress, U32, U1) +VALUE_OPCODE(ImageAtomicUMin64, U64, ImageResource, ImageAddress, U64, U1) VALUE_OPCODE(ImageAtomicSMax32, U32, ImageResource, ImageAddress, U32, U1) VALUE_OPCODE(ImageAtomicUMax32, U32, ImageResource, ImageAddress, U32, U1) VALUE_OPCODE(ImageAtomicUMax64, U64, ImageResource, ImageAddress, U64, U1) VALUE_OPCODE(ImageAtomicAnd32, U32, ImageResource, ImageAddress, U32, U1) +VALUE_OPCODE(ImageAtomicAnd64, U64, ImageResource, ImageAddress, U64, U1) VALUE_OPCODE(ImageAtomicOr32, U32, ImageResource, ImageAddress, U32, U1) +VALUE_OPCODE(ImageAtomicOr64, U64, ImageResource, ImageAddress, U64, U1) VALUE_OPCODE(ImageAtomicXor32, U32, ImageResource, ImageAddress, U32, U1) +VALUE_OPCODE(ImageAtomicXor64, U64, ImageResource, ImageAddress, U64, U1) VALUE_OPCODE(ImageAtomicFMin32, U32, ImageResource, ImageAddress, U32, U1) VALUE_OPCODE(ImageAtomicFMax32, U32, ImageResource, ImageAddress, U32, U1) VALUE_OPCODE(GetAttribute, U32, U32, U32) diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 9ddebe8c..5d6a25f5 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -33646,20 +33646,68 @@ TestCase ImageAtomicVariants() { return test; } -template TestCase ImageAtomicUMaxGlcAndExec() { - // Input and old texel: high DWORD ordering, unsigned sign boundary, and - // equal-high-DWORD ordering must all survive the RG32_UINT -> R64_UINT view. - constexpr uint64_t values[][2] = { - {0x0000000100000001ull, 0x00000000ffffffffull}, - {0x00000000ffffffffull, 0x0000000100000001ull}, - {0x8000000000000000ull, 0x7fffffffffffffffull}, - {0x12345678ffffffffull, 0x1234567800000001ull}, - {0xffffffffffffffffull, 0x8000000000000000ull}, - }; - constexpr u32 texel_dwords = wide ? 2 : 1; +// Input and old texel cover high-DWORD ordering, carry/wrap, the unsigned sign +// boundary, and equal-high-DWORD ordering through the RG32_UINT -> R64_UINT view. +constexpr uint64_t ImageAtomicInputs[][2] = { + {0x0000000100000001ull, 0x00000000ffffffffull}, + {0x00000000ffffffffull, 0x0000000100000001ull}, + {0x8000000000000000ull, 0x7fffffffffffffffull}, + {0x12345678ffffffffull, 0x1234567800000001ull}, + {0xffffffffffffffffull, 0x8000000000000000ull}, +}; + +struct ImageAtomicIntegerCase { + u32 encoding; + ShaderOpcode opcode; + const char *name32; + const char *name64; + const char *spirv; + uint64_t expected64[std::size(ImageAtomicInputs)]; + u32 expected32[std::size(ImageAtomicInputs)]; +}; + +constexpr ImageAtomicIntegerCase ImageAtomicIntegerCases[] = { + {0x0fu, ShaderOpcode::IMAGE_ATOMIC_SWAP, + "ImageAtomicSwap32GlcAndExec", "ImageAtomicSwap64GlcAndExec", "OpAtomicExchange", + {0x0000000100000001ull, 0x00000000ffffffffull, 0x8000000000000000ull, + 0x12345678ffffffffull, 0xffffffffffffffffull}, + {0x00000001u, 0xffffffffu, 0x00000000u, 0xffffffffu, 0xffffffffu}}, + {0x11u, ShaderOpcode::IMAGE_ATOMIC_ADD, + "ImageAtomicAdd32GlcAndExec", "ImageAtomicAdd64GlcAndExec", "OpAtomicIAdd", + {0x0000000200000000ull, 0x0000000200000000ull, 0xffffffffffffffffull, + 0x2468acf100000000ull, 0x7fffffffffffffffull}, + {0x00000000u, 0x00000000u, 0xffffffffu, 0x00000000u, 0xffffffffu}}, + {0x15u, ShaderOpcode::IMAGE_ATOMIC_UMIN, + "ImageAtomicUMin32GlcAndExec", "ImageAtomicUMin64GlcAndExec", "OpAtomicUMin", + {0x00000000ffffffffull, 0x00000000ffffffffull, 0x7fffffffffffffffull, + 0x1234567800000001ull, 0x8000000000000000ull}, + {0x00000001u, 0x00000001u, 0x00000000u, 0x00000001u, 0x00000000u}}, + {0x17u, ShaderOpcode::IMAGE_ATOMIC_UMAX, + "ImageAtomicUMax32GlcAndExec", "ImageAtomicUMax64GlcAndExec", "OpAtomicUMax", + {0x0000000100000001ull, 0x0000000100000001ull, 0x8000000000000000ull, + 0x12345678ffffffffull, 0xffffffffffffffffull}, + {0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu}}, + {0x18u, ShaderOpcode::IMAGE_ATOMIC_AND, + "ImageAtomicAnd32GlcAndExec", "ImageAtomicAnd64GlcAndExec", "OpAtomicAnd", + {0x0000000000000001ull, 0x0000000000000001ull, 0x0000000000000000ull, + 0x1234567800000001ull, 0x8000000000000000ull}, + {0x00000001u, 0x00000001u, 0x00000000u, 0x00000001u, 0x00000000u}}, + {0x19u, ShaderOpcode::IMAGE_ATOMIC_OR, + "ImageAtomicOr32GlcAndExec", "ImageAtomicOr64GlcAndExec", "OpAtomicOr", + {0x00000001ffffffffull, 0x00000001ffffffffull, 0xffffffffffffffffull, + 0x12345678ffffffffull, 0xffffffffffffffffull}, + {0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu}}, + {0x1au, ShaderOpcode::IMAGE_ATOMIC_XOR, + "ImageAtomicXor32GlcAndExec", "ImageAtomicXor64GlcAndExec", "OpAtomicXor", + {0x00000001fffffffeull, 0x00000001fffffffeull, 0xffffffffffffffffull, + 0x00000000fffffffeull, 0x7fffffffffffffffull}, + {0xfffffffeu, 0xfffffffeu, 0xffffffffu, 0xfffffffeu, 0xffffffffu}}, +}; + +TestCase ImageAtomicIntegerGlcAndExec(const ImageAtomicIntegerCase &atomic, bool wide) { + const u32 texel_dwords = wide ? 2 : 1; TestCase test; - test.name = wide ? "ImageAtomicUMax64CapturedGlcAndExec" - : "ImageAtomicUMax32GlcAndExec"; + test.name = wide ? atomic.name64 : atomic.name32; test.user_data = MakeStorageTextureData(wide ? Prospero::BufferFormat::k32_32UInt : Prospero::BufferFormat::k32UInt); test.has_user_data = true; @@ -33667,25 +33715,21 @@ template TestCase ImageAtomicUMaxGlcAndExec() { test.expected_storage_image_r32ui = test.storage_image_r32ui; auto &code = test.code; code.push_back(EncodeSop1(0x04, 12, 126)); // Preserve EXEC. - for (u32 i = 0; i < std::size(values); ++i) { - const auto input = values[i][0]; - const auto old = values[i][1]; + for (u32 i = 0; i < std::size(ImageAtomicInputs); ++i) { + const auto input = ImageAtomicInputs[i][0]; + const auto old = ImageAtomicInputs[i][1]; const bool glc = i != 0 && i != 4; AppendVMovU32(&code, 0, i & 3u); AppendVMovU32(&code, 1, i >> 2u); AppendVMovLiteral(&code, 2, static_cast(input)); AppendVMovLiteral(&code, 3, static_cast(input >> 32u)); - // Captured PS5 instruction: f05c0308 00000200, DMASK=3, VDATA=v[2:3]. - code.insert(code.end(), {0xf05c0108u | (wide ? 0x200u : 0u) | - (glc ? 0x2000u : 0u), - 0x00000200u}); + code.push_back(EncodeMimg0(atomic.encoding, wide ? 3u : 1u, 0, glc)); + code.push_back(EncodeMimg1(2, 0)); AppendStoreVgpr(&code, 2, i * 2); AppendStoreVgpr(&code, 3, i * 2 + 1); test.expected.push_back(static_cast(glc ? old : input)); test.expected.push_back(static_cast((wide && glc ? old : input) >> 32u)); - const uint64_t result = wide ? std::max(input, old) - : std::max(static_cast(input), - static_cast(old)); + const uint64_t result = wide ? atomic.expected64[i] : atomic.expected32[i]; for (u32 word = 0; word < texel_dwords; ++word) { test.storage_image_r32ui[i * texel_dwords + word] = static_cast(old >> (word * 32u)); @@ -33693,23 +33737,30 @@ template TestCase ImageAtomicUMaxGlcAndExec() { static_cast(result >> (word * 32u)); } } + // A leaked update must change the inactive texel for every operation. + const bool clear_bits = atomic.opcode == ShaderOpcode::IMAGE_ATOMIC_AND || + atomic.opcode == ShaderOpcode::IMAGE_ATOMIC_UMIN; + const u32 inactive = clear_bits ? 0xffffffffu : 0u; + for (u32 word = 0; word < texel_dwords; ++word) { + test.storage_image_r32ui[5u * texel_dwords + word] = inactive; + test.expected_storage_image_r32ui[5u * texel_dwords + word] = inactive; + } AppendVMovU32(&code, 0, 1); - AppendVMovU32(&code, 1, 1); // Untouched zero texel, so an EXEC leak is observable. + AppendVMovU32(&code, 1, 1); AppendVMovLiteral(&code, 2, 0xabcdef01u); AppendVMovLiteral(&code, 3, 0x87654321u); code.push_back(EncodeSop1(0x04, 126, InlineU32(0))); - code.insert(code.end(), {0xf05c2108u | (wide ? 0x200u : 0u), 0x00000200u}); + code.push_back(EncodeMimg0(atomic.encoding, wide ? 3u : 1u, 0, true)); + code.push_back(EncodeMimg1(2, 0)); code.push_back(EncodeSop1(0x04, 126, 12)); - AppendStoreVgpr(&code, 2, std::size(values) * 2); - AppendStoreVgpr(&code, 3, std::size(values) * 2 + 1); + AppendStoreVgpr(&code, 2, std::size(ImageAtomicInputs) * 2); + AppendStoreVgpr(&code, 3, std::size(ImageAtomicInputs) * 2 + 1); test.expected.insert(test.expected.end(), {0xabcdef01u, 0x87654321u}); AppendEnd(&code); test.opcodes = {ShaderOpcode::V_MOV_B32, ShaderOpcode::S_MOV_B64, - ShaderOpcode::IMAGE_ATOMIC_UMAX, ShaderOpcode::BUFFER_STORE_DWORD, - ShaderOpcode::S_ENDPGM}; - test.required_spirv = {"OpAtomicUMax", "OpImageTexelPointer", - wide ? "R64ui" : "R32ui"}; - if constexpr (wide) { + atomic.opcode, ShaderOpcode::BUFFER_STORE_DWORD, ShaderOpcode::S_ENDPGM}; + test.required_spirv = {atomic.spirv, "OpImageTexelPointer", wide ? "R64ui" : "R32ui"}; + if (wide) { test.required_spirv.push_back("Int64ImageEXT"); test.required_spirv.push_back("SPV_EXT_shader_image_int64"); } @@ -34802,8 +34853,10 @@ std::vector MakeCases() { AddCase(ImageAtomicSwapReturnsPreviousTexel); AddCase(ImageStoreAndAtomicUseSeparateBindings); AddCase(ImageAtomicVariants); - AddCase(ImageAtomicUMaxGlcAndExec); - AddCase(ImageAtomicUMaxGlcAndExec); + for (const auto &atomic : ImageAtomicIntegerCases) { + cases.push_back(ImageAtomicIntegerGlcAndExec(atomic, false)); + cases.push_back(ImageAtomicIntegerGlcAndExec(atomic, true)); + } AddCase(ImageAtomicFloatGlcAndExec); AddCase(ImageAtomicFloatGlcAndExec); AddCase(ImageAtomicFloatSpecialValues); @@ -40121,8 +40174,10 @@ int main(int argc, char **argv) { } if (argc == 2 && std::strcmp(argv[1], "--image-atomic64-only") == 0) { VulkanHarness vulkan; - RunCase(&vulkan, ImageAtomicUMaxGlcAndExec()); - RunCase(&vulkan, ImageAtomicUMaxGlcAndExec()); + for (const auto &atomic : ImageAtomicIntegerCases) { + RunCase(&vulkan, ImageAtomicIntegerGlcAndExec(atomic, false)); + RunCase(&vulkan, ImageAtomicIntegerGlcAndExec(atomic, true)); + } return 0; } if (argc == 2 && std::strcmp(argv[1], "--ds-atomics-only") == 0) { diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index 6fa74d59..c38e912d 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -4413,25 +4413,32 @@ void TestImageAtomicWidthDecoder() { decoded.dst.reg == 2u && decoded.dmask == 3u && !decoded.glc, "captured image atomic lost its 64-bit data width"); - for (const auto opcode : {0x10u, 0x17u}) { + for (const auto opcode : {0x0fu, 0x10u, 0x11u, 0x14u, 0x15u, 0x16u, + 0x17u, 0x18u, 0x19u, 0x1au, 0x1eu, 0x1fu}) { + const bool compare_swap = opcode == 0x10u; + const bool supports_64 = opcode != 0x10u && opcode != 0x14u && + opcode != 0x16u && opcode != 0x1eu && opcode != 0x1fu; for (uint32_t mask = 0; mask < 16u; ++mask) { - const uint32_t words[] = {EncodeMimg0(opcode, mask), captured[1]}; - DecodeInstruction(words, 0, decoded); - const bool compare_swap = opcode == 0x10u; - const bool supported = mask == (compare_swap ? 3u : 1u) || - (!compare_swap && mask == 3u); - Check((decoded.opcode != Opcode::UNSUPPORTED) == supported, - "image atomic accepted an invalid or unsupported width mask"); - if (supported) { - Check(decoded.data_bits == (!compare_swap && mask == 3u ? 64u : 32u), - "image atomic DMASK selected the wrong data width"); + for (const bool glc : {false, true}) { + const uint32_t words[] = {EncodeMimg0(opcode, mask, glc), captured[1]}; + DecodeInstruction(words, 0, decoded); + const bool supported = mask == (compare_swap ? 3u : 1u) || + (supports_64 && mask == 3u); + Check((decoded.opcode != Opcode::UNSUPPORTED) == supported, + "image atomic accepted an invalid or unsupported width mask"); + if (supported) { + const auto bits = !compare_swap && mask == 3u ? 64u : 32u; + Check(decoded.data_bits == bits && decoded.glc == glc && + decoded.data_dwords == (compare_swap ? 2u : bits / 32u), + "image atomic DMASK selected the wrong data width or return mode"); + } + const uint32_t d16_words[] = {words[0], words[1] | (1u << 31u)}; + DecodeInstruction(d16_words, 0, decoded); + Check(decoded.opcode == Opcode::UNSUPPORTED, + "image atomic incorrectly accepted D16 data"); } } } - const uint32_t unsupported[] = {EncodeMimg0(0x11u, 3u), captured[1]}; - DecodeInstruction(unsupported, 0, decoded); - Check(decoded.opcode == Opcode::UNSUPPORTED && decoded.data_bits == 64u, - "unsupported 64-bit image atomic silently decoded as 32-bit"); } void TestNewShaderDecoderArchitecture() {