diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h index b835b5a5..bc74f0eb 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h @@ -133,6 +133,7 @@ uint32_t EmitFloatCompare32(ValueEmitContext& ctx, const IR::Inst& inst) { return EmitNative(ctx.state, operand(0), operand(1)); } inline constexpr auto EmitFPOrdEqual32 = EmitFloatCompare32; +EMIT_NATIVE(FPOrdEqual64, OpFOrdEqual, U1, uint32_t, uint32_t) inline constexpr auto EmitFPUnordEqual32 = EmitFloatCompare32; inline constexpr auto EmitFPOrdNotEqual32 = EmitFloatCompare32; inline constexpr auto EmitFPUnordNotEqual32 = EmitFloatCompare32; diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h index 949cde89..3da20d0f 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h @@ -355,6 +355,7 @@ enum class Opcode { V_CMP_NEQ_F32, V_CMP_NLT_F32, V_CMP_TRU_F32, + V_CMP_EQ_F64, V_CMPX_LT_F32, V_CMPX_EQ_F32, V_CMPX_LE_F32, diff --git a/src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp b/src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp index 34dc2ad4..9c56d989 100644 --- a/src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp @@ -228,6 +228,7 @@ constexpr VopcOpcodeInfo VOPC_OPCODE_LIST[] = { {0x19u, Opcode::V_CMPX_NGE_F32}, {0x1au, Opcode::V_CMPX_NLG_F32}, {0x1bu, Opcode::V_CMPX_NGT_F32}, {0x1cu, Opcode::V_CMPX_NLE_F32}, {0x1du, Opcode::V_CMPX_NEQ_F32}, {0x1eu, Opcode::V_CMPX_NLT_F32}, + {0x22u, Opcode::V_CMP_EQ_F64, false}, {0x80u, Opcode::V_CMP_F_I32}, {0x81u, Opcode::V_CMP_LT_I32}, {0x82u, Opcode::V_CMP_EQ_I32}, {0x83u, Opcode::V_CMP_LE_I32}, {0x84u, Opcode::V_CMP_GT_I32}, {0x85u, Opcode::V_CMP_NE_I32}, @@ -829,6 +830,7 @@ bool IsVopcFloatCompareOpcode(Opcode opcode) { case Opcode::V_CMP_F_F32: case Opcode::V_CMP_LT_F32: case Opcode::V_CMP_EQ_F32: + case Opcode::V_CMP_EQ_F64: case Opcode::V_CMP_LE_F32: case Opcode::V_CMP_GT_F32: case Opcode::V_CMP_LG_F32: @@ -1139,7 +1141,7 @@ VopcSdwaFields DecodeVopcSdwaFields(uint32_t modifier) { } bool SupportsVopcSdwa(Opcode opcode) { - return opcode != Opcode::UNSUPPORTED; + return opcode != Opcode::UNSUPPORTED && opcode != Opcode::V_CMP_EQ_F64; } void DecodeVopcSdwa(uint32_t pc, std::span code, uint32_t word_index, diff --git a/src/graphics/shader/recompiler/frontend/translate/Compare.cpp b/src/graphics/shader/recompiler/frontend/translate/Compare.cpp index e864e9c2..59208f1e 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Compare.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Compare.cpp @@ -40,11 +40,13 @@ void Translator::EmitInteger16Compare(const Decoder::Instruction& inst, IR::Valu void Translator::EmitFloatCompare(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool half, bool cmpx) { + const auto type = IR::ArgTypeOf(opcode, 0); const auto lhs = - half ? IR::Value(ReadF16AsF32(inst.src0)) : ReadOperand(inst.src0, IR::Type::F32); + half ? IR::Value(ReadF16AsF32(inst.src0)) : ReadOperand(inst.src0, type); const auto rhs = - half ? IR::Value(ReadF16AsF32(inst.src1)) : ReadOperand(inst.src1, IR::Type::F32); - const IR::FPCompareFlags flags{.flush_input_denorms = !half && flush_f32_inputs}; + half ? IR::Value(ReadF16AsF32(inst.src1)) : ReadOperand(inst.src1, type); + const IR::FPCompareFlags flags{ + .flush_input_denorms = !half && type == IR::Type::F32 && flush_f32_inputs}; EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {lhs, rhs}, flags)), false, cmpx); } diff --git a/src/graphics/shader/recompiler/frontend/translate/Translate.cpp b/src/graphics/shader/recompiler/frontend/translate/Translate.cpp index 29afa14c..3479d5a7 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Translate.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Translate.cpp @@ -889,6 +889,10 @@ void IncludeInstructionVectorRegisters(const Decoder::Instruction& inst, uint32_ case Decoder::Opcode::V_MUL_F64: include_vector(inst.src1, 2u); [[fallthrough]]; case Decoder::Opcode::V_RCP_F64: include_vector(inst.dst, 2u); [[fallthrough]]; case Decoder::Opcode::V_CVT_F32_F64: include_vector(inst.src0, 2u); break; + case Decoder::Opcode::V_CMP_EQ_F64: + include_vector(inst.src0, 2u); + include_vector(inst.src1, 2u); + break; default: break; } if (inst.family == Decoder::Family::DS) { diff --git a/src/graphics/shader/recompiler/frontend/translate/Vector.cpp b/src/graphics/shader/recompiler/frontend/translate/Vector.cpp index 14a89ca4..23558d5f 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Vector.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Vector.cpp @@ -168,6 +168,9 @@ void Translator::EmitVector(const Decoder::Instruction& inst) { case O::V_CMP_EQ_F32: EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual32, false, false); return; + case O::V_CMP_EQ_F64: + EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual64, false, false); + return; case O::V_CMPX_EQ_F32: EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual32, false, true); return; diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc index b6d4a8c5..273079ca 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc @@ -155,6 +155,7 @@ VALUE_OPCODE(LogicalAnd, U1, U1, U1) VALUE_OPCODE(LogicalXor, U1, U1, U1) VALUE_OPCODE(LogicalNot, U1, U1) VALUE_OPCODE(FPOrdEqual32, U1, F32, F32) +VALUE_OPCODE(FPOrdEqual64, U1, F64, F64) VALUE_OPCODE(FPUnordEqual32, U1, F32, F32) VALUE_OPCODE(FPOrdNotEqual32, U1, F32, F32) VALUE_OPCODE(FPUnordNotEqual32, U1, F32, F32) diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 18c95754..cb2dbae3 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -22567,6 +22567,101 @@ TestCase VectorSpecialF32FlushesDenormalInputs() { O::V_SQRT_F32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } +TestCase VectorCompareF64Edges() { + using O = ShaderOpcode; + constexpr std::array, 12> pairs{{ + {0, 0x8000000000000000ull}, {1, 0}, {1, 1}, {1, 2}, + {0x11d04c7b00ull, 0}, {0x3ff0000000000000ull, 0x3ff0000000000001ull}, + {0xbff0000000000000ull, 0xbff0000000000000ull}, + {0x7ff0000000000000ull, 0x7ff0000000000000ull}, + {0x7ff0000000000000ull, 0xfff0000000000000ull}, + {0x7ff8000000000001ull, 0x7ff8000000000001ull}, + {0x7ff0000000000001ull, 0}, {0x7fefffffffffffffull, 0x7fefffffffffffffull}}}; + TestCase test; + test.name = "VectorCompareF64Edges"; + for (const auto& pair : pairs) + for (const auto bits : pair) + test.initial.insert(test.initial.end(), {u32(bits), u32(bits >> 32)}); + test.expected = test.initial; + const auto store_mask = [&](u32 sgpr, bool value) { + AppendStoreSgprPair(&test.code, sgpr, static_cast(test.expected.size())); + test.expected.insert(test.expected.end(), {u32(value), 0}); + }; + for (u32 i = 0; i < pairs.size(); ++i) { + for (u32 word = 0; word < 4; ++word) { + AppendVMovU32(&test.code, 31, (i * 4 + word) * 4); + AppendBufferLoadDword(&test.code, 1 + word, 31); + } + const auto lhs = std::bit_cast(pairs[i][0]); + const auto rhs = std::bit_cast(pairs[i][1]); + test.code.push_back(EncodeVopc(0x22, Vgpr(1), 3)); + store_mask(106, lhs == rhs); + AppendVop3(&test.code, 0x22, 20, Vgpr(1), Vgpr(3), 0, 1, 0, false, 0, 1); + store_mask(20, -std::abs(lhs) == rhs); + test.code.push_back(EncodeVop1(0x02, 100, Vgpr(3))); + test.code.push_back(EncodeVop1(0x02, 101, Vgpr(4))); + // Captured e6d76d24f59f8015 PC 0xe4: compare zero with s100:s101. + test.code.insert(test.code.end(), {0xd422006au, 0x0000c880u}); + store_mask(106, 0.0 == rhs); + } + AppendEnd(&test.code); + test.initial.resize(test.expected.size()); + test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_READFIRSTLANE_B32, + O::V_CMP_EQ_F64, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.required_spirv = {"OpCapability Float64", "OpTypeFloat 64", "OpFOrdEqual"}; + return test; +} + +TestCase VectorCompareF64WaveMasks(u32 wave_size) { + using O = ShaderOpcode; + constexpr std::array values{0, 1, 0x8000000000000000ull, + 0x7ff8000000000000ull}; + constexpr u32 low_exec = 0xeeeeeeeeu, high_exec = 0x77777777u; + constexpr u32 vcc_hi = 0x89abcdefu; + TestCase test; + test.name = wave_size == 32 ? "VectorCompareF64Wave32Masks" : "VectorCompareF64Wave64Masks"; + test.initial.assign(6 * wave_size, 0); + for (u32 lane = 0; lane < wave_size; ++lane) { + test.initial[lane] = u32(values[lane % 4]); + test.initial[wave_size + lane] = u32(values[lane % 4] >> 32); + } + test.expected = test.initial; + for (u32 lane = 0; lane < wave_size; ++lane) { + test.expected[2 * wave_size + lane] = low_exec & 0x55555555u; + test.expected[3 * wave_size + lane] = wave_size == 64 ? high_exec & 0x55555555u : vcc_hi; + test.expected[4 * wave_size + lane] = low_exec; + test.expected[5 * wave_size + lane] = wave_size == 64 ? high_exec : 0; + } + auto& code = test.code; + code.push_back(EncodeVop2(0x1a, 30, InlineU32(2), 0)); + AppendBufferLoadDword(&code, 1, 30); + AppendVMovU32(&code, 31, wave_size * 4); + code.push_back(EncodeVop2(0x25, 31, Vgpr(31), 30)); + AppendBufferLoadDword(&code, 2, 31); + code.push_back(EncodeSop1(0x04, 10, 126)); + AppendSMovLiteral(&code, 107, vcc_hi); + AppendSMovLiteral(&code, 126, low_exec); + if (wave_size == 64) AppendSMovLiteral(&code, 127, high_exec); + AppendVop3(&code, 0x22, 106, InlineU32(0), Vgpr(1)); + code.push_back(EncodeSMovB32(20, 126)); + code.push_back(EncodeSMovB32(21, 127)); + code.push_back(EncodeSop1(0x04, 126, 10)); + for (const auto [reg, offset] : std::array, 4>{{ + {106, 2}, {107, 3}, {20, 4}, {21, 5}}}) + AppendStoreSgprAtLaneDwordOffset(&code, reg, 0, offset * wave_size); + AppendEnd(&code); + test.compute_info.threads_num[0] = wave_size; + test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.compute_info.wave_size = wave_size; + test.has_compute_info = true; + test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::S_MOV_B64, O::V_LSHLREV_B32, + O::V_ADD_NC_U32, O::BUFFER_LOAD_DWORD, O::V_CMP_EQ_F64, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.required_spirv = {"OpFOrdEqual"}; + return test; +} + TestCase VectorF64CapturedScreenSpaceShadows() { using O = ShaderOpcode; TestCase test; @@ -30773,6 +30868,9 @@ std::vector MakeCases() { AddCase(CvtF32ToIntSaturatesNaNAndOutOfRange); AddCase(VectorSpecialF32FlushesDenormalInputs); AddCase(VectorRcpIflagF32IntegerReciprocal); + AddCase(VectorCompareF64Edges); + cases.push_back(VectorCompareF64WaveMasks(32)); + cases.push_back(VectorCompareF64WaveMasks(64)); AddCase(VectorF64CapturedScreenSpaceShadows); AddCase(VectorF64ModesModifiersAndExec); AddCase(VectorF64WideningConversions); @@ -35749,6 +35847,15 @@ int main(int argc, char **argv) { RunCase(&vulkan, VectorVopcSdwaCmpxClassF32CapturedExecMask()); return 0; } + if (argc == 2 && std::strcmp(argv[1], "--cmp-eq-f64-only") == 0) { + VulkanHarness vulkan; + RunCase(&vulkan, VectorCompareF64Edges()); + RunCase(&vulkan, VectorCompareF64WaveMasks(32)); + RunCase(&vulkan, VectorCompareF64WaveMasks(64)); + RunCase(&vulkan, VectorF64CapturedScreenSpaceShadows()); + RunCase(&vulkan, VectorF64ModesModifiersAndExec()); + return 0; + } if (argc == 2 && std::strcmp(argv[1], "--cmp-ngt-f16-only") == 0) { VulkanHarness vulkan; RunCase(&vulkan, VectorVopcCmpNgtF16CapturedSdwaAndEdges());