diff --git a/src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp b/src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp index f4836666..d8c78e03 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp @@ -47,8 +47,12 @@ constexpr OpcodeMap SOP1_OPCODE_LIST[] = { {0x0au, Opcode::S_WQM_B64}, {0x0bu, Opcode::S_BREV_B32}, {0x0cu, Opcode::S_BREV_B64}, + {0x0du, Opcode::S_BCNT0_I32_B32}, + {0x0eu, Opcode::S_BCNT0_I32_B64}, {0x0fu, Opcode::S_BCNT1_I32_B32}, {0x10u, Opcode::S_BCNT1_I32_B64}, + {0x11u, Opcode::S_FF0_I32_B32}, + {0x12u, Opcode::S_FF0_I32_B64}, {0x13u, Opcode::S_FF1_I32_B32}, {0x14u, Opcode::S_FF1_I32_B64}, {0x15u, Opcode::S_FLBIT_I32_B32}, diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp index 5cd02679..0df4218d 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp @@ -510,6 +510,10 @@ std::string InstructionToString(const Instruction& inst) { case Opcode::S_SEXT_I32_I16: case Opcode::S_BREV_B32: case Opcode::S_BREV_B64: + case Opcode::S_BCNT0_I32_B32: + case Opcode::S_BCNT0_I32_B64: + case Opcode::S_FF0_I32_B32: + case Opcode::S_FF0_I32_B64: case Opcode::S_BCNT1_I32_B32: case Opcode::S_FLBIT_I32_B32: case Opcode::S_FF1_I32_B32: diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h index 3ef35ff9..4277374c 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h @@ -50,8 +50,12 @@ enum class Opcode { S_ABSDIFF_I32, S_BREV_B32, S_BREV_B64, + S_BCNT0_I32_B32, + S_BCNT0_I32_B64, S_BCNT1_I32_B32, S_BCNT1_I32_B64, + S_FF0_I32_B32, + S_FF0_I32_B64, S_FF1_I32_B32, S_FF1_I32_B64, S_FLBIT_I32_B32, diff --git a/src/graphics/shader/recompiler/frontend/translate/Integer.cpp b/src/graphics/shader/recompiler/frontend/translate/Integer.cpp index 01e5645d..b5d06740 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Integer.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Integer.cpp @@ -221,8 +221,12 @@ void Translator::V_XOR3_B32(const Decoder::Instruction& inst) { WriteOperand(DestinationOperand(inst), result); } -void Translator::S_FF1_I32_B64(const Decoder::Instruction& inst) { - const auto source = ExtractU64(ReadU64(inst.src0)); +void Translator::S_FF_I32_B64(const Decoder::Instruction& inst, bool find_zero) { + auto source = ReadU32Pair(inst.src0); + if (find_zero) { + source[0] = ir.BitwiseNot(source[0]); + source[1] = ir.BitwiseNot(source[1]); + } const auto low_lsb = IR::U32(ir.Emit(IR::ValueOpcode::FindILsb32, {source[0]})); const auto high_lsb = IR::U32(ir.Emit(IR::ValueOpcode::FindILsb32, {source[1]})); const auto high_position = ir.IAdd(high_lsb, IR::U32(IR::Value(32u))); diff --git a/src/graphics/shader/recompiler/frontend/translate/Scalar.cpp b/src/graphics/shader/recompiler/frontend/translate/Scalar.cpp index b08797d1..519014fc 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Scalar.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Scalar.cpp @@ -178,12 +178,29 @@ void Translator::EmitScalar(const Decoder::Instruction& inst) { IR::U32(ir.Emit(IR::ValueOpcode::BitReverse32, {value[0]}))}); return; } + case O::S_BCNT0_I32_B32: + case O::S_FF0_I32_B32: { + const auto value = ir.BitwiseNot(ReadU32(inst.src0)); + const bool count = inst.opcode == O::S_BCNT0_I32_B32; + const auto result = IR::U32(ir.Emit(count ? IR::ValueOpcode::BitCount32 + : IR::ValueOpcode::FindILsb32, {value})); + WriteOperand(inst.dst, result); + if (count) { + ir.SetScc(ir.INotEqual(result, IR::U32(IR::Value(0u)))); + } + return; + } case O::S_BCNT1_I32_B32: return SimpleInteger(inst, IR::ValueOpcode::BitCount32, IR::Type::U32, false, false, true); + case O::S_BCNT0_I32_B64: case O::S_BCNT1_I32_B64: { // Vulkan bit counts operate on 32-bit words; avoid packing only to split again. - const auto value = ReadU32Pair(inst.src0); + auto value = ReadU32Pair(inst.src0); + if (inst.opcode == O::S_BCNT0_I32_B64) { + value[0] = ir.BitwiseNot(value[0]); + value[1] = ir.BitwiseNot(value[1]); + } const auto low = IR::U32(ir.Emit(IR::ValueOpcode::BitCount32, {value[0]})); const auto high = IR::U32(ir.Emit(IR::ValueOpcode::BitCount32, {value[1]})); const auto result = ir.IAdd(low, high); @@ -221,7 +238,8 @@ void Translator::EmitScalar(const Decoder::Instruction& inst) { return ComposedIntegerBinary(inst, IR::ValueOpcode::BitwiseOr32, false, true, true); case O::S_XNOR_B32: return ComposedIntegerBinary(inst, IR::ValueOpcode::BitwiseXor32, false, true, true); - case O::S_FF1_I32_B64: return S_FF1_I32_B64(inst); + case O::S_FF0_I32_B64: return S_FF_I32_B64(inst, true); + case O::S_FF1_I32_B64: return S_FF_I32_B64(inst, false); case O::S_FLBIT_I32_B32: return V_FFBH_32(inst, false); case O::S_FLBIT_I32_B64: return S_FLBIT_I32_B64(inst); diff --git a/src/graphics/shader/recompiler/frontend/translate/Translator.h b/src/graphics/shader/recompiler/frontend/translate/Translator.h index 7381d788..b3e95a5c 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Translator.h +++ b/src/graphics/shader/recompiler/frontend/translate/Translator.h @@ -187,7 +187,7 @@ private: void V_AND_OR_B32(const Decoder::Instruction& inst); void V_OR3_B32(const Decoder::Instruction& inst); void V_XOR3_B32(const Decoder::Instruction& inst); - void S_FF1_I32_B64(const Decoder::Instruction& inst); + void S_FF_I32_B64(const Decoder::Instruction& inst, bool find_zero); void V_FFBH_32(const Decoder::Instruction& inst, bool sign); void S_FLBIT_I32_B64(const Decoder::Instruction& inst); void Integer24(const Decoder::Instruction& inst, bool sign, bool addend); diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 972bb582..c34db80f 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -20860,6 +20860,66 @@ TestCase ScalarBrevB32PreservesScc() { O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } +TestCase ScalarZeroBitOps() { + using O = ShaderOpcode; + constexpr uint64_t inputs[] = {0, UINT64_MAX, 0xaaaaaaaa55555555ull, + 0xffffffff7fffffffull, 0xfffffffeffffffffull, 0x7fffffffffffffffull, + 0x00000000ffffffffull, 0xffffffff00000000ull}; + TestCase test; + test.name = "ScalarZeroBitOps"; + for (const auto input : inputs) + test.initial.insert(test.initial.end(), {u32(input), u32(input >> 32)}); + test.expected = test.initial; + auto &code = test.code; + const auto check = [&](u32 opcode, u32 source, uint64_t input, bool scc) { + const u32 width = opcode == 0x0e || opcode == 0x12 ? 64 : 32; + u32 count = 0, first = UINT32_MAX; + for (u32 bit = 0; bit < width; ++bit) { + if (((input >> bit) & 1u) == 0) { + ++count; + if (first == UINT32_MAX) first = bit; + } + } + const bool counting = opcode == 0x0d || opcode == 0x0e; + code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(scc))); + code.push_back(EncodeSop1(opcode, 20, source)); + if (source == 255) code.push_back(u32(input)); + code.push_back(EncodeSMovB32(22, 253)); // Capture SCC before readback. + AppendStoreSgpr(&code, 20, u32(test.expected.size())); + AppendStoreSgpr(&code, 21, u32(test.expected.size() + 1)); + AppendStoreSgpr(&code, 22, u32(test.expected.size() + 2)); + test.expected.insert(test.expected.end(), + {counting ? count : first, u32(input >> 32), u32(counting ? count != 0 : scc)}); + }; + for (u32 index = 0; index < std::size(inputs); ++index) { + for (u32 word = 0; word < 2; ++word) { + AppendVMovU32(&code, 30, (index * 2 + word) * sizeof(u32)); + AppendBufferLoadDword(&code, 1, 30); + code.push_back(EncodeVop1(0x02, 8 + word, Vgpr(1))); + } + for (const u32 opcode : {0x0du, 0x0eu, 0x11u, 0x12u}) { + for (const bool scc : {false, true}) { + code.push_back(EncodeSop1(0x04, 20, 8)); // Destination aliases the source. + check(opcode, 20, inputs[index], scc); + } + } + } + for (const u32 opcode : {0x0du, 0x0eu, 0x11u, 0x12u}) { + for (const u32 source : {128u, 193u, 255u}) { + const uint64_t input = source == 128 ? 0 : source == 193 ? UINT64_MAX : 0xffffffffull; + AppendSMovLiteral(&code, 21, u32(input >> 32)); + check(opcode, source, input, true); // Only inline -1 sign extends to B64. + } + } + AppendEnd(&code); + test.initial.resize(test.expected.size()); + test.opcodes = {O::S_BCNT0_I32_B32, O::S_BCNT0_I32_B64, + O::S_FF0_I32_B32, O::S_FF0_I32_B64, O::S_CMP_EQ_U32, O::S_MOV_B32, + O::S_MOV_B64, O::V_MOV_B32, O::V_READFIRSTLANE_B32, + O::BUFFER_LOAD_DWORD, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + return test; +} + TestCase ScalarSextMasks(u32 bits, u32 wave_size) { using O = ShaderOpcode; TestCase test; @@ -37061,6 +37121,7 @@ std::vector MakeCases() { cases.push_back(ScalarSubvectorLoops(64)); AddCase(ScalarGetpcWritesNextInstructionPc); AddCase(ScalarBitfieldPack); + AddCase(ScalarZeroBitOps); AddCase(ScalarBitcmpB64DynamicOperands); AddCase(ScalarBitcmpB64IntegerConstants); AddCase(ScalarBrevB32PreservesScc); @@ -42892,6 +42953,12 @@ int main(int argc, char **argv) { RunCase(&vulkan, Vop3pInt16Ops()); return 0; } + if (argc == 2 && std::strcmp(argv[1], "--scalar-zero-bit-only") == 0) { + VulkanHarness vulkan; + RunCase(&vulkan, ScalarZeroBitOps()); + RunCase(&vulkan, ScalarBitfieldPack()); + return 0; + } if (argc == 2 && std::strcmp(argv[1], "--buffer-format-store-only") == 0) { VulkanHarness vulkan; RunCase(&vulkan, BufferStoreFormatXyzwPackedUnormSkinningVectors());