#include "ControlFlow/GraphBuilder.hpp" #include "ControlFlow/Structurizer.hpp" #include "IntermediateRepresentation/IrProgram.hpp" #include "RdnaDecoder/RdnaInstructionDecoder.hpp" #include "Translation/InstructionTranslator.hpp" #include "Translation/ShaderInputInfoBuilder.hpp" #include #include #include #include #include #include #include #include #include using namespace ShaderRecompiler; namespace { constexpr std::uint32_t Lanes = 64; using LaneWords = std::array; struct Value { std::array component{}; }; class Evaluator { public: Evaluator(const IrProgram& program, std::uint32_t waveSize) : lanes(waveSize), exec(waveSize >= 64u ? ~0ull : (1ull << waveSize) - 1ull) { for (std::uint32_t lane = 0; lane < Lanes; ++lane) vector[0][lane] = lane; for (const auto& block : program.Blocks()) { for (const auto* inst : block->Instructions()) run(*inst); } } LaneWords output{}; bool exported = false; std::map maskTag; std::map threadBitValue; private: std::unordered_map values; std::array vector{}; std::array scalar{}; std::map threadBit; std::uint32_t lanes; std::uint64_t exec; std::uint64_t vcc = 0; bool scc = false; static std::uint32_t registerIndex(const IrValue& inst) { return inst.Argument(0)->Register().index; } Value operand(const IrValue* argument) { argument = argument->Resolve(); const auto found = values.find(argument); if (found != values.end()) return found->second; if (!argument->HasImmediate()) throw std::runtime_error("the evaluator met a value that has not been computed"); Value constant{}; const std::uint32_t word = argument->Type() == IrType::Bool ? (argument->ImmediateBool() ? 1u : 0u) : argument->ImmediateU32(); constant.component[0].fill(word); return constant; } Value fromBits(std::uint64_t bits) const { Value result{}; for (std::uint32_t lane = 0; lane < lanes; ++lane) result.component[0][lane] = static_cast((bits >> lane) & 1u); return result; } std::uint64_t toBits(const Value& value) const { std::uint64_t bits = 0; for (std::uint32_t lane = 0; lane < lanes; ++lane) bits |= static_cast(value.component[0][lane] & 1u) << lane; return bits; } static Value uniform(std::uint32_t word) { Value result{}; result.component[0].fill(word); return result; } std::uint32_t uniformWord(const Value& value) const { for (std::uint32_t lane = 1; lane < lanes; ++lane) { if (value.component[0][lane] != value.component[0][0]) throw std::runtime_error("a scalar register was written with a value that differs between lanes"); } return value.component[0][0]; } template Value lanewise(const IrValue& inst, TOperation operation) { const Value left = operand(inst.Argument(0)); const Value right = operand(inst.Argument(1)); Value result{}; for (std::uint32_t lane = 0; lane < Lanes; ++lane) result.component[0][lane] = operation(left.component[0][lane], right.component[0][lane]); return result; } Value select(const IrValue& inst) { const Value condition = operand(inst.Argument(0)); const Value active = operand(inst.Argument(1)); const Value inactive = operand(inst.Argument(2)); Value result{}; for (std::uint32_t lane = 0; lane < Lanes; ++lane) { result.component[0][lane] = (condition.component[0][lane] & 1u) != 0u ? active.component[0][lane] : inactive.component[0][lane]; } return result; } void run(const IrValue& inst) { switch (inst.Opcode()) { case IrOpcode::SetExec: exec = toBits(operand(inst.Argument(0))); return; case IrOpcode::SetExecLo: exec = (exec & ~0xffffffffull) | uniformWord(operand(inst.Argument(0))); return; case IrOpcode::SetExecHi: exec = (exec & 0xffffffffull) | (static_cast(uniformWord(operand(inst.Argument(0)))) << 32); return; case IrOpcode::GetExec: values[&inst] = fromBits(exec); return; case IrOpcode::GetExecLo: values[&inst] = uniform(static_cast(exec)); return; case IrOpcode::GetExecHi: values[&inst] = uniform(static_cast(exec >> 32)); return; case IrOpcode::SetVcc: vcc = toBits(operand(inst.Argument(0))); return; case IrOpcode::SetVccLo: vcc = (vcc & ~0xffffffffull) | uniformWord(operand(inst.Argument(0))); return; case IrOpcode::SetVccHi: vcc = (vcc & 0xffffffffull) | (static_cast(uniformWord(operand(inst.Argument(0)))) << 32); return; case IrOpcode::GetVcc: values[&inst] = fromBits(vcc); return; case IrOpcode::GetVccLo: values[&inst] = uniform(static_cast(vcc)); return; case IrOpcode::GetVccHi: values[&inst] = uniform(static_cast(vcc >> 32)); return; case IrOpcode::SetScc: scc = (uniformWord(operand(inst.Argument(0))) & 1u) != 0u; return; case IrOpcode::GetScc: values[&inst] = uniform(scc ? 1u : 0u); return; case IrOpcode::SetScalarRegister: scalar.at(registerIndex(inst)) = uniformWord(operand(inst.Argument(1))); return; case IrOpcode::GetScalarRegister: values[&inst] = uniform(scalar.at(registerIndex(inst))); return; case IrOpcode::SetScalarMaskTag: maskTag[registerIndex(inst)] = (uniformWord(operand(inst.Argument(1))) & 1u) != 0u; return; case IrOpcode::GetScalarMaskTag: { const auto found = maskTag.find(registerIndex(inst)); values[&inst] = uniform(found != maskTag.end() && found->second ? 1u : 0u); return; } case IrOpcode::SetThreadBitScalarRegister: threadBit[registerIndex(inst)] = operand(inst.Argument(1)).component[0]; threadBitValue[registerIndex(inst)] = inst.Argument(1)->Resolve(); return; case IrOpcode::GetThreadBitScalarRegister: { Value result{}; const auto found = threadBit.find(registerIndex(inst)); if (found != threadBit.end()) result.component[0] = found->second; values[&inst] = result; return; } case IrOpcode::SetVectorRegister: vector.at(registerIndex(inst)) = operand(inst.Argument(1)).component[0]; return; case IrOpcode::GetVectorRegister: { Value result{}; result.component[0] = vector.at(registerIndex(inst)); values[&inst] = result; return; } case IrOpcode::GetBuiltin: values[&inst] = uniform(0u); return; case IrOpcode::LaneId: { Value result{}; for (std::uint32_t lane = 0; lane < Lanes; ++lane) result.component[0][lane] = lane; values[&inst] = result; return; } case IrOpcode::Ballot: { const std::uint64_t bits = toBits(operand(inst.Argument(0))); Value result{}; result.component[0].fill(static_cast(bits)); result.component[1].fill(static_cast(bits >> 32)); values[&inst] = result; return; } case IrOpcode::CompositeExtractU32x4: { const Value source = operand(inst.Argument(0)); Value result{}; result.component[0] = source.component.at(operand(inst.Argument(1)).component[0][0]); values[&inst] = result; return; } case IrOpcode::CompositeConstructU32x4: { Value result{}; for (std::uint32_t index = 0; index < 4; ++index) result.component[index] = operand(inst.Argument(index)).component[0]; values[&inst] = result; return; } case IrOpcode::UGreaterThan32: values[&inst] = lanewise(inst, [](std::uint32_t a, std::uint32_t b) { return a > b ? 1u : 0u; }); return; case IrOpcode::ULessThan32: values[&inst] = lanewise(inst, [](std::uint32_t a, std::uint32_t b) { return a < b ? 1u : 0u; }); return; case IrOpcode::IEqual32: values[&inst] = lanewise(inst, [](std::uint32_t a, std::uint32_t b) { return a == b ? 1u : 0u; }); return; case IrOpcode::INotEqual32: values[&inst] = lanewise(inst, [](std::uint32_t a, std::uint32_t b) { return a != b ? 1u : 0u; }); return; case IrOpcode::LogicalAnd: values[&inst] = lanewise(inst, [](std::uint32_t a, std::uint32_t b) { return (a & b) & 1u; }); return; case IrOpcode::BitwiseAnd32: values[&inst] = lanewise(inst, [](std::uint32_t a, std::uint32_t b) { return a & b; }); return; case IrOpcode::ShiftRightLogical32: values[&inst] = lanewise(inst, [](std::uint32_t a, std::uint32_t b) { return a >> (b & 31u); }); return; case IrOpcode::SelectU1: case IrOpcode::SelectU32: values[&inst] = select(inst); return; case IrOpcode::SetAttribute: output = operand(inst.Argument(0)).component[0]; exported = true; return; default: throw std::runtime_error("the evaluator does not support IR opcode " + std::to_string(static_cast(inst.Opcode()))); } } }; constexpr std::uint32_t Vcc = 0x6au; constexpr std::uint32_t Exec = 0x7eu; constexpr std::uint32_t Zero = 0x80u; constexpr std::uint32_t MinusOne = 0xc1u; constexpr std::uint32_t S4 = 0x04u; constexpr std::uint32_t Literal = 0xffu; std::uint32_t vcmpLanesBelow(std::uint32_t count) { return 0x7d880080u + count; } std::uint32_t sCmpScc(bool value) { return value ? 0xbf068080u : 0xbf068180u; } std::uint32_t sCselectB64(std::uint32_t destination, std::uint32_t trueSource, std::uint32_t falseSource) { return 0x85800000u | (destination << 16) | (falseSource << 8) | trueSource; } std::uint32_t sMovB64(std::uint32_t destination, std::uint32_t source) { return 0xbe800400u | (destination << 16) | source; } std::array vCndmaskV1(std::uint32_t mask) { return {0xd5010001u, 0x80u | (129u << 9) | (mask << 18)}; } struct Outcome { LaneWords lanes{}; std::uint32_t waveSize = 64; bool maskTagS4 = false; bool constantSources = false; }; Outcome run(const std::vector& body, std::uint32_t mask, std::uint32_t waveSize = 64) { std::vector code = body; const auto select = vCndmaskV1(mask); code.insert(code.end(), {select[0], select[1], 0xf80008cfu, 0x01010101u, 0xbf810000u}); const auto decoded = RdnaInstructionDecoder{}.Decode(code); auto cfg = GraphBuilder{}.Build(decoded); Structurizer{}.Structurize(cfg); GuestContext context{}; context.waveSize = waveSize; context.vertex = ShaderVertexStageInfo{}; TranslateOptions options{}; options.stage = ShaderStageKind::Vertex; options.waveSize = waveSize; options.userDataCount = 0; options.inputInfo = BuildShaderStageInputInfo(ShaderStageKind::Vertex, context, waveSize); const auto program = InstructionTranslator{}.Translate(decoded, cfg, options); const Evaluator evaluator(program, waveSize); if (!evaluator.exported) throw std::runtime_error("the program exported nothing"); Outcome outcome; outcome.lanes = evaluator.output; outcome.waveSize = waveSize; const auto tag = evaluator.maskTag.find(S4); outcome.maskTagS4 = tag != evaluator.maskTag.end() && tag->second; const auto bit = evaluator.threadBitValue.find(S4); if (bit != evaluator.threadBitValue.end() && bit->second->Opcode() == IrOpcode::SelectU1) { outcome.constantSources = bit->second->Argument(1)->Resolve()->HasImmediate() && bit->second->Argument(2)->Resolve()->HasImmediate(); } return outcome; } int failures = 0; void expectLanes(const char* name, const Outcome& outcome, std::uint64_t expectedBits) { for (std::uint32_t lane = 0; lane < outcome.waveSize; ++lane) { const std::uint32_t expected = static_cast((expectedBits >> lane) & 1u); if (outcome.lanes[lane] != expected) { std::fprintf(stderr, "%s: lane %u holds %u, expected %u (expected mask 0x%016llx)\n", name, lane, outcome.lanes[lane], expected, static_cast(expectedBits)); ++failures; return; } } } void expectMaskPath(const char* name, const Outcome& outcome) { if (!outcome.maskTagS4) { std::fprintf(stderr, "%s: the destination pair was not marked as a mask\n", name); ++failures; } } void expectConstantSources(const char* name, const Outcome& outcome, bool expected) { if (outcome.constantSources != expected) { std::fprintf(stderr, "%s: the sources of the select are %sconstants\n", name, expected ? "not " : ""); ++failures; } } constexpr std::uint64_t AllLanes = ~0ull; std::uint64_t below(std::uint32_t count) { return count >= 64u ? AllLanes : (1ull << count) - 1ull; } void selectCase(const char* name, std::vector setup, std::uint32_t trueSource, std::uint32_t falseSource, std::uint64_t whenSccSet, std::uint64_t whenSccClear, bool maskPath, int constantSources = -1) { for (const bool sccValue : {true, false}) { auto body = setup; body.push_back(sCmpScc(sccValue)); body.push_back(sCselectB64(4u, trueSource, falseSource)); const auto outcome = run(body, S4); const std::string label = std::string(name) + (sccValue ? ", scc = 1" : ", scc = 0"); expectLanes(label.c_str(), outcome, sccValue ? whenSccSet : whenSccClear); if (maskPath) expectMaskPath(label.c_str(), outcome); if (constantSources >= 0) expectConstantSources(label.c_str(), outcome, constantSources != 0); } } } int main() { try { selectCase("vcc, exec", {vcmpLanesBelow(4)}, Vcc, Exec, below(4), AllLanes, true); selectCase("exec, vcc", {vcmpLanesBelow(4)}, Exec, Vcc, AllLanes, below(4), true); selectCase("wave64 halves, vcc, exec", {vcmpLanesBelow(40)}, Vcc, Exec, below(40), AllLanes, true); selectCase("wave64 halves, exec, vcc", {vcmpLanesBelow(40)}, Exec, Vcc, AllLanes, below(40), true); selectCase("exec, zero", {}, Exec, Zero, AllLanes, 0ull, true, 0); selectCase("zero, exec", {}, Zero, Exec, 0ull, AllLanes, true, 0); selectCase("zero, zero", {}, Zero, Zero, 0ull, 0ull, true, 1); selectCase("minus one, zero", {}, MinusOne, Zero, AllLanes, 0ull, true, 1); selectCase("minus one, exec", {}, MinusOne, Exec, AllLanes, AllLanes, true, 0); selectCase("destination is the true source", {vcmpLanesBelow(40), sMovB64(4u, Vcc)}, S4, Exec, below(40), AllLanes, false); selectCase("destination is the false source", {vcmpLanesBelow(40), sMovB64(4u, Vcc)}, Exec, S4, AllLanes, below(40), false); for (const bool sccValue : {true, false}) { const auto outcome = run({sCmpScc(sccValue), sCselectB64(4u, Literal, Exec), 0x0000ffffu}, S4); const std::string label = std::string("literal 0xffff, exec") + (sccValue ? ", scc = 1" : ", scc = 0"); expectLanes(label.c_str(), outcome, sccValue ? below(16) : AllLanes); } for (const bool sccValue : {true, false}) { const auto outcome = run({sCmpScc(sccValue), sCselectB64(4u, Literal, Zero), 0xffffffffu}, S4); const std::string label = std::string("wave64, literal all ones, zero") + (sccValue ? ", scc = 1" : ", scc = 0"); expectLanes(label.c_str(), outcome, sccValue ? below(32) : 0ull); expectConstantSources(label.c_str(), outcome, false); } for (const bool sccValue : {true, false}) { const auto outcome = run({sCmpScc(sccValue), sCselectB64(4u, Literal, Zero), 0xffffffffu}, S4, 32); const std::string label = std::string("wave32, literal all ones, zero") + (sccValue ? ", scc = 1" : ", scc = 0"); expectLanes(label.c_str(), outcome, sccValue ? below(32) : 0ull); expectConstantSources(label.c_str(), outcome, true); } for (const bool sccValue : {true, false}) { const auto outcome = run({vcmpLanesBelow(40), sCmpScc(sccValue), sCselectB64(Vcc, Vcc, Exec)}, Vcc); expectLanes(sccValue ? "destination vcc, scc = 1" : "destination vcc, scc = 0", outcome, sccValue ? below(40) : AllLanes); } } catch (const std::exception& error) { std::fprintf(stderr, "%s\n", error.what()); return 1; } return failures == 0 ? 0 : 1; }