diff --git a/src/graphics/shader/recompiler/ShaderRecompiler.cpp b/src/graphics/shader/recompiler/ShaderRecompiler.cpp index b81d886f..f6927ede 100644 --- a/src/graphics/shader/recompiler/ShaderRecompiler.cpp +++ b/src/graphics/shader/recompiler/ShaderRecompiler.cpp @@ -46,6 +46,7 @@ std::string MakeIrDump(const CFG::Graph& cfg, const IR::Program& ir) { std::string dump = "CFG:\n"; dump += CFG::GraphToString(cfg); dump += "\nIR:\n"; + dump += fmt::format("mode={}\n", ir.dispatcher_fallback ? "dispatcher" : "structured"); dump += ir.values != nullptr ? IR::ValueProgramToString(*ir.values) : IR::ProgramToString(ir); return dump; } diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h index da07a17b..5fb15bba 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h @@ -380,18 +380,18 @@ struct ValueEmitContext { const IR::MemoryInfo& Memory(const IR::Inst& inst) const; const IR::ExportInfo& Export(const IR::Inst& inst) const; uint32_t Label(const IR::Block* block) const; - size_t BlockIndex(const IR::Block* block) const; void Fail(const IR::Inst& inst, const char* reason); - EmitterState& state; - const IR::ValueProgram& program; - std::unordered_map definitions; - std::unordered_map labels; - const std::unordered_map* dispatcher_spills = nullptr; - const IR::Block* current_block = nullptr; - uint32_t scratch_u32_variable = 0; - bool failed = false; - std::string error; + EmitterState& state; + const IR::ValueProgram& program; + std::unordered_map definitions; + std::unordered_map labels; + const std::unordered_map* dispatcher_spills = nullptr; + std::unordered_map> dispatcher_block_loads; + const IR::Block* current_block = nullptr; + uint32_t scratch_u32_variable = 0; + bool failed = false; + std::string error; }; enum class VertexInputScalarKind { Float, Sint, Uint }; diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp index 0c72be99..54caad5b 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp @@ -36,24 +36,9 @@ void EmitKillIfPixelValidMaskInactive(EmitterState& state) { EmitKillIfBoolFalse(state, active); } -uint32_t SpillPointerType(EmitterState& state, IR::Type type) { - switch (type) { - case IR::Type::U1: return TypePointer(state, StorageClassFunction, TypeBool(state)); - case IR::Type::U8: - case IR::Type::U16: - case IR::Type::U32: - case IR::Type::F16: return TypePointer(state, StorageClassFunction, TypeU32(state)); - case IR::Type::F32: return TypePointer(state, StorageClassFunction, TypeF32(state)); - case IR::Type::U64: return TypePointer(state, StorageClassFunction, TypeU64(state)); - case IR::Type::U32x2: return TypePointer(state, StorageClassFunction, TypeU32Pair(state)); - case IR::Type::U32x3: - return TypePointer(state, StorageClassFunction, TypeU32Vector(state, 3)); - case IR::Type::U32x4: - return TypePointer(state, StorageClassFunction, TypeU32Vector(state, 4)); - case IR::Type::F32x2: - return TypePointer(state, StorageClassFunction, TypeF32Vector(state, 2)); - default: return 0; - } +uint32_t SpillPointerType(ValueEmitContext& ctx, IR::Type type) { + const auto value_type = ctx.TypeId(type); + return value_type == 0 ? 0 : TypePointer(ctx.state, StorageClassFunction, value_type); } struct DeferredPhiPatch { @@ -68,10 +53,8 @@ struct StructuredFunctionState { struct DispatcherFunctionState { std::unordered_map spills; - uint32_t pc_variable = 0; uint32_t header_label = 0; uint32_t select_label = 0; - uint32_t default_label = 0; uint32_t after_switch_label = 0; uint32_t continue_label = 0; uint32_t merge_label = 0; @@ -166,16 +149,13 @@ void EmitDispatcherTarget(ValueEmitContext& ctx, const DispatcherFunctionState& } } -void EmitDispatcherTerminator(ValueEmitContext& ctx, const DispatcherFunctionState& dispatcher, - const IR::Block* block, const IR::ValueBlockInfo& info, - uint32_t exit_label) { +uint32_t EmitDispatcherNextPc(ValueEmitContext& ctx, const DispatcherFunctionState& dispatcher, + const IR::Block* block, const IR::ValueBlockInfo& info) { const auto& term = info.terminator; switch (term.kind) { case CFG::TerminatorKind::Branch: EmitDispatcherTarget(ctx, dispatcher, block, term.true_block); - ctx.state.builder.AddFunction( - {OpStore, dispatcher.pc_variable, ConstantU32(ctx.state, term.true_block)}); - break; + return ConstantU32(ctx.state, term.true_block); case CFG::TerminatorKind::ConditionalBranch: { EmitDispatcherTarget(ctx, dispatcher, block, term.true_block); EmitDispatcherTarget(ctx, dispatcher, block, term.false_block); @@ -184,16 +164,12 @@ void EmitDispatcherTerminator(ValueEmitContext& ctx, const DispatcherFunctionSta ctx.Def(info.condition), ConstantU32(ctx.state, term.true_block), ConstantU32(ctx.state, term.false_block)}); - ctx.state.builder.AddFunction({OpStore, dispatcher.pc_variable, selected}); - break; + return selected; } case CFG::TerminatorKind::IndirectBranch: { for (const auto target: term.indirect_targets) { EmitDispatcherTarget(ctx, dispatcher, block, target); } - for (const auto target: term.indirect_selector_targets) { - EmitDispatcherTarget(ctx, dispatcher, block, target); - } uint32_t selected = ConstantU32(ctx.state, UINT32_MAX); if (!info.indirect_target.IsEmpty()) { const auto selector = ctx.Def(info.indirect_target); @@ -214,15 +190,10 @@ void EmitDispatcherTerminator(ValueEmitContext& ctx, const DispatcherFunctionSta selected = next; } } - ctx.state.builder.AddFunction({OpStore, dispatcher.pc_variable, selected}); - break; + return selected; } - default: - ctx.state.builder.AddFunction( - {OpStore, dispatcher.pc_variable, ConstantU32(ctx.state, UINT32_MAX)}); - break; + default: return ConstantU32(ctx.state, UINT32_MAX); } - ctx.state.builder.AddFunction({OpBranch, exit_label}); } bool EmitDirectInstruction(ValueEmitContext& ctx, const IR::Inst& inst) { @@ -346,12 +317,16 @@ void EmitDispatcherFunction(ValueEmitContext& ctx, const DispatcherFunctionState if (ctx.failed) { return; } - EmitDispatcherTerminator(ctx, dispatcher, entry, ctx.program.block_info.front(), - dispatcher.header_label); + const auto initial_pc = + EmitDispatcherNextPc(ctx, dispatcher, entry, ctx.program.block_info.front()); + const auto initial_parent = state.current_label; + state.builder.AddFunction({OpBranch, dispatcher.header_label}); EmitLabel(state, dispatcher.header_label); - const auto pc = state.builder.AllocateId(); - state.builder.AddFunction({OpLoad, TypeU32(state), pc, dispatcher.pc_variable}); + const auto pc = state.builder.AllocateId(); + const auto next_pc = state.builder.AllocateId(); + state.builder.AddFunction({OpPhi, TypeU32(state), pc, initial_pc, initial_parent, next_pc, + dispatcher.continue_label}); const auto done = state.builder.AllocateId(); state.builder.AddFunction( {OpIEqual, TypeBool(state), done, pc, ConstantU32(ctx.state, UINT32_MAX)}); @@ -363,16 +338,14 @@ void EmitDispatcherFunction(ValueEmitContext& ctx, const DispatcherFunctionState EmitLabel(state, dispatcher.select_label); state.builder.AddFunction( {OpSelectionMerge, dispatcher.after_switch_label, SelectionControlNone}); - std::vector words {OpSwitch, pc, dispatcher.default_label}; + std::vector words {OpSwitch, pc, dispatcher.after_switch_label}; for (size_t index = 1; index < ctx.program.blocks.size(); index++) { words.push_back(ctx.program.block_info[index].id); words.push_back(ctx.Label(ctx.program.blocks[index])); } state.builder.AddFunction(words); - EmitLabel(state, dispatcher.default_label); - state.builder.AddFunction( - {OpStore, dispatcher.pc_variable, ConstantU32(ctx.state, UINT32_MAX)}); - state.builder.AddFunction({OpBranch, dispatcher.after_switch_label}); + std::vector next_pc_words {OpPhi, TypeU32(state), next_pc, + ConstantU32(state, UINT32_MAX), dispatcher.select_label}; for (size_t index = 1; index < ctx.program.blocks.size() && !ctx.failed; index++) { EmitBlock(ctx, ctx.program.blocks[index], [&](const IR::Inst& inst) { @@ -381,13 +354,17 @@ void EmitDispatcherFunction(ValueEmitContext& ctx, const DispatcherFunctionState if (ctx.failed) { break; } - EmitDispatcherTerminator(ctx, dispatcher, ctx.program.blocks[index], - ctx.program.block_info[index], dispatcher.after_switch_label); + const auto selected = EmitDispatcherNextPc(ctx, dispatcher, ctx.program.blocks[index], + ctx.program.block_info[index]); + next_pc_words.push_back(selected); + next_pc_words.push_back(state.current_label); + state.builder.AddFunction({OpBranch, dispatcher.after_switch_label}); } if (ctx.failed) { return; } EmitLabel(state, dispatcher.after_switch_label); + state.builder.AddFunction(next_pc_words); state.builder.AddFunction({OpBranch, dispatcher.continue_label}); EmitLabel(state, dispatcher.continue_label); state.builder.AddFunction({OpBranch, dispatcher.header_label}); @@ -438,8 +415,14 @@ uint32_t ValueEmitContext::Def(IR::Value value) { if (dispatcher_spills != nullptr && current_block != nullptr && inst->Parent() != current_block) { if (const auto found = dispatcher_spills->find(inst); found != dispatcher_spills->end()) { + if (const auto loaded = dispatcher_block_loads.find(inst); + loaded != dispatcher_block_loads.end() && + loaded->second.first == state.current_label) { + return loaded->second.second; + } const auto id = state.builder.AllocateId(); state.builder.AddFunction({OpLoad, TypeId(inst->GetType()), id, found->second}); + dispatcher_block_loads.insert_or_assign(inst, std::pair {state.current_label, id}); return id; } } @@ -515,10 +498,6 @@ uint32_t ValueEmitContext::Label(const IR::Block* block) const { return labels.at(block); } -size_t ValueEmitContext::BlockIndex(const IR::Block* block) const { - return static_cast(std::ranges::find(program.blocks, block) - program.blocks.begin()); -} - void ValueEmitContext::Fail(const IR::Inst& inst, const char* reason) { failed = true; error = fmt::format("typed opcode {} {}", IR::ValueOpcodeName(inst.GetOpcode()), reason); @@ -534,14 +513,14 @@ bool EmitValueProgram(EmitterState& state, const IR::ValueProgram& program, std: for (const auto* block: program.blocks) { ctx.labels.emplace(block, state.builder.AllocateId()); } - if (program.dispatcher_fallback) { + if (state.program.dispatcher_fallback) { auto& dispatch = dispatcher.emplace(); for (const auto* block: program.blocks) { for (const auto& inst: *block) { if (inst.GetOpcode() != IR::ValueOpcode::Phi) { continue; } - if (SpillPointerType(state, inst.GetType()) == 0) { + if (SpillPointerType(ctx, inst.GetType()) == 0) { ctx.Fail(inst, "cannot be stored by the dispatcher"); break; } @@ -551,10 +530,11 @@ bool EmitValueProgram(EmitterState& state, const IR::ValueProgram& program, std: const auto mark_cross_block = [&](IR::Value value, const IR::Block* consumer) { value = value.Resolve(); const auto* definition = value.TryInstruction(); - if (definition == nullptr || definition->Parent() == consumer) { + if (definition == nullptr || definition->Parent() == consumer || + definition->Parent() == program.blocks.front()) { return; } - if (SpillPointerType(state, definition->GetType()) == 0) { + if (SpillPointerType(ctx, definition->GetType()) == 0) { ctx.Fail(*definition, "cannot be stored by the dispatcher"); return; } @@ -575,10 +555,8 @@ bool EmitValueProgram(EmitterState& state, const IR::ValueProgram& program, std: mark_cross_block(program.block_info[index].condition, program.blocks[index]); mark_cross_block(program.block_info[index].indirect_target, program.blocks[index]); } - dispatch.pc_variable = state.builder.AllocateId(); dispatch.header_label = state.builder.AllocateId(); dispatch.select_label = state.builder.AllocateId(); - dispatch.default_label = state.builder.AllocateId(); dispatch.after_switch_label = state.builder.AllocateId(); dispatch.continue_label = state.builder.AllocateId(); dispatch.merge_label = state.builder.AllocateId(); @@ -610,15 +588,12 @@ bool EmitValueProgram(EmitterState& state, const IR::ValueProgram& program, std: TypePointer(state, StorageClassFunction, TypeU32(state)), state.pixel_valid_mask_variable, StorageClassFunction}); } - if (program.dispatcher_fallback) { - state.builder.AddFunction({OpVariable, - TypePointer(state, StorageClassFunction, TypeU32(state)), - dispatcher->pc_variable, StorageClassFunction}); + if (state.program.dispatcher_fallback) { for (const auto* block: program.blocks) { for (const auto& inst: *block) { if (const auto found = dispatcher->spills.find(&inst); found != dispatcher->spills.end()) { - state.builder.AddFunction({OpVariable, SpillPointerType(state, inst.GetType()), + state.builder.AddFunction({OpVariable, SpillPointerType(ctx, inst.GetType()), found->second, StorageClassFunction}); } } @@ -641,7 +616,7 @@ bool EmitValueProgram(EmitterState& state, const IR::ValueProgram& program, std: EmitStorageBufferOffsets(state); if (program.blocks.empty()) { EmitReturn(ctx); - } else if (program.dispatcher_fallback) { + } else if (state.program.dispatcher_fallback) { EmitDispatcherFunction(ctx, *dispatcher); } else { EmitStructuredFunction(ctx); diff --git a/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.cpp b/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.cpp index fef6f37d..3a9f2fff 100644 --- a/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.cpp +++ b/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.cpp @@ -682,6 +682,66 @@ void RebuildPredecessors(Graph& graph) { } } +void PruneUnreachableBlocks(Graph& graph) { + if (graph.entry_block >= graph.blocks.size()) { + return; + } + + std::vector reachable(graph.blocks.size(), false); + std::vector pending {graph.entry_block}; + while (!pending.empty()) { + const auto block_id = pending.back(); + pending.pop_back(); + if (block_id >= graph.blocks.size() || reachable[block_id]) { + continue; + } + reachable[block_id] = true; + for (const auto successor: graph.blocks[block_id].successors) { + pending.push_back(successor); + } + } + if (std::ranges::all_of(reachable, [](bool value) { return value; })) { + return; + } + + std::vector id_map(graph.blocks.size(), UINT32_MAX); + std::vector blocks; + blocks.reserve(std::ranges::count(reachable, true)); + for (uint32_t old_id = 0; old_id < graph.blocks.size(); old_id++) { + if (!reachable[old_id]) { + continue; + } + id_map[old_id] = static_cast(blocks.size()); + blocks.push_back(std::move(graph.blocks[old_id])); + } + + graph.entry_block = RemapId(graph.entry_block, id_map); + graph.failure_block = RemapId(graph.failure_block, id_map); + graph.blocks = std::move(blocks); + for (auto& block: graph.blocks) { + block.id = RemapId(block.id, id_map); + RemapIds(block.successors, id_map); + block.predecessors.clear(); + block.dominators.clear(); + block.post_dominators.clear(); + auto& terminator = block.terminator; + terminator.true_block = RemapId(terminator.true_block, id_map); + terminator.false_block = RemapId(terminator.false_block, id_map); + terminator.merge_block = RemapId(terminator.merge_block, id_map); + terminator.continue_block = RemapId(terminator.continue_block, id_map); + for (auto& target: terminator.indirect_targets) { + target = RemapId(target, id_map); + } + for (auto& target: terminator.indirect_selector_targets) { + target = RemapId(target, id_map); + } + } + graph.back_edges.clear(); + graph.natural_loops.clear(); + graph.components.clear(); + RebuildPredecessors(graph); +} + void ComputeDominators(Graph& graph) { const auto count = static_cast(graph.blocks.size()); const auto all = AllBlockIds(count); @@ -1820,7 +1880,6 @@ bool BuildGraph(const Decoder::Program& program, Graph& graph, std::string* erro labels.insert(end_pc); std::map setpc_targets; - bool indirect_setpc = false; for (uint32_t i = 0; i < program.instructions.size(); i++) { const auto& inst = program.instructions[i]; const auto next_pc = InstructionEndPc(inst); @@ -1857,7 +1916,6 @@ bool BuildGraph(const Decoder::Program& program, Graph& graph, std::string* erro } labels.insert(target); } - indirect_setpc = indirect_setpc || target_info.indirect; setpc_targets.emplace(inst.pc, std::move(target_info)); if (next_pc <= end_pc) { labels.insert(next_pc); @@ -1935,9 +1993,6 @@ bool BuildGraph(const Decoder::Program& program, Graph& graph, std::string* erro block.terminator.indirect_selector_targets.push_back( pc_to_block.at(target_info.selector_target_pcs[i])); } - if (target_info.table_load_pc != UINT32_MAX) { - AddUnique(graph.code_table_load_pcs, target_info.table_load_pc); - } } else { block.terminator.kind = TerminatorKind::Branch; block.terminator.condition = BranchCondition::Always; @@ -1998,6 +2053,22 @@ bool BuildGraph(const Decoder::Program& program, Graph& graph, std::string* erro for (auto& block: graph.blocks) { SortUnique(block.predecessors); } + PruneUnreachableBlocks(graph); + graph.code_table_load_pcs.clear(); + bool indirect_setpc = false; + for (const auto& block: graph.blocks) { + if (block.terminator.kind != TerminatorKind::IndirectBranch) { + continue; + } + indirect_setpc = true; + if (block.inst_begin >= block.inst_end || block.inst_end > program.instructions.size()) { + continue; + } + const auto found = setpc_targets.find(program.instructions[block.inst_end - 1u].pc); + if (found != setpc_targets.end() && found->second.table_load_pc != UINT32_MAX) { + AddUnique(graph.code_table_load_pcs, found->second.table_load_pc); + } + } SortUnique(graph.code_table_load_pcs); ComputeDominators(graph); diff --git a/src/graphics/shader/recompiler/frontend/translate/Translate.cpp b/src/graphics/shader/recompiler/frontend/translate/Translate.cpp index 3804db4e..abcaf183 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Translate.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Translate.cpp @@ -758,7 +758,6 @@ bool TranslateProgram(const IR::Program& source, IR::ValueProgram& result, const ShaderPixelInputInfo* pixel_input_info, const ShaderComputeInputInfo* compute_input_info, std::string* error) { result = {}; - result.dispatcher_fallback = source.dispatcher_fallback; const uint32_t wave_size = source.wave_size; const bool per_invocation_masks = source.lane_mask_mode == ShaderLaneMaskMode::PerInvocation; uint32_t vector_limit = 1u; diff --git a/src/graphics/shader/recompiler/ir/ShaderIR.cpp b/src/graphics/shader/recompiler/ir/ShaderIR.cpp index abe1c1ba..3802c5af 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIR.cpp +++ b/src/graphics/shader/recompiler/ir/ShaderIR.cpp @@ -1181,12 +1181,11 @@ bool LowerProgram(const Decoder::Program& decoded, const CFG::Graph& cfg, Shader EXIT("invalid shader wave size: %u\n", wave_size); } - program = {}; - program.stage = stage; - program.wave_size = wave_size; - program.dispatcher_fallback = cfg.irreducible; - program.cfg_failure_kind = cfg.failure_kind; - program.fallback_reason = cfg.unsupported_reason; + program = {}; + program.stage = stage; + program.wave_size = wave_size; + program.cfg_failure_kind = cfg.failure_kind; + program.fallback_reason = cfg.unsupported_reason; program.blocks.clear(); if (cfg.blocks.empty()) { diff --git a/src/graphics/shader/recompiler/ir/ValueProgram.cpp b/src/graphics/shader/recompiler/ir/ValueProgram.cpp index 0509b4b8..f5df0433 100644 --- a/src/graphics/shader/recompiler/ir/ValueProgram.cpp +++ b/src/graphics/shader/recompiler/ir/ValueProgram.cpp @@ -142,6 +142,7 @@ bool ValidateValueProgram(const ValueProgram& program, bool require_ssa, std::st return Fail(error, "value IR block storage is inconsistent"); } std::unordered_set blocks; + std::unordered_map block_indices; std::unordered_map blocks_by_id; std::unordered_set instructions; for (size_t block_index = 0; block_index < program.blocks.size(); block_index++) { @@ -153,6 +154,10 @@ bool ValidateValueProgram(const ValueProgram& program, bool require_ssa, std::st if (!blocks.insert(block).second) { return Fail(error, "value IR block pointer is duplicated"); } + block_indices.emplace(block, block_index); + if (program.block_info[block_index].id == UINT32_MAX) { + return Fail(error, "value IR block uses the reserved exit id"); + } if (!blocks_by_id.emplace(program.block_info[block_index].id, block).second) { return Fail(error, "value IR block id is duplicated"); } @@ -162,6 +167,9 @@ bool ValidateValueProgram(const ValueProgram& program, bool require_ssa, std::st } } } + if (!program.blocks.empty() && !program.blocks.front()->ImmPredecessors().empty()) { + return Fail(error, "value IR entry block has a predecessor"); + } for (size_t block_index = 0; block_index < program.blocks.size(); block_index++) { const auto* block = program.blocks[block_index]; @@ -224,12 +232,16 @@ bool ValidateValueProgram(const ValueProgram& program, bool require_ssa, std::st return Fail(error, "value IR conditional branch condition is invalid"); } break; - case CFG::TerminatorKind::IndirectBranch: + case CFG::TerminatorKind::IndirectBranch: { if (!validate_control_value(program.block_info[block_index].indirect_target, Type::U32)) { return Fail(error, "value IR indirect branch selector is invalid"); } + std::unordered_set indirect_targets; for (const auto target: terminator.indirect_targets) { + if (!indirect_targets.insert(target).second) { + return Fail(error, "value IR indirect branch target is duplicated"); + } if (!add_target(target)) { return Fail(error, "value IR indirect branch target is missing"); } @@ -247,6 +259,7 @@ bool ValidateValueProgram(const ValueProgram& program, bool require_ssa, std::st } } break; + } case CFG::TerminatorKind::Return: case CFG::TerminatorKind::Unsupported: break; } @@ -440,6 +453,100 @@ bool ValidateValueProgram(const ValueProgram& program, bool require_ssa, std::st } } } + + if (program.blocks.empty()) { + return true; + } + + std::vector reachable(program.blocks.size(), false); + std::vector pending {0u}; + while (!pending.empty()) { + const auto block_index = pending.back(); + pending.pop_back(); + if (reachable[block_index]) { + continue; + } + reachable[block_index] = true; + for (const auto* successor: program.blocks[block_index]->ImmSuccessors()) { + pending.push_back(block_indices.at(successor)); + } + } + if (!std::ranges::all_of(reachable, [](bool value) { return value; })) { + return Fail(error, "value IR contains an unreachable block"); + } + + std::vector> dominators(program.blocks.size(), + std::vector(program.blocks.size(), true)); + dominators.front().assign(program.blocks.size(), false); + dominators.front().front() = true; + bool changed = true; + while (changed) { + changed = false; + for (size_t block_index = 1; block_index < program.blocks.size(); block_index++) { + std::vector next(program.blocks.size(), true); + for (const auto* predecessor: program.blocks[block_index]->ImmPredecessors()) { + const auto predecessor_index = block_indices.at(predecessor); + for (size_t candidate = 0; candidate < next.size(); candidate++) { + next[candidate] = next[candidate] && dominators[predecessor_index][candidate]; + } + } + next[block_index] = true; + if (next != dominators[block_index]) { + dominators[block_index] = std::move(next); + changed = true; + } + } + } + + std::unordered_map instruction_positions; + for (const auto* block: program.blocks) { + size_t position = 0; + for (const auto& inst: *block) { + instruction_positions.emplace(&inst, position++); + } + } + const auto dominates = [&](const Block* definition, const Block* use) { + return dominators[block_indices.at(use)][block_indices.at(definition)]; + }; + const auto control_dominates = [&](Value value, const Block* use) { + const auto* definition = value.TryInstruction(); + return definition == nullptr || definition->Parent() == use || + dominates(definition->Parent(), use); + }; + + for (size_t block_index = 0; block_index < program.blocks.size(); block_index++) { + const auto* block = program.blocks[block_index]; + for (const auto& inst: *block) { + for (size_t arg_index = 0; arg_index < inst.NumArgs(); arg_index++) { + const auto* definition = inst.Arg(arg_index).TryInstruction(); + if (definition == nullptr) { + continue; + } + if (inst.GetOpcode() == ValueOpcode::Phi) { + const auto* predecessor = inst.PhiBlock(arg_index); + if (definition->Parent() != predecessor && + !dominates(definition->Parent(), predecessor)) { + return Fail(error, + "value IR Phi incoming definition does not dominate its edge"); + } + } else if (definition->Parent() == block) { + if (instruction_positions.at(definition) >= instruction_positions.at(&inst)) { + return Fail(error, + "value IR instruction uses a same-block definition before it"); + } + } else if (!dominates(definition->Parent(), block)) { + return Fail(error, "value IR instruction definition does not dominate its use"); + } + } + } + const auto& info = program.block_info[block_index]; + if (!info.condition.IsEmpty() && !control_dominates(info.condition, block)) { + return Fail(error, "value IR branch condition definition does not dominate its use"); + } + if (!info.indirect_target.IsEmpty() && !control_dominates(info.indirect_target, block)) { + return Fail(error, "value IR indirect selector definition does not dominate its use"); + } + } return true; } @@ -472,8 +579,7 @@ std::string ValueProgramToString(const ValueProgram& program) { } }; - std::string text = - fmt::format("mode={}\n", program.dispatcher_fallback ? "dispatcher" : "structured"); + std::string text; for (size_t block_index = 0; block_index < program.blocks.size(); block_index++) { text += fmt::format("Block ${} pc=0x{:08x}..0x{:08x}\n", block_index, program.block_info[block_index].start_pc, diff --git a/src/graphics/shader/recompiler/ir/ValueProgram.h b/src/graphics/shader/recompiler/ir/ValueProgram.h index fbfb7355..0cf58059 100644 --- a/src/graphics/shader/recompiler/ir/ValueProgram.h +++ b/src/graphics/shader/recompiler/ir/ValueProgram.h @@ -65,7 +65,6 @@ struct ValueProgram { std::vector descriptor_sources; std::vector srt_reads; std::vector dynamic_reads; - bool dispatcher_fallback = false; }; bool ValidateValueProgram(const ValueProgram& program, bool require_ssa, std::string* error); diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index 4c8f29fd..d4e5dfa0 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -7854,7 +7854,7 @@ void TestNewShaderRecompilerCfgNestedEarlyExitSharedTerminal() { CheckSpirvBinaryValidates(result.spirv); } -void TestNewShaderRecompilerCfgExternallyEnteredSelectionDispatcher() { +void TestNewShaderRecompilerCfgPrunesUnreachableSelectionEntry() { const uint32_t shader[] = { EncodeSopp(0x02, 1), // entry -> header, skipping unreachable entry EncodeSopp(0x02, 2), // unreachable entry -> shared selection arm @@ -7874,24 +7874,22 @@ void TestNewShaderRecompilerCfgExternallyEnteredSelectionDispatcher() { error.c_str()); const auto original_coverage = CfgInstructionCoverage(graph, decoded.instructions.size()); - Check(!ShaderRecompiler::CFG::Structurize(graph, &error) && - Common::ContainsStr(error, "externally entered region block"), - "improper externally-entered selection was accepted as structured"); + Check(original_coverage[1] == 0u, + "CFG retained an unreachable external selection entry"); + Check(ShaderRecompiler::CFG::Structurize(graph, &error), error.c_str()); Check(CfgInstructionCoverage(graph, decoded.instructions.size()) == original_coverage, - "externally-entered selection handling duplicated semantic code"); + "selection structurization changed reachable semantic code"); auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; ShaderRecompiler::CompileResult result; Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(result.program.dispatcher_fallback && - Common::ContainsStr(result.ir_dump, - "externally entered region block"), - "improper externally-entered selection lost its dispatcher reason"); - Check(SpirvInstructionOpcodeCount(result.spirv, 251) != 0u, - "externally-entered selection dispatcher lacks OpSwitch"); + Check(!result.program.dispatcher_fallback && + Common::ContainsStr(result.ir_dump, "mode=structured") && + SpirvInstructionOpcodeCount(result.spirv, 251) == 0u, + "unreachable selection entry still forced dispatcher mode"); CheckSpirvBinaryValidates(result.spirv); } @@ -7937,16 +7935,38 @@ void TestNewShaderRecompilerDispatcherSpillsU32x3() { std::string error; Check(TryRecompile(shader, options, result, &error), error.c_str()); Check(result.program.dispatcher_fallback && - result.program.values->blocks.size() >= 2u, + result.program.values->blocks.size() >= 3u, "U32x3 spill fixture did not select dispatcher mode"); + const auto before = MeasureSpirv(result.spirv); + { + auto prologue_program = result.program; + IR::IREmitter definition(prologue_program.values->blocks[0]); + const auto vector = + definition.Emit(IR::ValueOpcode::CompositeConstructU32x3, + {IR::Value(1u), IR::Value(2u), IR::Value(3u)}); + IR::IREmitter use(prologue_program.values->blocks[1]); + use.Emit(IR::ValueOpcode::CompositeExtractU32x3, {vector, IR::Value(2u)}); + Check(Spirv::AnalyzeProgramRequirements(prologue_program, &error), + error.c_str()); + std::vector spirv; + Check(Spirv::EmitProgram(prologue_program, result.resources, + options.input_info, spirv, &error), + error.c_str()); + CheckSpirvBinaryValidates(spirv); + const auto after = MeasureSpirv(spirv); + Check(after.function_variables == before.function_variables && + after.loads == before.loads && after.stores == before.stores, + "dispatcher spilled a value defined by its dominating prologue"); + } auto program = result.program; - IR::IREmitter definition(program.values->blocks[0]); + IR::IREmitter definition(program.values->blocks[1]); const auto vector = definition.Emit(IR::ValueOpcode::CompositeConstructU32x3, {IR::Value(1u), IR::Value(2u), IR::Value(3u)}); - IR::IREmitter use(program.values->blocks[1]); + IR::IREmitter use(program.values->blocks[2]); use.Emit(IR::ValueOpcode::CompositeExtractU32x3, {vector, IR::Value(2u)}); + use.Emit(IR::ValueOpcode::CompositeExtractU32x3, {vector, IR::Value(1u)}); Check(Spirv::AnalyzeProgramRequirements(program, &error), error.c_str()); std::vector spirv; @@ -7954,7 +7974,6 @@ void TestNewShaderRecompilerDispatcherSpillsU32x3() { &error), error.c_str()); CheckSpirvBinaryValidates(spirv); - const auto before = MeasureSpirv(result.spirv); const auto after = MeasureSpirv(spirv); Check(after.function_variables == before.function_variables + 1u && after.loads == before.loads + 1u && @@ -8025,13 +8044,13 @@ void TestNewShaderRecompilerU64PairLowering() { Check(TryRecompile(dispatcher_shader, options, result, &error), error.c_str()); Check(result.program.dispatcher_fallback && - result.program.values->blocks.size() >= 2u, + result.program.values->blocks.size() >= 3u, "U64 spill fixture did not select dispatcher mode"); program = result.program; - IR::IREmitter definition(program.values->blocks[0]); + IR::IREmitter definition(program.values->blocks[1]); const auto vector = definition.Emit(IR::ValueOpcode::CompositeConstructU64, {IR::Value(1u), IR::Value(2u)}); - IR::IREmitter use(program.values->blocks[1]); + IR::IREmitter use(program.values->blocks[2]); use.Emit(IR::ValueOpcode::CompositeExtractU64, {vector, IR::Value(1u)}); Check(Spirv::AnalyzeProgramRequirements(program, &error), error.c_str()); spirv.clear(); @@ -8262,8 +8281,62 @@ void TestNewShaderRecompilerSetpcJumpTable() { CheckSpirvBinaryValidates(result.spirv); } +void TestNewShaderRecompilerPrunesUnreachableSetpcMetadata() { + const uint32_t shader[] = { + EncodeSop2(0x07, 0, 0, 129), // s_min_u32 s0, s0, 1 + EncodeSop2(0x1e, 0, 0, 131), // s_lshl_b32 s0, s0, 3 + EncodeSop1(0x1f, 8, 0), // s_getpc_b64 s[8:9] + EncodeSop2(0x00, 8, 8, 255), // s_add_u32 s8, s8, literal + 0x0000003cu, + EncodeSop2(0x04, 9, 9, 128), // s_addc_u32 s9, s9, 0 + EncodeSmem0(0x01, 4, 4), + 0u, // reachable s_load_dwordx2 s[4:5] + EncodeSopp(0x02, 5), // skip the otherwise valid jump-table dispatch + EncodeSopp(0x0c, 0), // unreachable s_waitcnt 0 + EncodeSop1(0x1f, 10, 0), + EncodeSop2(0x00, 10, 10, 4), + EncodeSop2(0x04, 11, 11, 5), + EncodeSop1(0x20, 0, 10), // unreachable s_setpc_b64 s[10:11] + EncodeSMovB32(1, 4), // reachable use of the loaded s4 + EncodeSopp(0x02, 1), + EncodeSMovB32(2, 129), + EncodeSopp(0x01, 0), + 0x0000000cu, + 0x00000000u, + 0x00000014u, + 0x00000000u, + }; + + ShaderRecompiler::Decoder::Program decoded; + std::string error; + Check(ShaderRecompiler::Decoder::DecodeProgram(std::span{shader}, decoded, + &error), + error.c_str()); + ShaderRecompiler::CFG::Graph graph; + Check(ShaderRecompiler::CFG::BuildGraph(decoded, graph, &error), + error.c_str()); + Check(!graph.irreducible && graph.code_table_load_pcs.empty(), + "unreachable S_SETPC retained jump-table metadata or dispatcher mode"); + + ShaderRecompiler::IR::Program ir; + Check(ShaderRecompiler::IR::LowerProgram(decoded, graph, ShaderType::Compute, + 64u, ir, &error), + error.c_str()); + const auto scalar_loads = + std::ranges::count_if(ir.blocks, [](const auto &block) { + return std::ranges::any_of(block.instructions, [](const auto &inst) { + return inst.op == ShaderRecompiler::IR::Opcode::SLoadDword && + inst.pc == 0x18u; + }); + }); + Check(scalar_loads == 1u, + "unreachable S_SETPC metadata suppressed a reachable scalar load"); +} + void TestNewShaderRecompilerSetpcDwordJumpTable() { const uint32_t shader[] = { + EncodeSopp(0x02, 1), // skip one unreachable block + EncodeSMovB32(100, 129), // unreachable EncodeSop2(0x07, 106, 0, 130), // s_min_u32 vcc_lo, s0, 2 EncodeSop2(0x1e, 106, 106, 130), // s_lshl_b32 vcc_lo, vcc_lo, 2 EncodeSop1(0x1f, 4, 0), // s_getpc_b64 s[4:5] @@ -8311,13 +8384,16 @@ void TestNewShaderRecompilerSetpcDwordJumpTable() { Check(jump != result.program.values->block_info.end() && jump->terminator.indirect_targets.size() == 2 && jump->terminator.indirect_target_pcs == - std::vector({0x30u, 0x38u}) && + std::vector({0x38u, 0x40u}) && jump->terminator.indirect_selector_values == std::vector({0u, 4u, 8u}) && jump->terminator.indirect_selector_targets.size() == 3 && - block_pc(jump->terminator.indirect_selector_targets[0]) == 0x30u && - block_pc(jump->terminator.indirect_selector_targets[1]) == 0x38u && - block_pc(jump->terminator.indirect_selector_targets[2]) == 0x30u, + block_pc(jump->terminator.indirect_selector_targets[0]) == 0x38u && + block_pc(jump->terminator.indirect_selector_targets[1]) == 0x40u && + block_pc(jump->terminator.indirect_selector_targets[2]) == 0x38u && + std::ranges::none_of( + result.program.values->block_info, + [](const auto &info) { return info.start_pc == 0x04u; }), "subtractive S_SETPC_B64 table targets or selector mapping changed"); Check(!Common::ContainsStr(result.ir_dump, "SLoadDword"), "subtractive S_SETPC_B64 table load reached normal IR"); @@ -8689,6 +8765,210 @@ void TestValuePhiValidation() { }; validate_cfg(true); validate_cfg(false); + { + ValueProgram program; + for (uint32_t id = 0; id < 2; id++) { + program.block_storage.push_back(std::make_unique()); + program.blocks.push_back(program.block_storage.back().get()); + program.block_info.push_back({.id = id}); + } + program.block_info[0].terminator.kind = + ShaderRecompiler::CFG::TerminatorKind::Branch; + program.block_info[0].terminator.true_block = 1; + program.block_info[1].terminator.kind = + ShaderRecompiler::CFG::TerminatorKind::Branch; + program.block_info[1].terminator.true_block = 0; + program.blocks[0]->AddBranch(program.blocks[1]); + program.blocks[1]->AddBranch(program.blocks[0]); + std::string error; + Check(!ValidateValueProgram(program, true, &error) && + Common::ContainsStr(error, "entry block has a predecessor"), + "value IR accepted a backedge to its synthetic entry block"); + } + { + ValueProgram program; + for (uint32_t id = 0; id < 3; id++) { + program.block_storage.push_back(std::make_unique()); + program.blocks.push_back(program.block_storage.back().get()); + program.block_info.push_back({.id = id}); + } + program.block_info[0].terminator.kind = + ShaderRecompiler::CFG::TerminatorKind::Branch; + program.block_info[0].terminator.true_block = 1; + auto &indirect = program.block_info[1]; + indirect.terminator.kind = + ShaderRecompiler::CFG::TerminatorKind::IndirectBranch; + indirect.terminator.indirect_targets = {1}; + indirect.terminator.indirect_selector_values = {0}; + indirect.terminator.indirect_selector_targets = {2}; + indirect.indirect_target = Value(0u); + program.blocks[0]->AddBranch(program.blocks[1]); + program.blocks[1]->AddBranch(program.blocks[1]); + std::string error; + Check(!ValidateValueProgram(program, true, &error) && + Common::ContainsStr(error, + "selector target is not a CFG successor"), + "value IR accepted an indirect selector target outside its target " + "set"); + } + { + ValueProgram program; + program.block_storage.push_back(std::make_unique()); + program.blocks.push_back(program.block_storage.back().get()); + program.block_info.push_back({.id = UINT32_MAX}); + std::string error; + Check(!ValidateValueProgram(program, true, &error) && + Common::ContainsStr(error, "reserved exit id"), + "value IR accepted the dispatcher exit sentinel as a block id"); + } + { + ValueProgram program; + for (uint32_t id = 0; id < 2; id++) { + program.block_storage.push_back(std::make_unique()); + program.blocks.push_back(program.block_storage.back().get()); + program.block_info.push_back({.id = id}); + } + program.block_info[0].terminator.kind = + ShaderRecompiler::CFG::TerminatorKind::Branch; + program.block_info[0].terminator.true_block = 1; + auto &indirect = program.block_info[1]; + indirect.terminator.kind = + ShaderRecompiler::CFG::TerminatorKind::IndirectBranch; + indirect.terminator.indirect_targets = {1, 1}; + indirect.indirect_target = Value(0u); + program.blocks[0]->AddBranch(program.blocks[1]); + program.blocks[1]->AddBranch(program.blocks[1]); + std::string error; + Check(!ValidateValueProgram(program, true, &error) && + Common::ContainsStr(error, "target is duplicated"), + "value IR accepted duplicate indirect branch targets"); + } + { + ValueProgram program; + for (uint32_t id = 0; id < 2; id++) { + program.block_storage.push_back(std::make_unique()); + program.blocks.push_back(program.block_storage.back().get()); + program.block_info.push_back({.id = id}); + } + std::string error; + Check(!ValidateValueProgram(program, true, &error) && + Common::ContainsStr(error, "unreachable block"), + "value IR accepted an unreachable block"); + } + { + ValueProgram program; + program.block_storage.push_back(std::make_unique()); + auto *block = program.block_storage.back().get(); + program.blocks.push_back(block); + program.block_info.push_back({.id = 0}); + auto &use = + block->AppendNewInst(ValueOpcode::IAdd32, {Value(1u), Value(2u)}); + auto &definition = + block->AppendNewInst(ValueOpcode::IAdd32, {Value(3u), Value(4u)}); + use.SetArg(0, Value(&definition)); + std::string error; + Check(!ValidateValueProgram(program, true, &error) && + Common::ContainsStr(error, "same-block definition"), + "value IR accepted a same-block forward use"); + } + { + ValueProgram program; + for (uint32_t id = 0; id < 4; id++) { + program.block_storage.push_back(std::make_unique()); + program.blocks.push_back(program.block_storage.back().get()); + program.block_info.push_back({.id = id}); + } + program.block_info[0].terminator.kind = + ShaderRecompiler::CFG::TerminatorKind::ConditionalBranch; + program.block_info[0].terminator.true_block = 1; + program.block_info[0].terminator.false_block = 2; + program.block_info[0].condition = Value(true); + for (uint32_t id : {1u, 2u}) { + program.block_info[id].terminator.kind = + ShaderRecompiler::CFG::TerminatorKind::Branch; + program.block_info[id].terminator.true_block = 3; + } + program.blocks[0]->AddBranch(program.blocks[1]); + program.blocks[0]->AddBranch(program.blocks[2]); + program.blocks[1]->AddBranch(program.blocks[3]); + program.blocks[2]->AddBranch(program.blocks[3]); + auto &definition = program.blocks[1]->AppendNewInst(ValueOpcode::IAdd32, + {Value(1u), Value(2u)}); + program.blocks[2]->AppendNewInst(ValueOpcode::IAdd32, + {Value(&definition), Value(3u)}); + std::string error; + Check(!ValidateValueProgram(program, true, &error) && + Common::ContainsStr(error, "does not dominate its use"), + "value IR accepted a non-dominating ordinary definition"); + } + { + ValueProgram program; + for (uint32_t id = 0; id < 4; id++) { + program.block_storage.push_back(std::make_unique()); + program.blocks.push_back(program.block_storage.back().get()); + program.block_info.push_back({.id = id}); + } + program.block_info[0].terminator.kind = + ShaderRecompiler::CFG::TerminatorKind::ConditionalBranch; + program.block_info[0].terminator.true_block = 1; + program.block_info[0].terminator.false_block = 2; + program.block_info[0].condition = Value(true); + for (uint32_t id : {1u, 2u}) { + program.block_info[id].terminator.kind = + ShaderRecompiler::CFG::TerminatorKind::Branch; + program.block_info[id].terminator.true_block = 3; + } + program.blocks[0]->AddBranch(program.blocks[1]); + program.blocks[0]->AddBranch(program.blocks[2]); + program.blocks[1]->AddBranch(program.blocks[3]); + program.blocks[2]->AddBranch(program.blocks[3]); + auto &definition = program.blocks[1]->AppendNewInst(ValueOpcode::IAdd32, + {Value(1u), Value(2u)}); + auto &phi = program.blocks[3]->AppendNewInst(ValueOpcode::Phi); + phi.SetFlags(Type::U32); + phi.AddPhiOperand(program.blocks[1], Value(3u)); + phi.AddPhiOperand(program.blocks[2], Value(&definition)); + std::string error; + Check(!ValidateValueProgram(program, true, &error) && + Common::ContainsStr(error, "does not dominate its edge"), + "value IR accepted an unavailable Phi-edge definition"); + } + { + ValueProgram program; + for (uint32_t id = 0; id < 5; id++) { + program.block_storage.push_back(std::make_unique()); + program.blocks.push_back(program.block_storage.back().get()); + program.block_info.push_back({.id = id}); + } + program.block_info[0].terminator.kind = + ShaderRecompiler::CFG::TerminatorKind::ConditionalBranch; + program.block_info[0].terminator.true_block = 1; + program.block_info[0].terminator.false_block = 2; + program.block_info[0].condition = Value(true); + program.block_info[1].terminator.kind = + ShaderRecompiler::CFG::TerminatorKind::Branch; + program.block_info[1].terminator.true_block = 3; + program.block_info[2].terminator.kind = + ShaderRecompiler::CFG::TerminatorKind::ConditionalBranch; + program.block_info[2].terminator.true_block = 3; + program.block_info[2].terminator.false_block = 4; + program.block_info[3].terminator.kind = + ShaderRecompiler::CFG::TerminatorKind::Branch; + program.block_info[3].terminator.true_block = 4; + program.blocks[0]->AddBranch(program.blocks[1]); + program.blocks[0]->AddBranch(program.blocks[2]); + program.blocks[1]->AddBranch(program.blocks[3]); + program.blocks[2]->AddBranch(program.blocks[3]); + program.blocks[2]->AddBranch(program.blocks[4]); + program.blocks[3]->AddBranch(program.blocks[4]); + auto &condition = program.blocks[1]->AppendNewInst(ValueOpcode::IEqual32, + {Value(1u), Value(2u)}); + program.block_info[2].condition = Value(&condition); + std::string error; + Check(!ValidateValueProgram(program, true, &error) && + Common::ContainsStr(error, "branch condition definition"), + "value IR accepted a non-dominating branch condition"); + } } void TestWqmConstantPropagation() { @@ -10996,23 +11276,25 @@ void TestNewShaderRecompilerSpirvSizeBaselines() { EncodeSopp(0x01), }; const auto dispatcher_result = compile("dispatcher", dispatcher, - {.words = 258, - .instructions = 78, - .variables = 4, - .function_variables = 4, - .loads = 4, - .stores = 14, - .labels = 14, + {.words = 242, + .instructions = 67, + .variables = 3, + .function_variables = 3, + .loads = 3, + .stores = 6, + .phis = 2, + .labels = 13, .loop_merges = 1, .selection_merges = 1, - .branches = 11, + .branches = 10, .conditional_branches = 1, .switches = 1}); Check(dispatcher_result.program.dispatcher_fallback && Common::ContainsStr(dispatcher_result.ir_dump, "Phi") && - SpirvInstructionOpcodeCount(dispatcher_result.spirv, 245u) == 0u && + SpirvInstructionOpcodeCount(dispatcher_result.spirv, 245u) == 2u && SpirvInstructionOpcodeCount(dispatcher_result.spirv, 251u) == 1u, - "dispatcher size fixture no longer exercises whole-function dispatch"); + "dispatcher size fixture lost its two control Phis or switch"); + CheckSpirvPhiParents(dispatcher_result.spirv); } } // namespace @@ -11078,7 +11360,7 @@ int main() { TestNewShaderRecompilerCfgLoopSharedRegion(); TestNewShaderRecompilerCfgOverlappingEarlyExitLadder(); TestNewShaderRecompilerCfgNestedEarlyExitSharedTerminal(); - TestNewShaderRecompilerCfgExternallyEnteredSelectionDispatcher(); + TestNewShaderRecompilerCfgPrunesUnreachableSelectionEntry(); TestNewShaderRecompilerCfgIrreducibleDispatcher(); TestNewShaderRecompilerDispatcherSpillsU32x3(); TestNewShaderRecompilerU64PairLowering(); @@ -11089,6 +11371,7 @@ int main() { TestNewShaderRecompilerBranchConditionForms(); TestNewShaderRecompilerSetpcBranch(); TestNewShaderRecompilerSetpcJumpTable(); + TestNewShaderRecompilerPrunesUnreachableSetpcMetadata(); TestNewShaderRecompilerSetpcDwordJumpTable(); TestTypedEntryStateIsMinimal(); TestWqmConstantPropagation();