diff --git a/src/graphics/shader/recompiler/ir/ShaderIR.h b/src/graphics/shader/recompiler/ir/ShaderIR.h index 60996df8..5e5dc8ee 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIR.h +++ b/src/graphics/shader/recompiler/ir/ShaderIR.h @@ -19,6 +19,7 @@ #include #include #include +#include #include namespace Xbyak { @@ -613,6 +614,7 @@ struct ResourcePlan { std::vector memory_info; std::vector descriptor_sources; std::vector control_flow; + std::unordered_map> control_phi_predecessors; std::vector srt_reads; bool requires_specialization_memory = false; std::vector> source_reads; @@ -628,6 +630,7 @@ struct ResourcePlan { mutable uint32_t evaluation_depth = 0; mutable std::vector active_sources; mutable std::vector visited_blocks; + mutable std::vector reachable_blocks; mutable std::vector pending_blocks; mutable std::vector material_keys; }; diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp index 82303d21..653ac00b 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp @@ -1122,49 +1122,79 @@ ResourcePlan ExtractResourcePlan(const Program& program) { for (const auto& read: program.srt_reads) { plan.srt_reads.push_back({Clone(read.value), read.flat_offset}); } - // A proven uniform factor can decide a branch even when its other lanes are unknown. - // Keep only that Boolean structure, never the varying shader dependency graph. - Value unknown; + // Retain integer control expressions with opaque varying leaves. Their identities + // matter when an EXEC factor also guards a selected value; never clone float work. bool capture_control_reads = false; - std::unordered_map predicates; + bool uniform_control = false; + const auto control_value_count = plan.value_storage.size(); + struct ControlValue { + Value value; + bool complete = false; + }; + std::unordered_map predicates; std::function ClonePredicate = [&](Value value) -> Value { value = value.Resolve(); if (value.IsEmpty()) return {}; const auto* inst = value.TryInstruction(); - if (inst == nullptr) + if (inst == nullptr) { + uniform_control |= value.GetType() == Type::U1; return ValidateRuntimeValue(program, value, RuntimeValueType::Integer) ? value : Value {}; - // Preserve shared EXEC predicates, including subgraphs with no uniform factor. + } const auto [found, inserted] = predicates.try_emplace(inst); - if (!inserted) return found->second; - auto& result = found->second; + auto& entry = found->second; + auto& result = entry.value; + const auto opaque = [&]() -> Value { + if (value.GetType() != Type::U1 && value.GetType() != Type::U32) return {}; + return result = Value(&plan.value_storage.emplace_back( + value.GetType() == Type::U1 ? ValueOpcode::UndefU1 : ValueOpcode::UndefU32)); + }; + // A recursive projection remains opaque, including unstructured loop Phis. + if (!inserted) return entry.complete || !result.IsEmpty() ? result : opaque(); + const auto finish = [&](Value projected) { + entry.complete = true; + return result = projected; + }; if (ValidateRuntimeValue(program, value, RuntimeValueType::Integer)) { + uniform_control = true; capture_control_reads |= !ValidateRuntimeValue(program, value, RuntimeValueType::ImmutableInteger); - return result = Clone(value); + return finish(Clone(value)); } const auto op = inst->GetOpcode(); - if (op == ValueOpcode::ConditionRef || op == ValueOpcode::LogicalNot) { - const auto operand = ClonePredicate(inst->Arg(0)); - if (operand.IsEmpty() || op == ValueOpcode::ConditionRef) return result = operand; - auto& node = plan.value_storage.emplace_back(op); - node.SetArg(0, operand); - return result = Value(&node); - } - if (op != ValueOpcode::LogicalAnd && op != ValueOpcode::LogicalOr) return {}; - auto left = ClonePredicate(inst->Arg(0)); - auto right = ClonePredicate(inst->Arg(1)); - if (left.IsEmpty() && right.IsEmpty()) return {}; - if (left.IsEmpty() || right.IsEmpty()) { - if (unknown.IsEmpty()) unknown = Value(&plan.value_storage.emplace_back(ValueOpcode::UndefU1)); - if (left.IsEmpty()) left = unknown; - if (right.IsEmpty()) right = unknown; + if (op == ValueOpcode::ConditionRef) return finish(ClonePredicate(inst->Arg(0))); + if (op == ValueOpcode::Phi) { + if (inst->Parent() == nullptr || inst->Parent()->terminator.loop_header || + inst->NumPhiBlocks() != inst->NumArgs() || inst->NumArgs() == 0u || + (value.GetType() != Type::U1 && value.GetType() != Type::U32)) return finish(opaque()); + std::vector predecessors; + for (size_t index = 0; index < inst->NumArgs(); ++index) { + const auto predecessor = std::ranges::find(program.blocks, inst->PhiBlock(index)); + if (predecessor == program.blocks.end()) return finish(opaque()); + predecessors.push_back(static_cast(predecessor - program.blocks.begin())); + } + auto& node = plan.value_storage.emplace_back(op, inst->Flags()); + plan.control_phi_predecessors.emplace(&node, std::move(predecessors)); + for (size_t index = 0; index < inst->NumArgs(); ++index) + node.AddPhiOperand(nullptr, ClonePredicate(inst->Arg(index))); + return finish(Value(&node)); } + if (op != ValueOpcode::LogicalAnd && op != ValueOpcode::LogicalOr && + op != ValueOpcode::LogicalNot && op != ValueOpcode::IEqual32 && + op != ValueOpcode::INotEqual32 && op != ValueOpcode::ULessThanEqual32 && + op != ValueOpcode::SelectU1 && op != ValueOpcode::SelectU32) return finish(opaque()); auto& node = plan.value_storage.emplace_back(op); - node.SetArg(0, left); - node.SetArg(1, right); - return result = Value(&node); + for (size_t index = 0; index < inst->NumArgs(); ++index) + node.SetArg(index, ClonePredicate(inst->Arg(index))); + return finish(Value(&node)); }; plan.control_flow = ResourceControlFlow(program, ClonePredicate); + if (!uniform_control) { + plan.control_flow.clear(); + plan.control_phi_predecessors.clear(); + auto first = std::next(plan.value_storage.begin(), control_value_count); + for (auto it = first; it != plan.value_storage.end(); ++it) it->Invalidate(); + plan.value_storage.erase(first, plan.value_storage.end()); + } plan.uniform_fill = AnalyzeUniformFill(program); for (uint32_t i = 0; i < plan.uniform_fill.fill.words; ++i) { plan.uniform_fill.values[i] = Clone(plan.uniform_fill.values[i]); diff --git a/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp b/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp index 75f5c13b..c42090a7 100644 --- a/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp +++ b/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp @@ -585,6 +585,19 @@ bool SrtWalker::Arg(const Inst& inst, size_t index, uint64_t& result) { } bool SrtWalker::EvaluatePhi(const Inst& inst, uint64_t& result) { + if (const auto control = m_program.control_phi_predecessors.find(&inst); + control != m_program.control_phi_predecessors.end()) { + bool found = false; + for (size_t index = 0; index < inst.NumArgs(); ++index) { + if (m_reachable_blocks != nullptr && + !m_reachable_blocks->at(control->second[index])) continue; + uint64_t incoming = 0; + if (!Arg(inst, index, incoming) || (found && incoming != result)) return false; + result = incoming; + found = true; + } + return found; + } const auto value = ResolveInvariantPhi(m_program, Value(const_cast(&inst))); return !value.IsEmpty() && EvaluateWide(value, result); } @@ -1019,7 +1032,13 @@ bool SrtWalker::EvaluateInst(const Inst& inst, uint64_t& result) { } return false; case ValueOpcode::ULessThanEqual32: - if (binary()) { + // A zero lower bound accepts every U32, including a varying lane value. + if (!Arg(inst, 0, a)) return false; + if (a == 0u) { + result = 1u; + return true; + } + if (Arg(inst, 1, b)) { result = static_cast(a) <= static_cast(b); return true; } @@ -1043,9 +1062,22 @@ bool SrtWalker::EvaluateInst(const Inst& inst, uint64_t& result) { result = 0u; return true; } - if (!Arg(inst, 1, b) || (b != 0u && !left)) return false; - result = b != 0u; - return true; + if (Arg(inst, 1, b) && (b == 0u || left)) { + result = b != 0u; + return true; + } + if (!left) { + // Only lanes satisfying the left EXEC factor can make the conjunction + // true. Use a separate memo so this assumption cannot escape its RHS. + SrtWalker active(m_program, m_clean_evaluator != nullptr + ? CleanRuntime(m_runtime) : m_runtime, nullptr, inst.Arg(0).Resolve()); + active.m_reachable_blocks = m_reachable_blocks; + if (active.EvaluateWide(inst.Arg(1), b) && b == 0u) { + result = 0u; + return true; + } + } + return false; } case ValueOpcode::LogicalOr: { const bool left = Arg(inst, 0, a); @@ -1124,35 +1156,48 @@ bool SrtWalker::RefreshFlatBuffer(std::vector& flat) { } return true; } + auto& visited = m_program.visited_blocks; + auto& reachable = m_program.reachable_blocks; + auto& pending = m_program.pending_blocks; + visited.assign(m_program.control_flow.size(), 1u); + m_reachable_blocks = &visited; + if (m_clean_evaluator != nullptr) m_clean_evaluator->m_reachable_blocks = &visited; + for (;;) { + reachable.assign(m_program.control_flow.size(), 0u); + pending.clear(); + pending.push_back(0u); + while (!pending.empty()) { + const auto index = pending.back(); + pending.pop_back(); + if (reachable.at(index)) continue; + reachable[index] = 1u; + const auto& block = m_program.control_flow[index]; + uint32_t condition = 0; + auto& predicate = m_clean_evaluator != nullptr ? *m_clean_evaluator : *this; + if (!block.condition.IsEmpty() && + (m_runtime.read_specialization_memory != nullptr || !m_program.capture_specialization_reads) && + predicate.Evaluate(block.condition, condition)) { + pending.push_back(block.successors[condition != 0u ? 0u : 1u]); + } else { + pending.insert(pending.end(), block.successors.begin(), block.successors.end()); + } + } + if (reachable == visited) break; + visited.swap(reachable); + if (m_program.control_phi_predecessors.empty()) break; + // Only unknown values need another evaluation: successful Phi facts remain + // true as their possible predecessors shrink. Keep uniform-read memos stable. + } flat.assign(m_program.srt_reads.size(), 0u); active.assign(m_program.descriptor_sources.size(), 1u); - for (const auto& block: m_program.control_flow) { + for (const auto& block: m_program.control_flow) for (const auto source: block.sources) active.at(source) = 0u; - } - auto& visited = m_program.visited_blocks; - auto& pending = m_program.pending_blocks; - visited.assign(m_program.control_flow.size(), 0u); - pending.clear(); - pending.push_back(0u); - while (!pending.empty()) { - const auto index = pending.back(); - pending.pop_back(); - if (visited.at(index)) continue; - visited[index] = 1u; + for (uint32_t index = 0; index < visited.size(); ++index) { + if (!visited[index]) continue; const auto& block = m_program.control_flow[index]; for (const auto source: block.sources) active[source] = 1u; - for (const auto slot: block.srt_reads) { + for (const auto slot: block.srt_reads) if (!refresh(slot)) return false; - } - uint32_t condition = 0; - auto& predicate = m_clean_evaluator != nullptr ? *m_clean_evaluator : *this; - if (!block.condition.IsEmpty() && - (m_runtime.read_specialization_memory != nullptr || !m_program.capture_specialization_reads) && - predicate.Evaluate(block.condition, condition)) { - pending.push_back(block.successors[condition != 0u ? 0u : 1u]); - } else { - pending.insert(pending.end(), block.successors.begin(), block.successors.end()); - } } return true; } diff --git a/src/graphics/shader/recompiler/ir/passes/SrtWalker.h b/src/graphics/shader/recompiler/ir/passes/SrtWalker.h index 5aa8d70d..2bbc8f98 100644 --- a/src/graphics/shader/recompiler/ir/passes/SrtWalker.h +++ b/src/graphics/shader/recompiler/ir/passes/SrtWalker.h @@ -123,6 +123,7 @@ private: SrtRuntime m_runtime; SrtWalker* m_clean_evaluator = nullptr; Value m_active_mask; + const std::vector* m_reachable_blocks = nullptr; ResourcePlan::EvaluationContext& m_context; }; diff --git a/tests/ResourceMaterializationTests.cpp b/tests/ResourceMaterializationTests.cpp index 893481d9..667d7360 100644 --- a/tests/ResourceMaterializationTests.cpp +++ b/tests/ResourceMaterializationTests.cpp @@ -673,7 +673,7 @@ void TestSharedControlPredicates() { program.blocks[i]->terminator.false_block = program.blocks[2]; } auto plan = ExtractResourcePlan(program); - Check(plan.value_storage.size() == depth + 5 && + Check(plan.value_storage.size() <= 2u * depth + 12u && plan.control_flow.size() == 4 && plan.control_flow[0].condition == plan.control_flow[1].condition && !plan.capture_specialization_reads, @@ -793,6 +793,156 @@ void TestFiniteImageRefreshReusesScalarReads() { "finite image refresh grew reusable resource storage after warmup"); } +void TestEmptyTraversalSkipsUnmappedDescriptor(bool independent_mask) { + using namespace Libs::Graphics::ShaderRecompiler::IR; + using Libs::Graphics::ShaderRecompiler::CFG::TerminatorKind; + Program program; + program.stage = Libs::Graphics::ShaderType::Compute; + program.srt_plan_complete = program.resource_tracking_complete = true; + for (uint32_t i = 0; i < 6; ++i) AddValueBlock(program); + auto &entry = *program.blocks[0]; + auto &loop = *program.blocks[1]; + auto &body = *program.blocks[2]; + auto &merge = *program.blocks[3]; + auto &use = *program.blocks[4]; + auto &exit = *program.blocks[5]; + const auto emit = [](Block &block, ValueOpcode opcode, std::initializer_list args) { + return Value(&block.AppendNewInst(opcode, args)); + }; + const auto branch = [](Block &block, Block &target, Value condition = {}, Block *other = nullptr) { + block.terminator.kind = other ? TerminatorKind::ConditionalBranch : TerminatorKind::Branch; + block.terminator.true_block = ⌖ + block.terminator.false_block = other; + block.condition = condition; + block.AddBranch(&target); + if (other) block.AddBranch(other); + }; + const auto lane = emit(entry, ValueOpcode::GetBuiltin, + {Value(static_cast(StageInputKind::LocalInvocationId)), Value(0u)}); + const auto active = emit(entry, ValueOpcode::INotEqual32, {lane, Value(0u)}); + const auto initial_mask = independent_mask + ? emit(entry, ValueOpcode::IEqual32, {lane, Value(0u)}) : active; + const auto initial = emit(entry, ValueOpcode::SelectU32, + {initial_mask, Value(UINT32_MAX), Value(0u)}); + program.memory_info.push_back({.kind = ResourceKind::ScalarAddress, .planning_only = true}); + const auto address = emit(entry, ValueOpcode::GetAddressResource, {Value(0x1000u), Value(0u)}); + const auto count = emit(entry, ValueOpcode::LoadAddressU32, + {address, Value(0u), Value(0u), Value(true)}); + count.Instruction()->SetFlags(MemoryFlags{.index = 0}); + const auto rejected = emit(entry, ValueOpcode::ULessThanEqual32, {count, lane}); + const auto accepted = emit(entry, ValueOpcode::LogicalNot, {rejected}); + branch(entry, loop, emit(entry, ValueOpcode::LogicalAnd, {active, accepted}), &merge); + + // A nonempty traversal contains a varying loop-carried value. Its cycle must + // remain unknown; only the proven empty traversal can select the bypass arm. + auto &carried = loop.AppendNewInst(ValueOpcode::Phi); + carried.SetFlags(Type::U32); + const auto next = emit(body, ValueOpcode::IAdd32, {Value(&carried), Value(1u)}); + carried.AddPhiOperand(&entry, lane); + carried.AddPhiOperand(&body, next); + branch(loop, body, emit(loop, ValueOpcode::INotEqual32, {Value(&carried), Value(UINT32_MAX)}), &merge); + loop.terminator.loop_header = true; + loop.terminator.continue_block = &body; + loop.terminator.merge_block = &merge; + branch(body, loop); + auto &selected = merge.AppendNewInst(ValueOpcode::Phi); + selected.SetFlags(Type::U32); + selected.AddPhiOperand(&entry, initial); + selected.AddPhiOperand(&loop, Value(&carried)); + const auto valid = emit(merge, ValueOpcode::INotEqual32, {Value(&selected), Value(UINT32_MAX)}); + branch(merge, use, emit(merge, ValueOpcode::LogicalAnd, {active, valid}), &exit); + branch(use, exit); + DescriptorSource source; + source.dwords = {Value(0x8000u), Value(1u << 16u), Value(16u), Value(3u << 28u)}; + source.dword_count = 4; + program.descriptor_sources.push_back(source); + program.info.buffers.push_back({.source = 0, .read = true}); + program.memory_info.push_back({.kind = ResourceKind::Buffer, .resource = 0}); + const auto buffer = emit(use, ValueOpcode::GetBufferResource, + {source.dwords[0], source.dwords[1], source.dwords[2], source.dwords[3]}); + emit(use, ValueOpcode::LoadBufferU32, {buffer, Value(0u), Value(0u), Value(0u), Value(true)}) + .Instruction()->SetFlags(MemoryFlags{.index = 1}); + auto plan = ExtractResourcePlan(program); + struct Reads { uint32_t count = 0; bool clean = true; } reads; + const SrtRuntime runtime{ + .userdata = &reads, + .read_specialization_memory = +[](void *data, uint64_t address, std::span words) { + const auto &reads = *static_cast(data); + if (!reads.clean || address != 0x1000u || words.size() != 1) return false; + words[0] = reads.count; + return true; + }}; + ResourceSnapshot snapshot; + ResourceSpecialization specialization; + for (const auto state : {Reads{0u, true}, Reads{4u, true}, Reads{0u, false}, Reads{0u, true}}) { + reads = state; + uint32_t condition = 99; + Check(!SrtWalker(plan, runtime).Evaluate(plan.control_flow[3].condition, condition), + "direct merge evaluation used missing or previous runtime reachability"); + Check(MaterializeResources(plan, runtime, snapshot, specialization), + "partial traversal proof rejected a valid resource snapshot"); + const bool inactive = !independent_mask && reads.clean && reads.count == 0u; + Check(snapshot.buffers[0].dwords[0] == (inactive ? 0u : 0x8000u), + "empty traversal kept its descriptor or an uncertain traversal lost it"); + Check(specialization.buffers[0].zero_stride_oob == inactive, + "inactive traversal did not expose dead vector reads to specialization"); + if (inactive) { + Check(!snapshot.specialization_reads.empty(), "empty traversal lost its clean scalar read"); + for (const auto &read : snapshot.specialization_reads) + Check(read == std::pair{0x1000u, 4u}, + "empty traversal proof captured an unrelated memory range"); + } + } +} + +void TestCyclicControlPredicatesRemainUnknown() { + using namespace Libs::Graphics::ShaderRecompiler::IR; + using Libs::Graphics::ShaderRecompiler::CFG::TerminatorKind; + Program program; + program.stage = Libs::Graphics::ShaderType::Compute; + program.srt_plan_complete = program.resource_tracking_complete = true; + for (uint32_t i = 0; i < 4; ++i) AddValueBlock(program); + auto &entry = *program.blocks[0]; + auto &left = *program.blocks[1]; + auto &right = *program.blocks[2]; + auto &exit = *program.blocks[3]; + const auto lane = Value(&entry.AppendNewInst(ValueOpcode::GetBuiltin, + {Value(static_cast(StageInputKind::LocalInvocationId)), Value(0u)})); + const auto u = Value(&entry.AppendNewInst(ValueOpcode::INotEqual32, {lane, Value(0u)})); + const auto v = Value(&entry.AppendNewInst(ValueOpcode::INotEqual32, {lane, Value(1u)})); + auto &p = left.AppendNewInst(ValueOpcode::Phi); + auto &q = right.AppendNewInst(ValueOpcode::Phi); + p.SetFlags(Type::U1); + q.SetFlags(Type::U1); + const auto a = Value(&left.AppendNewInst(ValueOpcode::LogicalAnd, {u, Value(&p)})); + const auto b = Value(&right.AppendNewInst(ValueOpcode::LogicalAnd, {v, Value(&q)})); + p.AddPhiOperand(&entry, Value(true)); + p.AddPhiOperand(&right, b); + q.AddPhiOperand(&entry, Value(true)); + q.AddPhiOperand(&left, a); + const auto branch = [](Block &block, Value condition, Block &yes, Block &no) { + block.condition = condition; + block.terminator.kind = TerminatorKind::ConditionalBranch; + block.terminator.true_block = &yes; + block.terminator.false_block = &no; + block.AddBranch(&yes); + block.AddBranch(&no); + }; + // Two loop entries have no single header whose Phi can be discarded up front. + branch(entry, u, left, right); + branch(left, a, right, exit); + branch(right, b, left, exit); + auto plan = ExtractResourcePlan(program); + SrtWalker walker(plan, {}); + uint32_t value = 99; + Check(!walker.Evaluate(plan.control_flow[1].condition, value) && + !walker.Evaluate(plan.control_flow[2].condition, value), + "cyclic varying control acquired an unsupported uniform value"); + std::vector flat; + Check(walker.RefreshFlatBuffer(flat) && plan.visited_blocks == std::vector({1, 1, 1, 1}), + "cyclic control lost a possible branch during reachability"); +} + void TestMixedSamplerVariantsShareRuntimeDescriptor() { using namespace Libs::Graphics::ShaderRecompiler::IR; auto program = MixedSamplerProgram(); @@ -853,6 +1003,9 @@ int main() { TestFailedMaterializationRejectsStage(); TestSharedControlPredicates(); TestFiniteImageRefreshReusesScalarReads(); + for (const bool independent_mask : {false, true}) + TestEmptyTraversalSkipsUnmappedDescriptor(independent_mask); + TestCyclicControlPredicatesRemainUnknown(); TestMixedSamplerVariantsShareRuntimeDescriptor(); std::puts("ResourceMaterializationTests: all cases passed"); return 0;