diff --git a/src/graphics/shader/recompiler/ShaderRecompiler.cpp b/src/graphics/shader/recompiler/ShaderRecompiler.cpp index 3c1aebe0..be96be3e 100644 --- a/src/graphics/shader/recompiler/ShaderRecompiler.cpp +++ b/src/graphics/shader/recompiler/ShaderRecompiler.cpp @@ -817,7 +817,7 @@ TranslateResult TranslateProgram(const Decoder::Program& decoded, const CompileO MakeIrDump(cfg_dump, ir).c_str()); } } - IR::TrackResources(ir, decoded, native_cfg); + IR::TrackResources(ir, decoded, native_cfg, options.input_info); TranslateResult result; result.program = std::move(ir); if (options.dump_ir) { diff --git a/src/graphics/shader/recompiler/ir/ShaderIR.h b/src/graphics/shader/recompiler/ir/ShaderIR.h index 9967d165..0bb39967 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIR.h +++ b/src/graphics/shader/recompiler/ir/ShaderIR.h @@ -27,6 +27,9 @@ class CodeGenerator; namespace Libs::Graphics::ShaderRecompiler::IR { +// Finite descriptor domains also bound the arithmetic used to prove their indices. +constexpr uint32_t MaxIndirectDescriptorProbes = 65536u; + enum class ResourceKind { None, ScalarBuffer, diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp index 3534d34b..81d4bf91 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp @@ -19,7 +19,6 @@ namespace Libs::Graphics::ShaderRecompiler::IR { namespace { constexpr uint64_t AddressMask = 0x0000ffffffffffffull; -constexpr uint64_t MaxIndirectDescriptorProbes = 65536u; bool SpecializationFail(std::string_view message) { std::fprintf(stderr, "shader resource specialization failed: %.*s\n", @@ -782,55 +781,16 @@ static bool IsFullDispatchPredicate(Value value, uint32_t depth = 0) { return axis.IsImmediate() && axis.U32() < 3 && extent->Arg(0).Resolve() == axis; } -// Nonnegative affine coefficients for constant, local and workgroup coordinates. Reject modular -// arithmetic that could wrap; runtime coverage also bounds the largest invocation index. -static std::optional> FillIndex(Value value, uint32_t axis, Value guard, - uint32_t depth = 0) { - value = ResolveActiveU32(value, guard); - if (depth > 32 || value.GetType() != Type::U32) { - return {}; - } - if (value.IsImmediate()) { - return std::array {value.U32(), 0, 0}; - } - const auto* inst = value.TryInstruction(); - if (inst == nullptr) { - return {}; - } - const auto op = inst->GetOpcode(); - if (op == ValueOpcode::GetBuiltin && inst->Arg(1) == Value(axis)) { - if (inst->Arg(0) == Value(static_cast(StageInputKind::LocalInvocationId))) { - return std::array {0, 1, 0}; - } - if (inst->Arg(0) == Value(static_cast(StageInputKind::WorkgroupId))) { - return std::array {0, 0, 1}; - } - } - if (op != ValueOpcode::IAdd32 && op != ValueOpcode::IMul32 && - op != ValueOpcode::ShiftLeftLogical32) { - return {}; - } - auto left = FillIndex(inst->Arg(0), axis, guard, depth + 1); - auto right = FillIndex(inst->Arg(1), axis, guard, depth + 1); - if (!left || !right) { - return {}; - } - if (op == ValueOpcode::IMul32 && ((*right)[1] != 0 || (*right)[2] != 0)) { - std::swap(left, right); - } - if (op != ValueOpcode::IAdd32 && ((*right)[1] != 0 || (*right)[2] != 0)) { - return {}; - } - if (op == ValueOpcode::ShiftLeftLogical32) { - if ((*right)[0] >= 32) return {}; - (*right)[0] = uint64_t {1} << (*right)[0]; - } - for (uint32_t i = 0; i < left->size(); ++i) { - (*left)[i] = - op == ValueOpcode::IAdd32 ? (*left)[i] + (*right)[i] : (*left)[i] * (*right)[0]; - if ((*left)[i] > UINT32_MAX) return {}; - } - return left; +static std::optional> FillIndex(Value value, uint32_t axis, Value guard) { + return AffineIndex<3>(value, [&](Value at) { return ResolveActiveU32(at, guard); }, + [&](Value at) -> std::optional { + const auto* inst = at.TryInstruction(); + if (inst == nullptr || inst->GetOpcode() != ValueOpcode::GetBuiltin || + inst->Arg(1) != Value(axis)) return {}; + if (inst->Arg(0) == Value(static_cast(StageInputKind::LocalInvocationId))) return 1u; + if (inst->Arg(0) == Value(static_cast(StageInputKind::WorkgroupId))) return 2u; + return {}; + }); } static UniformFillPlan AnalyzeUniformFill(const Program& program) { diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp index 2cd8244c..838ed633 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp @@ -337,8 +337,10 @@ private: class Tracker { public: - Tracker(Program& program, const Decoder::Program& decoded, const CFG::Graph& native_cfg) + Tracker(Program& program, const Decoder::Program& decoded, const CFG::Graph& native_cfg, + ShaderStageInputInfo input) : m_program(program), m_decoded(decoded), m_native_cfg(native_cfg), + m_workgroup(program.stage == ShaderType::Compute ? input.compute : nullptr), m_scalar_writes(std::move(program.scalar_writes)), m_info(program.info) { std::ranges::sort(m_scalar_writes, {}, &Program::ScalarWrite::pc); m_info.buffers.clear(); @@ -1223,7 +1225,7 @@ private: bool MatchAffineOffset(Value value, Value guard, AffineOffset& out, uint32_t depth = 0) const { if (depth > 16u) return false; - value = value.Resolve(); + value = ActiveValue(value, guard); uint32_t immediate = 0; if (ImmediateU32(value, immediate)) { out.offset = immediate; @@ -1231,10 +1233,6 @@ private: } const auto* inst = value.TryInstruction(); if (inst == nullptr) return false; - if (inst->GetOpcode() == ValueOpcode::SelectU32 && inst->NumArgs() == 3u && - Implies(guard, inst->Arg(0))) { - return MatchAffineOffset(inst->Arg(1), guard, out, depth + 1u); - } if (inst->GetOpcode() == ValueOpcode::IAdd32 && inst->NumArgs() == 2u) { AffineOffset left, right; if (!MatchAffineOffset(inst->Arg(0), guard, left, depth + 1u) || @@ -1559,6 +1557,269 @@ private: }); } + Value ActiveValue(Value value, Value guard) const { + value = value.Resolve(); + while (const auto* select = value.TryInstruction()) { + if (select->GetOpcode() != ValueOpcode::SelectU32 || + !Implies(guard, select->Arg(0))) break; + value = select->Arg(1).Resolve(); + } + return value; + } + + bool Reaches(const Block* from, const Block* to, const Block* stop = nullptr) const { + std::vector pending {to}; + for (size_t index = 0; index < pending.size(); ++index) { + if (pending[index] == stop) continue; + if (pending[index] == from) return true; + for (const auto* previous: pending[index]->ImmPredecessors()) + if (std::ranges::find(pending, previous) == pending.end()) pending.push_back(previous); + } + return false; + } + + bool InCycle(const Block* block) const { + return std::ranges::any_of(block->ImmPredecessors(), + [&](const Block* previous) { return Reaches(block, previous); }); + } + + static bool BeforeInBlock(const Inst& before, const Inst& after) { + if (before.Parent() != after.Parent()) return false; + for (const auto& inst: *before.Parent()) { + if (&inst == &before) return true; + if (&inst == &after) return false; + } + return false; + } + + bool InitializedBefore(const Inst& initial, const Inst& use, Value witness) const { + if (initial.Parent() == use.Parent()) return BeforeInBlock(initial, use); + std::vector pending {use.Parent()}; + for (size_t index = 0; index < pending.size(); ++index) { + const auto* current = pending[index]; + if (current == initial.Parent()) continue; + if (current->ImmPredecessors().empty()) return false; + for (const auto* previous: current->ImmPredecessors()) { + const auto edge = ConditionalEdge(previous, current); + // The proven local-zero invocation makes an all-inactive bypass impossible. + if (edge && edge->lanes == LaneQuantifier::All && !edge->positive && + Implies(witness, edge->condition)) continue; + if (std::ranges::find(pending, previous) == pending.end()) pending.push_back(previous); + } + } + return true; + } + + std::optional> LocalIndex(Value value, Value active, + Value& induction, bool allow_induction) const { + return AffineIndex<5>(value, [&](Value at) { return ActiveValue(at, active); }, + [&](Value at) -> std::optional { + const auto* inst = at.TryInstruction(); + if (inst == nullptr) return {}; + uint32_t axis = 0; + if (inst->GetOpcode() == ValueOpcode::GetBuiltin && + inst->Arg(0) == Value(static_cast(StageInputKind::LocalInvocationId)) && + ImmediateU32(inst->Arg(1), axis) && axis < 3u) return axis + 1u; + if (!allow_induction || inst->GetOpcode() != ValueOpcode::Phi || + (!induction.IsEmpty() && induction != at)) return {}; + induction = at; + return 4u; + }); + } + + bool LinearLocalIndex(const std::array& index) const { + uint64_t stride = 1u; + for (uint32_t axis = 0; axis < 3u; ++axis) { + if (m_workgroup->threads_num[axis] != 1u && index[axis + 1u] != stride) return false; + stride *= m_workgroup->threads_num[axis]; + } + return index[0] == 0u; + } + + bool LocalZeroWitness(Value guard) const { + const auto* equal = SimplifyGuard(guard).TryInstruction(); + if (equal == nullptr || equal->GetOpcode() != ValueOpcode::IEqual32) return false; + for (uint32_t arg = 0; arg < 2u; ++arg) { + if (equal->Arg(arg).Resolve() != Value(0u)) continue; + Value induction; + const auto index = LocalIndex(equal->Arg(arg ^ 1u), guard, induction, false); + if (index && LinearLocalIndex(*index)) return true; + } + return false; + } + + Value UnsignedBound(Value guard, Value value) const { + const auto* test = SimplifyGuard(guard).TryInstruction(); + if (test == nullptr) return {}; + if (test->GetOpcode() == ValueOpcode::LogicalAnd) { + const auto left = UnsignedBound(test->Arg(0), value); + return left.IsEmpty() ? UnsignedBound(test->Arg(1), value) : left; + } + if (test->GetOpcode() == ValueOpcode::ULessThan32 && + EquivalentValue(m_program, test->Arg(0), value)) return test->Arg(1); + if (test->GetOpcode() == ValueOpcode::LogicalNot) { + const auto* inverse = test->Arg(0).Resolve().TryInstruction(); + if (inverse != nullptr && inverse->GetOpcode() == ValueOpcode::ULessThanEqual32 && + EquivalentValue(m_program, inverse->Arg(1), value)) return inverse->Arg(0); + } + return {}; + } + + Value SharedLoadBound(Value value, Value active) { + const auto* read = ActiveValue(value, active).TryInstruction(); + if (read == nullptr || read->GetOpcode() != ValueOpcode::LoadSharedU32 || + m_workgroup == nullptr || m_program.stage != ShaderType::Compute || + read->Arg(1).Resolve() != Value(true)) return {}; + uint64_t group_size = 1u; + for (const auto dimension: m_workgroup->threads_num) { + if (dimension == 0u || dimension > m_workgroup->wave_size) return {}; + group_size *= dimension; + } + // Indexed LDS operations within one native wave execute in order. Multiwave + // append lists require a separate workgroup synchronization proof. + if (group_size != m_workgroup->wave_size || group_size != m_program.wave_size || + (group_size != 32u && group_size != 64u)) return {}; + const auto memory = [&](const Inst& inst) -> const MemoryInfo* { + const auto index = inst.Flags().index; + if (index >= m_program.memory_info.size()) return nullptr; + const auto& info = m_program.memory_info[index]; + return info.kind == ResourceKind::Lds && info.data_bits == 32u && + info.secondary_offset == 0u && info.component_index == 0u ? &info : nullptr; + }; + const auto constant_address = [&](const Inst& inst, uint32_t& address) { + const auto* info = memory(inst); + if (info == nullptr || !ImmediateU32(ActiveValue(inst.Arg(0), inst.Arg(inst.NumArgs() - 1u)), address) || + uint64_t {address} + info->offset > UINT32_MAX) return false; + address += info->offset; + return (address & 3u) == 0u; + }; + const auto* read_memory = memory(*read); + AffineOffset read_offset; + if (read_memory == nullptr || !MatchAffineOffset(read->Arg(0), active, read_offset) || + read_offset.stride != 4u || read_offset.offset + read_memory->offset > UINT32_MAX) return {}; + const auto* limit = BoundedLoop(read_offset.index, read->Parent(), + [](const Inst& compare, uint32_t) { return compare.GetOpcode() == ValueOpcode::ULessThan32; }); + if (limit == nullptr) return {}; + const auto* counter_read = ActiveValue(limit->Arg(1), active).TryInstruction(); + uint32_t consumed_counter = 0; + if (counter_read == nullptr || counter_read->GetOpcode() != ValueOpcode::LoadSharedU32 || + counter_read->Arg(1).Resolve() != Value(true) || + !constant_address(*counter_read, consumed_counter)) return {}; + + struct Initializer { const Inst* inst; uint32_t address; uint32_t words; }; + struct Append { const Inst* atomic; const Inst* store = nullptr; uint32_t counter; uint32_t base = 0; Value bound; }; + std::vector initializers; + std::vector appends; + std::vector stores; + for (const auto* block: m_program.blocks) { + for (const auto& inst: *block) { + const auto op = inst.GetOpcode(); + if (AddressOpcodeInfoOf(op).access == AddressAccess::Write) { + const auto index = inst.Flags().index; + if (index >= m_program.memory_info.size() || + m_program.memory_info[index].kind == ResourceKind::Flat || + m_program.memory_info[index].kind == ResourceKind::FlatLocal) return {}; + } + const auto access = SharedAccessOf(op); + if (access == SharedAccess::None || access == SharedAccess::Read) continue; + const auto* info = memory(inst); + if (info == nullptr) return {}; + const auto guard = inst.Arg(inst.NumArgs() - 1u); + uint32_t address = 0; + if (access == SharedAccess::Atomic) { + if (op != ValueOpcode::SharedAtomicIAdd32 || + ActiveValue(inst.Arg(1), guard) != Value(1u) || + !constant_address(inst, address) || + uint64_t {address} + 4u > uint64_t {m_workgroup->lds_size_dwords} * 4u || + address >= 0x10000u || + std::ranges::any_of(appends, [&](const Append& at) { return at.counter == address; })) return {}; + appends.push_back({.atomic = &inst, .counter = address}); + } else if (access == SharedAccess::Write) { + const auto words = SharedComponentCount(op); + bool zero = LocalZeroWitness(guard) && constant_address(inst, address) && + address % (words * 4u) == 0u; + for (uint32_t word = 0; word < words; ++word) + zero = zero && ActiveValue(inst.Arg(word + 1u), guard) == Value(0u); + if (zero) initializers.push_back({&inst, address, words}); + else if (op == ValueOpcode::WriteSharedU32) stores.push_back(&inst); + else return {}; + } else return {}; + } + } + if (appends.empty() || stores.size() != appends.size() || initializers.empty()) return {}; + for (const auto& initial: initializers) { + if (InCycle(initial.inst->Parent())) return {}; + // Every initialized word is a counter; arbitrary shared writes are not append provenance. + for (uint32_t word = 0; word < initial.words; ++word) + if (std::ranges::none_of(appends, [&](const Append& at) { + return uint64_t {initial.address} + word * 4u == at.counter; + })) return {}; + } + for (auto& append: appends) { + const auto guard = append.atomic->Arg(2); + if (Reaches(counter_read->Parent(), append.atomic->Parent())) return {}; + uint32_t initialized = 0; + for (const auto& initial: initializers) { + const auto witness = initial.inst->Arg(initial.inst->NumArgs() - 1u); + if (!InitializedBefore(*initial.inst, *append.atomic, witness) || + !InitializedBefore(*initial.inst, *counter_read, witness)) return {}; + if (append.counter >= initial.address && + uint64_t {append.counter} < uint64_t {initial.address} + initial.words * 4u) ++initialized; + } + if (initialized != 1u) return {}; + for (const auto* store: stores) { + if (!EquivalentValue(m_program, guard, store->Arg(2)) || + !BeforeInBlock(*append.atomic, *store)) continue; + AffineOffset address; + const auto* info = memory(*store); + if (!MatchAffineOffset(store->Arg(0), guard, address) || + address.index != Value(const_cast(append.atomic)) || address.stride != 4u || + (address.offset + info->offset) % 4u != 0u || + address.offset + info->offset + uint64_t {MaxIndirectDescriptorProbes} * 4u > UINT32_MAX || + std::ranges::any_of(appends, [&](const Append& at) { + return address.offset + info->offset <= at.counter; + })) continue; + if (append.store != nullptr) return {}; + const auto payload = ActiveValue(store->Arg(1), guard); + Value induction; + const auto index = LocalIndex(payload, guard, induction, true); + if (!index || !LinearLocalIndex(*index) || (*index)[4] != 1u || induction.IsEmpty()) return {}; + const auto* bound = BoundedInduction(induction, append.atomic->Parent(), + [&](uint32_t initial, uint32_t step) { return initial == 0u && step == group_size; }, + [&](const Inst& compare, uint32_t initial) { + return compare.GetOpcode() == ValueOpcode::ULessThan32 && + ValidateRuntimeValue(m_program, compare.Arg(1), RuntimeValueType::Integer) && + !InCycle(induction.Instruction()->PhiBlock(initial)); + }); + if (bound == nullptr || !EquivalentValue(m_program, UnsignedBound(guard, payload), bound->Arg(1))) return {}; + // Each re-entry must advance the producer induction; an inner append + // cycle could otherwise increment the counter twice for one candidate. + if (std::ranges::any_of(append.atomic->Parent()->ImmPredecessors(), [&](const Block* previous) { + return Reaches(append.atomic->Parent(), previous, induction.Instruction()->Parent()); + })) return {}; + append.store = store; + append.base = static_cast(address.offset + info->offset); + append.bound = bound->Arg(1); + } + if (append.store == nullptr) return {}; + } + if (std::ranges::none_of(appends, [&](const Append& at) { + return at.counter == consumed_counter && at.base == read_offset.offset + read_memory->offset; + })) return {}; + // Materialization validates every producer bound through this union against + // MaxIndirectDescriptorProbes. Thus counters, lane enumeration and byte addresses + // cannot wrap. Overlapping payload lists remain within the union; OOB LDS reads yield zero. + Value bound = appends.front().bound; + for (size_t index = 1; index < appends.size(); ++index) { + if (EquivalentValue(m_program, bound, appends[index].bound)) continue; + auto& maximum = m_program.value_storage.emplace_back(ValueOpcode::UMax32); + maximum.SetArg(0, bound); + maximum.SetArg(1, appends[index].bound); + bound = Value(&maximum); + } + return bound; + } + bool MatchScalarMaterialKey(Value key, DescriptorSource::IndirectDescriptor& indirect, DescriptorSource& material_source) { const auto* read = key.Resolve().TryInstruction(); @@ -1585,11 +1846,19 @@ private: AffineOffset affine; if (!MatchAffineOffset(lane->Arg(0), lane->Arg(1), affine) || affine.index.IsEmpty() || affine.stride == 0u) return false; - // U32 multiplication can wrap. Its complete byte-offset domain is one - // congruence class, not the source V#'s structured record stride. - const auto divisor = std::gcd(affine.stride, uint64_t {1} << 32u); - stride = static_cast(std::max(divisor, 4u)); - offset = static_cast(affine.offset % divisor) & ~3u; + count = SharedLoadBound(affine.index, lane->Arg(1)); + if (!count.IsEmpty() && ((affine.stride | affine.offset) & 3u) == 0u && + affine.offset + memory->offset <= UINT32_MAX) { + stride = static_cast(affine.stride); + offset = static_cast(affine.offset); + } else { + count = {}; + // Without a bounded source, U32 multiplication's complete offset domain + // is one congruence class rather than the V#'s structured record stride. + const auto divisor = std::gcd(affine.stride, uint64_t {1} << 32u); + stride = static_cast(std::max(divisor, 4u)); + offset = static_cast(affine.offset % divisor) & ~3u; + } } if (!MakeRuntimeTableSource(*read, material_source)) return false; indirect.selector.emplace(DescriptorSource::IndirectDescriptor::SelectorRead { @@ -2639,6 +2908,7 @@ private: Program& m_program; const Decoder::Program& m_decoded; const CFG::Graph& m_native_cfg; + const ShaderWorkgroupInputInfo* m_workgroup; std::vector m_scalar_writes; std::vector m_resolved_handles; std::vector m_srt_visiting; @@ -2655,8 +2925,9 @@ private: } // namespace -void TrackResources(Program& program, const Decoder::Program& decoded, const CFG::Graph& native_cfg) { - Tracker(program, decoded, native_cfg).Run(); +void TrackResources(Program& program, const Decoder::Program& decoded, const CFG::Graph& native_cfg, + ShaderStageInputInfo input) { + Tracker(program, decoded, native_cfg, input).Run(); } } // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.h b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.h index 03e53023..9b6a701a 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.h +++ b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.h @@ -6,7 +6,8 @@ namespace Libs::Graphics::ShaderRecompiler::IR { // Resolves native descriptor sources, plans their scalar reads, and assigns dense resource bindings. -void TrackResources(Program& program, const Decoder::Program& decoded, const CFG::Graph& native_cfg); +void TrackResources(Program& program, const Decoder::Program& decoded, const CFG::Graph& native_cfg, + ShaderStageInputInfo input = {}); } // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp b/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp index 06731ee7..c9c49c55 100644 --- a/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp +++ b/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp @@ -141,6 +141,7 @@ bool IsRuntimeUniformOp(ValueOpcode op) { case ValueOpcode::IMul64: case ValueOpcode::UMulHi: case ValueOpcode::UMin32: + case ValueOpcode::UMax32: case ValueOpcode::ShiftLeftLogical32: case ValueOpcode::ShiftLeftLogical64: case ValueOpcode::ShiftRightLogical32: @@ -793,8 +794,11 @@ bool SrtWalker::EvaluateInst(const Inst& inst, uint64_t& result) { } return false; case ValueOpcode::UMin32: + case ValueOpcode::UMax32: if (binary()) { - result = std::min(static_cast(a), static_cast(b)); + result = inst.GetOpcode() == ValueOpcode::UMin32 + ? std::min(static_cast(a), static_cast(b)) + : std::max(static_cast(a), static_cast(b)); return true; } return false; diff --git a/src/graphics/shader/recompiler/ir/passes/SrtWalker.h b/src/graphics/shader/recompiler/ir/passes/SrtWalker.h index 2083ec6e..2c78b8ca 100644 --- a/src/graphics/shader/recompiler/ir/passes/SrtWalker.h +++ b/src/graphics/shader/recompiler/ir/passes/SrtWalker.h @@ -3,6 +3,7 @@ #include "graphics/shader/recompiler/ir/ShaderIR.h" +#include #include namespace Libs::Graphics::ShaderRecompiler::IR { @@ -49,6 +50,48 @@ void CompileSrtWalker(ResourcePlan& program); // Uses the strict reader for values that affect shader specialization. SrtRuntime CleanRuntime(SrtRuntime runtime); +// Nonnegative affine coefficients: element zero is constant, remaining elements are +// caller-selected leaves. Runtime range proofs must additionally exclude U32 wrap. +template +std::optional> AffineIndex( + Value value, const auto& resolve, const auto& leaf, uint32_t depth = 0) { + value = resolve(value); + if (depth > 32u || value.GetType() != Type::U32) return {}; + std::array result {}; + if (value.IsImmediate()) { + result[0] = value.U32(); + return result; + } + if (const auto index = leaf(value)) { + result[*index] = 1u; + return result; + } + const auto* inst = value.TryInstruction(); + if (inst == nullptr) return {}; + const auto op = inst->GetOpcode(); + if (op != ValueOpcode::IAdd32 && op != ValueOpcode::IMul32 && + op != ValueOpcode::ShiftLeftLogical32) return {}; + auto left = AffineIndex(inst->Arg(0), resolve, leaf, depth + 1u); + auto right = AffineIndex(inst->Arg(1), resolve, leaf, depth + 1u); + if (!left || !right) return {}; + const auto constant = [](const auto& coefficients) { + return std::ranges::all_of(std::span(coefficients).subspan(1u), + [](uint64_t coefficient) { return coefficient == 0u; }); + }; + if (op == ValueOpcode::IMul32 && !constant(*right)) std::swap(left, right); + if (op != ValueOpcode::IAdd32 && !constant(*right)) return {}; + if (op == ValueOpcode::ShiftLeftLogical32) { + if ((*right)[0] >= 32u) return {}; + (*right)[0] = uint64_t {1} << (*right)[0]; + } + for (size_t index = 0; index < Size; ++index) { + (*left)[index] = op == ValueOpcode::IAdd32 + ? (*left)[index] + (*right)[index] : (*left)[index] * (*right)[0]; + if ((*left)[index] > UINT32_MAX) return {}; + } + return left; +} + // One memoized evaluation session shared by the entire shader resource refresh. class SrtWalker { public: diff --git a/tests/ResourceTrackingTests.cpp b/tests/ResourceTrackingTests.cpp index 93dc8222..ef0db04e 100644 --- a/tests/ResourceTrackingTests.cpp +++ b/tests/ResourceTrackingTests.cpp @@ -125,7 +125,7 @@ struct Fixture { Value(0u), Value(0u), Value(0u)}); } - void PlanAndTrack() { + void PlanAndTrack(Libs::Graphics::ShaderStageInputInfo input = {}) { for (size_t index = 0; index < program.blocks.size(); ++index) { const auto condition = program.blocks[index]->condition; if (!condition.IsEmpty()) @@ -147,7 +147,7 @@ struct Fixture { Emit(ValueOpcode::ReferenceU32, {value}, 0, target); } } - TrackResources(program, {}, {}); + TrackResources(program, {}, {}, input); } }; @@ -1912,6 +1912,285 @@ void TestScalarBufferSelectorDomain() { "large V# bounds extended the native U32 domain or invented an OOB zero key"); } +void TestSharedAppendSelectorBounds() { + namespace CFG = Libs::Graphics::ShaderRecompiler::CFG; + enum class Variant { + Valid, PayloadOob, MissingInit, InactiveInit, UnknownInit, DuplicateAtomic, + WrongAtomicGuard, UnguardedCandidate, WrongStep, WrongLocalIndex, Multiwave, + OuterReentry, InnerAppendCycle, EarlyCounterRead, WrongConsumerBound, ExtraWriter, CounterAlias, + PayloadWrap, CounterOob + }; + const auto make_plan = [](Variant variant) { + Fixture fixture; + ShaderComputeInputInfo compute; + compute.threads_num[0] = 8u; + compute.threads_num[1] = variant == Variant::Multiwave ? 16u : 8u; + compute.threads_num[2] = 1u; + compute.lds_size_dwords = variant == Variant::PayloadOob ? 2u : + variant == Variant::CounterOob ? 1u : 640u; + compute.host_subgroup_size = 32u; + fixture.program.wave_size = compute.wave_size = 64u; + const auto branch = [](Block *from, Block *to) { + from->AddBranch(to); + from->terminator = {.kind = CFG::TerminatorKind::Branch, .true_block = to}; + }; + const auto choose = [&](Block *from, Block *yes, Block *no, Value condition, + CFG::BranchCondition kind) { + from->AddBranch(yes); + from->AddBranch(no); + from->condition = fixture.Emit(ValueOpcode::ConditionRef, {condition}, kind, from); + from->terminator = {.kind = CFG::TerminatorKind::ConditionalBranch, + .true_block = yes, .false_block = no}; + }; + const auto shared = [&](uint32_t offset) { + MemoryInfo memory; + memory.kind = ResourceKind::Lds; + memory.offset = offset; + return fixture.AddMemory(memory, 0x10u); + }; + auto *entry = fixture.block; + auto *initialize = fixture.AddBlock(); + auto *initialized = fixture.AddBlock(); + const auto material = fixture.Buffer({fixture.UserData(0), fixture.UserData(1), + fixture.UserData(2), fixture.UserData(3)}); + const auto table = fixture.Address(fixture.UserData(4), fixture.UserData(5)); + const auto counts = fixture.Address(fixture.UserData(6), fixture.UserData(7)); + std::array count; + for (uint32_t index = 0; index < count.size(); ++index) { + MemoryInfo memory; + memory.kind = ResourceKind::ScalarAddress; + memory.offset = index * 4u; + count[index] = fixture.Emit(ValueOpcode::LoadAddressU32, + {counts, Value(0u), Value(0u), Value(true)}, fixture.AddMemory(memory, 0x20u)); + } + const auto x = fixture.Emit(ValueOpcode::GetBuiltin, + {Value(static_cast(StageInputKind::LocalInvocationId)), Value(0u)}); + const auto y = fixture.Emit(ValueOpcode::GetBuiltin, + {Value(static_cast(StageInputKind::LocalInvocationId)), Value(1u)}); + const auto local = fixture.Emit(ValueOpcode::IAdd32, + {x, fixture.Emit(ValueOpcode::ShiftLeftLogical32, {y, Value(3u)})}); + auto zero_lane = fixture.Emit(ValueOpcode::IEqual32, {local, Value(0u)}); + if (variant == Variant::InactiveInit) zero_lane = Value(false); + if (variant == Variant::UnknownInit) + zero_lane = fixture.Emit(ValueOpcode::LogicalAnd, + {zero_lane, fixture.Emit(ValueOpcode::IEqual32, {fixture.UserData(8), Value(0u)})}); + choose(entry, initialize, initialized, zero_lane, CFG::BranchCondition::ExecNonZero); + if (variant != Variant::MissingInit) + fixture.Emit(ValueOpcode::WriteSharedU32x2, + {Value(0u), Value(0u), Value(0u), zero_lane}, shared(0u), initialize); + branch(initialize, initialized); + Value final_counter; + if (variant == Variant::EarlyCounterRead) + final_counter = fixture.Emit(ValueOpcode::LoadSharedU32, + {Value(0u), Value(true)}, shared(0u), initialized); + auto *previous = initialized; + for (uint32_t list = 0; list < 2u; ++list) { + auto *header = fixture.AddBlock(); + auto *cull = fixture.AddBlock(); + auto *append = fixture.AddBlock(); + auto *latch = fixture.AddBlock(); + auto *exit = fixture.AddBlock(); + branch(previous, header); + auto &batch = header->AppendNewInst(ValueOpcode::Phi); + batch.SetFlags(Type::U32); + const auto batch_valid = fixture.Emit(ValueOpcode::ULessThan32, + {Value(&batch), count[list]}, 0, header); + choose(header, cull, exit, batch_valid, CFG::BranchCondition::SccNonZero); + const auto local_candidate = variant == Variant::WrongLocalIndex + ? fixture.Emit(ValueOpcode::IAdd32, {local, x}, 0, cull) : local; + const auto candidate = fixture.Emit(ValueOpcode::IAdd32, {Value(&batch), local_candidate}, 0, cull); + const auto even = fixture.Emit(ValueOpcode::IEqual32, + {fixture.Emit(ValueOpcode::BitwiseAnd32, {candidate, Value(1u)}, 0, cull), Value(0u)}, 0, cull); + auto active = even; + if (variant != Variant::UnguardedCandidate) { + const auto candidate_valid = list == 0u + ? fixture.Emit(ValueOpcode::LogicalNot, + {fixture.Emit(ValueOpcode::ULessThanEqual32, {count[list], candidate}, 0, cull)}, 0, cull) + : fixture.Emit(ValueOpcode::ULessThan32, {candidate, count[list]}, 0, cull); + active = fixture.Emit(ValueOpcode::LogicalAnd, + {candidate_valid, even}, 0, cull); + } + choose(cull, append, latch, active, CFG::BranchCondition::ExecNonZero); + const auto atomic_flags = shared(list == 0u || variant == Variant::CounterAlias ? 4u : 0u); + const auto atomic_guard = variant == Variant::WrongAtomicGuard ? Value(true) : active; + const auto ordinal = fixture.Emit(ValueOpcode::SharedAtomicIAdd32, + {Value(0u), Value(1u), atomic_guard}, atomic_flags, append); + if (variant == Variant::DuplicateAtomic && list == 1u) + fixture.Emit(ValueOpcode::SharedAtomicIAdd32, + {Value(0u), Value(1u), active}, atomic_flags, append); + const auto address = fixture.Emit(ValueOpcode::ShiftLeftLogical32, + {fixture.Emit(ValueOpcode::SelectU32, {active, ordinal, Value(UINT32_MAX)}, 0, append), Value(2u)}, + 0, append); + const auto payload = fixture.Emit(ValueOpcode::SelectU32, + {active, candidate, Value(UINT32_MAX)}, 0, append); + const auto base = list == 0u ? 1032u : variant == Variant::PayloadWrap ? 0xfffffff8u : 8u; + fixture.Emit(ValueOpcode::WriteSharedU32, {address, payload, active}, shared(base), append); + if (variant == Variant::InnerAppendCycle && list == 1u) { + auto *repeat = fixture.AddBlock(); + branch(append, repeat); + choose(repeat, append, latch, even, CFG::BranchCondition::ExecNonZero); + } else { + branch(append, latch); + } + const auto step = fixture.Emit(ValueOpcode::IAdd32, + {Value(&batch), Value(variant == Variant::WrongStep ? 32u : 64u)}, 0, latch); + branch(latch, header); + batch.AddPhiOperand(previous, Value(0u)); + batch.AddPhiOperand(latch, step); + previous = exit; + } + if (variant == Variant::ExtraWriter) + fixture.Emit(ValueOpcode::WriteSharedU32, + {Value(0u), Value(UINT32_MAX), Value(true)}, shared(12u), previous); + if (final_counter.IsEmpty()) + final_counter = fixture.Emit(ValueOpcode::LoadSharedU32, + {Value(0u), Value(true)}, shared(0u), previous); + auto *header = fixture.AddBlock(); + auto *body = fixture.AddBlock(); + auto *latch = fixture.AddBlock(); + auto *exit = fixture.AddBlock(); + branch(previous, header); + auto &index = header->AppendNewInst(ValueOpcode::Phi); + index.SetFlags(Type::U32); + const auto in_list = fixture.Emit(ValueOpcode::ULessThan32, + {Value(&index), variant == Variant::WrongConsumerBound ? count[1] : final_counter}, 0, header); + choose(header, body, exit, in_list, CFG::BranchCondition::VccNonZero); + fixture.block = body; + const auto selected = fixture.Emit(ValueOpcode::LoadSharedU32, + {fixture.Emit(ValueOpcode::ShiftLeftLogical32, {Value(&index), Value(2u)}), Value(true)}, shared(8u)); + const auto offset32 = fixture.Emit(ValueOpcode::ShiftLeftLogical32, {selected, Value(5u)}); + const auto offset = fixture.Emit(ValueOpcode::ReadFirstLane, + {fixture.Emit(ValueOpcode::IAdd32, + {fixture.Emit(ValueOpcode::ShiftLeftLogical32, {offset32, Value(2u)}), offset32}), Value(true)}); + MemoryInfo scalar; + scalar.kind = ResourceKind::ScalarBuffer; + scalar.offset = 144u; + const auto key = fixture.Emit(ValueOpcode::ReadConstBuffer, + {material, offset}, fixture.AddMemory(scalar, 0x30u)); + const auto record = fixture.Emit(ValueOpcode::ShiftLeftLogical32, {key, Value(5u)}); + std::array words; + for (uint32_t word = 0; word < words.size(); ++word) { + scalar.kind = ResourceKind::ScalarAddress; + scalar.offset = word * 4u; + words[word] = fixture.Emit(ValueOpcode::LoadAddressU32, + {table, record, Value(0u), Value(true)}, fixture.AddMemory(scalar, 0x40u)); + } + MemoryInfo sample; + sample.kind = ResourceKind::Image; + sample.image_dimension = Decoder::ImageDimension::Dim2D; + fixture.Emit(ValueOpcode::ImageSampleRaw, + {fixture.Image(words), fixture.Sampler({Value(0u), Value(0u), Value(0u), Value(0u)}), + fixture.ImageAddress(), Value(true)}, fixture.AddMemory(sample, 0x50u)); + branch(body, latch); + const auto next = fixture.Emit(ValueOpcode::IAdd32, {Value(&index), Value(1u)}, 0, latch); + branch(latch, header); + index.AddPhiOperand(previous, Value(0u)); + index.AddPhiOperand(latch, next); + if (variant == Variant::OuterReentry) branch(exit, initialized); + fixture.PlanAndTrack({.compute = &compute}); + const auto &indirect = fixture.program.descriptor_sources[ + fixture.program.info.images[0].source].indirect_descriptor; + const bool proven = variant == Variant::Valid || variant == Variant::PayloadOob; + Check(indirect && indirect->selector && indirect->selector->offset == 144u && + indirect->selector->stride == (proven ? 160u : 32u) && + indirect->key_count.IsEmpty() != proven && + (proven ? indirect->selector_first == Value(0u) : indirect->selector_first.IsEmpty()), + proven ? "shared append provenance did not narrow the material record domain" : + "shared append provenance accepted an unproved initialization, bound or ordering"); + return ExtractResourcePlan(fixture.program); + }; + auto plan = make_plan(Variant::Valid); + make_plan(Variant::PayloadOob); // Ordinary LDS reads beyond the allocation return zero. + for (const auto variant : {Variant::MissingInit, Variant::InactiveInit, Variant::UnknownInit, + Variant::DuplicateAtomic, Variant::WrongAtomicGuard, Variant::UnguardedCandidate, + Variant::WrongStep, Variant::WrongLocalIndex, Variant::Multiwave, Variant::OuterReentry, + Variant::InnerAppendCycle, Variant::EarlyCounterRead, Variant::WrongConsumerBound, Variant::ExtraWriter, + Variant::CounterAlias, Variant::PayloadWrap, Variant::CounterOob}) + make_plan(variant); + + LinearTestMemory memory; + for (uint32_t index = 0; index < 8u; ++index) memory.words[(144u + 160u * index) / 4u] = index + 1u; + memory.words[176u / 4u] = 0xdeadbeefu; // A different field reachable only through unbounded U32 wrap. + for (uint32_t key = 0; key <= 8u; ++key) { + const auto word = (0x1000u + key * 32u) / 4u; + memory.words[word] = 0x100u + key; + memory.words[word + 1u] = static_cast( + Libs::Graphics::Prospero::BufferFormat::k32_32_32_32Float) << 20u; + memory.words[word + 3u] = Libs::Graphics::DstSel(4, 5, 6, 7) | + (static_cast(Libs::Graphics::Prospero::ImageType::kColor2D) << 28u); + } + std::array user_data {0x1000u, 160u << 16u, 8u, 0u, 0x2000u, 0u, 0x1800u, 0u, 0u}; + const SrtRuntime runtime {.user_data = user_data, .read_memory = ReadLinearTestMemory, + .userdata = &memory, .read_specialization_memory = ReadLinearTestMemory}; + ResourceSnapshot snapshot; + ResourceSpecialization specialization; + memory.fail_address = memory.watched_address = 0x1090u; + Check(MaterializeResources(plan, runtime, snapshot, specialization) && + snapshot.images.size() == 1u && memory.descriptor_reads == 0u && memory.watched_reads == 0u, + "empty shared append lists probed material keys or image descriptors"); + memory.fail_address = 0x10b0u; + for (const auto counts : {std::array{1u, 2u}, std::array{3u, 1u}, std::array{1u, 1u}, std::array{0u, 0u}}) { + memory.words[0x800u / 4u] = counts[0]; + memory.words[0x804u / 4u] = counts[1]; + const auto expected = std::max(counts[0], counts[1]); + Check(MaterializeResources(plan, runtime, snapshot, specialization) && + snapshot.images.size() == std::max(expected, 1u) && + (expected == 0u || snapshot.images.back().dwords[0] == 0x100u + expected), + "shared append union bound or refreshed runtime counts selected stale material keys"); + } + memory.words[0x800u / 4u] = 1u; + memory.words[0x804u / 4u] = 2u; + memory.words[(144u + 160u) / 4u] = 4u; + Check(MaterializeResources(plan, runtime, snapshot, specialization) && + snapshot.images.size() == 2u && snapshot.images.back().dwords[0] == 0x104u, + "shared append materialization reused stale selector contents"); + user_data[2] = 1u; + Check(!MaterializeResources(plan, runtime, snapshot, specialization), + "shared append materialization accepted a range outside its V# bounds"); + user_data[2] = 8u; + for (const auto too_large : {65537u, UINT32_MAX}) { + for (uint32_t list = 0; list < 2u; ++list) { + memory.words[0x800u / 4u] = memory.words[0x804u / 4u] = 0u; + memory.words[0x800u / 4u + list] = too_large; + memory.watched_reads = 0u; + Check(!MaterializeResources(plan, runtime, snapshot, specialization) && memory.watched_reads == 0u, + "inactive consumer hid an oversized producer or wrapped append address"); + } + } + + struct BoundaryMemory { uint32_t keys_read = 0; } boundary; + const auto read_boundary = +[](void *userdata, uint64_t address, std::span words) { + auto &memory = *static_cast(userdata); + std::ranges::fill(words, 0u); + if (words.size() == 1u && (address == 0x40000000u || address == 0x40000004u)) { + words[0] = address == 0x40000000u ? 65536u : 0u; + return true; + } + if (words.size() == 1u && address >= 0x1090u && + (address - 0x1090u) % 160u == 0u && (address - 0x1090u) / 160u < 65536u) { + words[0] = 1u; + ++memory.keys_read; + return true; + } + if (words.size() != 8u || address != 0x50000020u) return false; + words[0] = 0x101u; + words[1] = static_cast( + Libs::Graphics::Prospero::BufferFormat::k32_32_32_32Float) << 20u; + words[3] = Libs::Graphics::DstSel(4, 5, 6, 7) | + (static_cast(Libs::Graphics::Prospero::ImageType::kColor2D) << 28u); + return true; + }; + user_data[2] = 65536u; + user_data[4] = 0x50000000u; + user_data[6] = 0x40000000u; + Check(MaterializeResources(plan, + {.user_data = user_data, .read_memory = read_boundary, .userdata = &boundary, + .read_specialization_memory = read_boundary}, snapshot, specialization) && + boundary.keys_read == 65536u && snapshot.images.size() == 1u && + snapshot.images[0].dwords[0] == 0x101u, + "shared append materialization rejected its exact nonwrapping probe limit"); +} + void TestComputeBufferFill() { struct Options { bool scalar = false; @@ -4131,6 +4410,7 @@ int main() { Run("image descriptor fields", TestImageDescriptorFields); Run("draw-uniform scalar image", TestUniformScalarBufferImage); Run("scalar buffer selector domain", TestScalarBufferSelectorDomain); + Run("shared append selector bounds", TestSharedAppendSelectorBounds); Run("descending image range", TestDescendingImageRange); Run("mixed indirect image samplers", TestMixedIndirectImageSamplers); Run("SRT runtime", TestSrtFlatteningAndRuntimeMemoization);