From d08a6cef64e813ba24564cbb51928e306424c199 Mon Sep 17 00:00:00 2001 From: nmzik Date: Tue, 13 Oct 2026 06:07:49 +0200 Subject: [PATCH] shader: specialize guarded packed material selectors --- src/graphics/shader/recompiler/ir/ShaderIR.h | 3 + .../ir/passes/ResourceMaterialization.cpp | 31 ++++--- .../recompiler/ir/passes/ResourceTracking.cpp | 88 ++++++++++++------ tests/ResourceTrackingTests.cpp | 90 +++++++++++++++++-- 4 files changed, 165 insertions(+), 47 deletions(-) diff --git a/src/graphics/shader/recompiler/ir/ShaderIR.h b/src/graphics/shader/recompiler/ir/ShaderIR.h index 4d80b530..60996df8 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIR.h +++ b/src/graphics/shader/recompiler/ir/ShaderIR.h @@ -514,6 +514,9 @@ struct DescriptorSource { uint32_t stride = 0; uint32_t offset = 0; uint32_t maximum = UINT32_MAX; + uint32_t bit_offset = 0; + uint32_t bit_mask = UINT32_MAX; + bool global = false; // Four DWORD bitmap words per record; set-bit ranks subtract from the last key. bool bitset = false; std::optional rows; diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp index ff30c3aa..82303d21 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp @@ -205,16 +205,14 @@ void MarkCleanReads(const std::unordered_set& planned_reads, } } -bool ReadScalarTable(uint64_t base, uint64_t size, uint64_t dynamic_offset, - const SrtRuntime& runtime, std::span words) { - const auto offset = dynamic_offset & ~uint64_t {3}; +bool ReadTable(uint64_t base, uint64_t size, uint64_t offset, + const SrtRuntime& runtime, std::span words) { const auto count = std::min(words.size(), offset < size ? (size - offset) / 4u : 0u); std::ranges::fill(words.subspan(count), 0u); if (count == 0u) { return true; } - base &= AddressMask & ~uint64_t {3}; - if (offset > AddressMask - base) { + if (base > AddressMask || offset > AddressMask - base) { return false; } const auto address = base + offset; @@ -224,6 +222,11 @@ bool ReadScalarTable(uint64_t base, uint64_t size, uint64_t dynamic_offset, runtime.read_specialization_memory(runtime.userdata, address, prefix); } +bool ReadScalarTable(uint64_t base, uint64_t size, uint64_t offset, + const SrtRuntime& runtime, std::span words) { + return ReadTable(base & AddressMask & ~uint64_t {3}, size, offset & ~uint64_t {3}, runtime, words); +} + bool NormalizeIndirectStoreBuffer(DescriptorValue& value, bool indexed, bool formatted) { ShaderBufferResource descriptor; if (!DecodeBufferDescriptor(value, descriptor)) return false; @@ -282,14 +285,19 @@ bool MaterializeIndirectDescriptor(const ResourcePlan& p } return true; }; + const auto read_selector = [&](uint64_t base, uint64_t size, uint64_t offset, + std::span words) { + return selector->global ? ReadTable(base, size, offset, runtime, words) + : ReadScalarTable(base, size, offset, runtime, words); + }; const auto read_keys = [&](uint64_t base, uint64_t size, uint64_t first, uint64_t step, uint64_t count) { keys.resize(count); if (step == 4u) { - return ReadScalarTable(base, size, first, runtime, keys); + return read_selector(base, size, first, keys); } for (auto& key: keys) { - if (!ReadScalarTable(base, size, first, runtime, {&key, 1})) + if (!read_selector(base, size, first, {&key, 1})) return false; first += step; } @@ -516,8 +524,8 @@ bool MaterializeIndirectDescriptor(const ResourcePlan& p static_cast(index) * selector->stride; if (offset > UINT32_MAX) return false; uint32_t key = 0; - if (!ReadScalarTable(material_base, UINT64_MAX, static_cast(offset), - runtime, {&key, 1})) return false; + if (!read_selector(material_base, UINT64_MAX, static_cast(offset), + {&key, 1})) return false; keys.push_back(key); mask &= mask - 1u; } @@ -538,8 +546,9 @@ bool MaterializeIndirectDescriptor(const ResourcePlan& p if (count < domain_count) keys.push_back(0u); // An out-of-bounds scalar read returns zero. } if (selector != nullptr && selector->byte_ranges.empty()) { - if (selector->maximum != UINT32_MAX) - for (auto& key: keys) key = std::min(key, selector->maximum); + if (selector->bit_offset != 0u || selector->bit_mask != UINT32_MAX || selector->maximum != UINT32_MAX) + for (auto& key: keys) + key = std::min((key >> selector->bit_offset) & selector->bit_mask, selector->maximum); std::ranges::sort(keys); keys.erase(std::unique(keys.begin(), keys.end()), keys.end()); } diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp index 1ef1196e..397952e0 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp @@ -1195,8 +1195,11 @@ private: ImpliesNonzero(guard, test->Arg(arg ^ 1u))) return true; } const auto* inst = guard.TryInstruction(); - return inst != nullptr && inst->GetOpcode() == ValueOpcode::LogicalAnd && - (Implies(inst->Arg(0), required) || Implies(inst->Arg(1), required)); + if (inst == nullptr) return false; + if (inst->GetOpcode() == ValueOpcode::LogicalAnd) + return Implies(inst->Arg(0), required) || Implies(inst->Arg(1), required); + return inst->GetOpcode() == ValueOpcode::LogicalOr && + Implies(inst->Arg(0), required) && Implies(inst->Arg(1), required); } Value EqualLocalValue(Value guard, const auto& matches) const { @@ -1441,39 +1444,74 @@ private: return selected; } + const Inst* SelectorReadValue(Value value, Value active, + DescriptorSource::IndirectDescriptor::SelectorRead& selector) const { + const auto* read = ActiveValue(value, active).TryInstruction(); + if (read != nullptr && read->GetOpcode() == ValueOpcode::UMin32) { + if (ImmediateU32(read->Arg(1), selector.maximum)) value = read->Arg(0); + else if (ImmediateU32(read->Arg(0), selector.maximum)) value = read->Arg(1); + else return nullptr; + read = ActiveValue(value, active).TryInstruction(); + } + if (read != nullptr && read->GetOpcode() == ValueOpcode::BitFieldUExtract) { + uint32_t width; + if (!ImmediateU32(read->Arg(1), selector.bit_offset) || + !ImmediateU32(read->Arg(2), width) || selector.bit_offset >= 32u || + width == 0u || width > 32u - selector.bit_offset) return nullptr; + selector.bit_mask = width == 32u ? UINT32_MAX : (1u << width) - 1u; + read = ActiveValue(read->Arg(0), active).TryInstruction(); + } + return read; + } + bool MatchUniformizedMaterialKey(Value key, const Inst& image, DescriptorSource::IndirectDescriptor& indirect, DescriptorSource& material_source) { const auto* selected = UniformizedMaterialValue(key, image); if (selected == nullptr) return false; const auto active = selected->Arg(0).Resolve(); - const auto* read = selected->Arg(1).Resolve().TryInstruction(); + DescriptorSource::IndirectDescriptor::SelectorRead selector; + const auto* read = SelectorReadValue(selected->Arg(1), active, selector); if (read == nullptr || read->GetOpcode() != ValueOpcode::LoadAddressU32 || - read->NumArgs() != 4u || !EquivalentValue(m_program, read->Arg(3), active)) + read->NumArgs() != 4u || !Implies(active, read->Arg(3))) return false; uint32_t high = 1u; if (!ImmediateU32(read->Arg(2), high) || high != 0u) return false; const auto memory_index = read->Flags().index; if (memory_index >= m_program.memory_info.size()) return false; const auto& memory = m_program.memory_info[memory_index]; - if (memory.kind != ResourceKind::Global || memory.data_bits != 32u || + if (memory.kind != ResourceKind::Global || memory.address_is_full || memory.offset > INT32_MAX || + memory.data_bits != 32u || memory.data_dwords != 1u || !MemoryIndexBelongsTo(memory_index, *read)) return false; const auto* material_handle = read->Arg(0).Resolve().TryInstruction(); if (material_handle == nullptr || material_handle->GetOpcode() != ValueOpcode::GetAddressResource || !MakeRuntimeTableSource(*read, material_source)) return false; - AffineOffset offset; - if (!MatchAffineOffset(read->Arg(1), active, offset) || offset.index.IsEmpty() || - offset.stride == 0u || offset.offset + memory.offset > UINT32_MAX || - offset.offset + memory.offset + 31u * offset.stride + 4u > - static_cast(UINT32_MAX) + 1u) return false; - const auto mask = BoundedSetBitMask(offset.index, active); - if (mask.IsEmpty()) return false; - indirect.selector.emplace(DescriptorSource::IndirectDescriptor::SelectorRead{.source = InternSource(material_source), - .stride = static_cast(offset.stride), - .offset = static_cast(offset.offset + memory.offset)}); - indirect.key_count = Value(32u); - indirect.selector_mask = mask; + const auto bits = PossibleU32Bits(ActiveValue(read->Arg(1), active)); + const auto shift = std::countr_zero(bits); + const auto count = bits == 0u ? 0u : uint64_t {bits >> shift} + 1u; + if (shift >= 2u && count != 0u && count <= MaxIndirectDescriptorProbes && + std::has_single_bit(count) && uint64_t {bits} + memory.offset + 4u <= UINT32_MAX + 1ull) { + selector.stride = 1u << shift; + selector.offset = memory.offset; + indirect.key_count = Value(static_cast(count)); + indirect.selector_first = Value(0u); + } else { + AffineOffset offset; + if (!MatchAffineOffset(read->Arg(1), active, offset) || offset.index.IsEmpty() || + offset.stride == 0u || offset.offset + memory.offset > UINT32_MAX || + offset.offset + memory.offset + 31u * offset.stride + 4u > + static_cast(UINT32_MAX) + 1u) return false; + const auto mask = BoundedSetBitMask(offset.index, active); + if (mask.IsEmpty()) return false; + selector.stride = static_cast(offset.stride); + selector.offset = static_cast(offset.offset + memory.offset); + indirect.key_count = Value(32u); + indirect.selector_mask = mask; + } + selector.source = InternSource(material_source); + selector.global = true; + indirect.selector = std::move(selector); return true; } @@ -2043,13 +2081,8 @@ private: bool MatchScalarMaterialKey(Value key, DescriptorSource::IndirectDescriptor& indirect, DescriptorSource& material_source) { - const auto* read = key.Resolve().TryInstruction(); - uint32_t maximum = UINT32_MAX; - if (read != nullptr && read->GetOpcode() == ValueOpcode::UMin32) { - if (ImmediateU32(read->Arg(1), maximum)) read = read->Arg(0).Resolve().TryInstruction(); - else if (ImmediateU32(read->Arg(0), maximum)) read = read->Arg(1).Resolve().TryInstruction(); - else return false; - } + DescriptorSource::IndirectDescriptor::SelectorRead selector; + const auto* read = SelectorReadValue(key, Value(true), selector); if (read == nullptr) return false; uint32_t memory_index = 0; const auto* memory = ScalarReadMemory(*read, memory_index); @@ -2105,9 +2138,10 @@ private: } } if (!MakeRuntimeTableSource(*read, material_source)) return false; - indirect.selector.emplace(DescriptorSource::IndirectDescriptor::SelectorRead { - .source = InternSource(material_source), .stride = stride, - .offset = offset + (memory->offset & ~3u), .maximum = maximum}); + selector.source = InternSource(material_source); + selector.stride = stride; + selector.offset = offset + (memory->offset & ~3u); + indirect.selector = std::move(selector); indirect.key_count = count; if (rows) indirect.selector->rows.emplace( DescriptorSource::IndirectDescriptor::SelectorRead::CountedRows { diff --git a/tests/ResourceTrackingTests.cpp b/tests/ResourceTrackingTests.cpp index 8bc0c891..b415dd8d 100644 --- a/tests/ResourceTrackingTests.cpp +++ b/tests/ResourceTrackingTests.cpp @@ -189,12 +189,11 @@ bool ReadLinearTestMemory(void *userdata, uint64_t address, std::span if (memory == nullptr || address < memory->base || values.size_bytes() > memory->words.size() * sizeof(uint32_t) || address - memory->base > memory->words.size() * sizeof(uint32_t) - values.size_bytes() || - (address & 3u) != 0u || (memory->fail_address >= address && memory->fail_address - address < values.size_bytes())) { return false; } - std::copy_n(memory->words.begin() + (address - memory->base) / sizeof(uint32_t), - values.size(), values.begin()); + std::memcpy(values.data(), reinterpret_cast(memory->words.data()) + + (address - memory->base), values.size_bytes()); ++memory->reads; if (values.size() == 8u) ++memory->descriptor_reads; if (memory->watched_address >= address && @@ -1866,12 +1865,15 @@ void TestUniformizedMaterialImageKeys() { enum class Variant { Valid, Plain, WrongUpdate, WrongEquality, WrongExit, WrongCarry, WrongBackedge, AndNot, Subset, FirstLane, FirstLaneDirect, FirstLaneAndNot, EmptyFirstEntry, WrongFirstBackedge, - WideningFirstEntry, WideningFirstBackedge, WrongFirstLoadMask + WideningFirstEntry, WideningFirstBackedge, WrongFirstLoadMask, + Packed, PackedUnalignedOffset, PackedNegativeOffset, PackedFullAddress, + PackedWrongGuard, PackedSparseOffset, PackedWrongField }; const auto make_plan = [](Variant variant) { Fixture fixture; fixture.program.wave_size = 64u; const bool first_lane = variant >= Variant::FirstLane; + const bool packed = variant >= Variant::Packed; const bool first_lane_loop = first_lane && variant != Variant::FirstLaneDirect; const uint32_t sample_entry_id = first_lane_loop ? 10u : 9u; const auto branch = [&](Value predicate, CFG::BranchCondition kind, Block *block) { @@ -2002,8 +2004,17 @@ void TestUniformizedMaterialImageKeys() { const auto index = Value(&index_phi); const auto below = fixture.Emit( ValueOpcode::SGreaterThan32, {Value(32u), index}, 0, merge); - const auto material_guard = fixture.Emit( + auto material_guard = fixture.Emit( ValueOpcode::LogicalAnd, {active_on_entry, below}, 0, merge); + const auto packed_load_guard = material_guard; + if (packed) { + const auto left = fixture.Emit(ValueOpcode::LogicalAnd, + {material_guard, fixture.Emit(ValueOpcode::IEqual32, {arbitrary, Value(0u)}, 0, merge)}, 0, merge); + const auto right = fixture.Emit(ValueOpcode::LogicalAnd, + {variant == Variant::PackedWrongGuard ? Value(true) : material_guard, + fixture.Emit(ValueOpcode::IEqual32, {arbitrary, Value(1u)}, 0, merge)}, 0, merge); + material_guard = fixture.Emit(ValueOpcode::LogicalOr, {left, right}, 0, merge); + } fixture.program.blocks[5]->terminator = { .kind = CFG::TerminatorKind::Branch, .true_block = fixture.program.blocks[6u]}; @@ -2015,24 +2026,36 @@ void TestUniformizedMaterialImageKeys() { ValueOpcode::ShiftLeftLogical32, {selected_scale, Value(3u)}, 0, choose); const auto times_nine = fixture.Emit( ValueOpcode::IAdd32, {times_eight, selected_scale}, 0, choose); - const auto material_offset = fixture.Emit( + auto material_offset = fixture.Emit( ValueOpcode::SelectU32, {material_guard, fixture.Emit(ValueOpcode::IAdd32, {times_nine, Value(0xc00u)}, 0, choose), times_nine}, 0, choose); + if (packed) material_offset = fixture.Emit(ValueOpcode::BitwiseAnd32, + {Value(variant == Variant::PackedSparseOffset ? 0xd0u : 0xf0u), scaled}, 0, choose); const auto base = fixture.Address(fixture.UserData(0u), fixture.UserData(1u)); MemoryInfo material_memory; material_memory.kind = ResourceKind::Global; + if (packed) material_memory.offset = variant == Variant::PackedUnalignedOffset ? 9u : 8u; + if (variant == Variant::PackedNegativeOffset) material_memory.offset = static_cast(-4096); + material_memory.address_is_full = variant == Variant::PackedFullAddress; const auto load_mask = variant == Variant::WrongFirstLoadMask ? fixture.Emit(ValueOpcode::LogicalNot, {material_guard}, 0, choose) - : material_guard; + : packed ? packed_load_guard : material_guard; const auto loaded = fixture.Emit( ValueOpcode::LoadAddressU32, {base, material_offset, Value(0u), load_mask}, fixture.AddMemory(material_memory, 0x1a88u), choose); + auto projected = loaded; + if (packed) { + projected = fixture.Emit(ValueOpcode::SelectU32, {load_mask, loaded, arbitrary}, 0, choose); + projected = fixture.Emit(ValueOpcode::BitFieldUExtract, + {projected, Value(variant == Variant::PackedWrongField ? 17u : 16u), Value(16u)}, 0, choose); + projected = fixture.Emit(ValueOpcode::UMin32, {Value(16383u), projected}, 0, choose); + } const auto local_key = fixture.Emit( - ValueOpcode::SelectU32, {material_guard, loaded, arbitrary}, 0, choose); + ValueOpcode::SelectU32, {material_guard, projected, arbitrary}, 0, choose); Inst *sample_active_phi = nullptr; auto sample_active = material_guard; if (first_lane) { @@ -2148,7 +2171,15 @@ void TestUniformizedMaterialImageKeys() { fixture.PlanAndTrack(); const auto source = fixture.program.info.images[0].source; const auto &indirect = fixture.program.descriptor_sources[source].indirect_descriptor; - Check(indirect && indirect->selector->stride == 0x90u && + if (packed) + Check(indirect && indirect->selector && indirect->selector->stride == 16u && + indirect->selector->offset == (variant == Variant::PackedUnalignedOffset ? 9u : 8u) && + indirect->selector->global && indirect->selector->bit_offset == 16u && + indirect->selector->bit_mask == 0xffffu && indirect->selector->maximum == 16383u && + indirect->key_count == Value(16u) && indirect->selector_first == Value(0u) && + indirect->selector_mask.IsEmpty(), + "packed uniformized selector lost its bounded record domain or field projection"); + else Check(indirect && indirect->selector->stride == 0x90u && indirect->selector->offset == 0xc00u && indirect->table_offset == 0x20e0u && !indirect->selector_mask.IsEmpty() && @@ -2239,6 +2270,47 @@ void TestUniformizedMaterialImageKeys() { CheckFatal([&] { make_plan(variant); }, "not a valid runtime value", "first-lane material key escaped its active load mask"); } + + const auto packed = make_plan(Variant::Packed); + memory.words.resize(0x84000u / 4u); + for (uint32_t record = 0; record < 16u; ++record) + memory.words[(record * 16u + 8u) / 4u] = (7u << 16u) | 0xdead; + memory.words[(16u + 8u) / 4u] = UINT32_MAX; + memory.words[(32u + 8u) / 4u] = (4096u << 16u) | 0xbeef; + set_descriptor(base + 0x20e0u + 16383u * 32u, 0x600u); + memory.fail_address = base + 16u * 16u + 8u; + memory.reads = memory.descriptor_reads = 0u; + Check(MaterializeResources(packed, runtime, snapshot, specialization) && + memory.reads == 19u && memory.descriptor_reads == 3u && snapshot.images.size() == 3u && + snapshot.images[0].dwords[0] == 0x200u && snapshot.images[1].dwords[0] == 0x400u && + snapshot.images[2].dwords[0] == 0x600u, + "packed selector did not extract before clamping or read outside its sixteen records"); + memory.fail_address = base + 15u * 16u + 8u; + Check(!MaterializeResources(packed, runtime, snapshot, specialization), + "packed selector accepted an unreadable final record"); + memory.fail_address = UINT64_MAX; + const auto unaligned = make_plan(Variant::PackedUnalignedOffset); + for (const auto [base_offset, immediate] : {std::pair{1u, 8u}, {0u, 9u}, {3u, 9u}}) { + user_data[0] = base + base_offset; + for (uint32_t record = 0; record < 16u; ++record) { + const uint32_t word = ((record == 1u ? 16383u : record == 2u ? 4096u : 7u) << 16u) | 0xdead; + std::memcpy(reinterpret_cast(memory.words.data()) + + base_offset + immediate + record * 16u, &word, sizeof(word)); + } + Check(MaterializeResources(immediate == 8u ? packed : unaligned, runtime, snapshot, specialization) && + snapshot.images.size() == 3u && snapshot.images[0].dwords[0] == 0x200u && + snapshot.images[1].dwords[0] == 0x400u && snapshot.images[2].dwords[0] == 0x600u && + std::ranges::find(snapshot.specialization_reads, + std::pair{base + base_offset + immediate, 4u}) != + snapshot.specialization_reads.end(), + "global selector did not preserve exact base-plus-offset while scalar descriptor reads aligned the base"); + } + user_data[0] = base; + for (const auto variant : {Variant::PackedNegativeOffset, Variant::PackedFullAddress, + Variant::PackedWrongGuard, Variant::PackedSparseOffset, + Variant::PackedWrongField}) + CheckFatal([&] { make_plan(variant); }, "not a valid runtime value", + "packed selector accepted an unproved OR guard, offset domain, or field extraction"); } void TestImageDescriptorFields() {