diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp index 8a541959..a56c339c 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp @@ -366,20 +366,22 @@ bool MaterializeIndirectDescriptor(const ResourcePlan& p } else if (!indirect.selector_first.IsEmpty()) { ShaderBufferResource material; uint32_t first = 0, count = 0; - if (!DecodeBufferDescriptor(material_value, material) || material.Type() != 0u || - material.SwizzleEnabled() || - material.AddTid() || material.OutOfBounds() != 0u || - uint64_t {selector->offset} + 4u > material.Stride() || + if (!DecodeBufferDescriptor(material_value, material) || !clean.Evaluate(indirect.selector_first, first) || - !clean.Evaluate(indirect.key_count, count) || count > MaxIndirectDescriptorProbes || - uint64_t {first} + count > material.NumRecords()) + !clean.Evaluate(indirect.key_count, count) || count > MaxIndirectDescriptorProbes) return false; - if (count != 0u && (uint64_t {first} + count - 1u) * material.Stride() + - selector->offset + 4u > - uint64_t {UINT32_MAX} + 1u) + // SMEM uses its instruction's byte stride; V# stride only supplies its bounds. + const auto step = selector->stride != 0u ? selector->stride : material.Stride(); + if (selector->stride == 0u && + (material.Type() != 0u || material.SwizzleEnabled() || material.AddTid() || + material.OutOfBounds() != 0u || uint64_t {selector->offset} + 4u > step || + uint64_t {first} + count > material.NumRecords())) return false; + const auto end = count != 0u + ? (uint64_t {first} + count - 1u) * step + selector->offset + 4u : 0u; + if (end > uint64_t {UINT32_MAX} + 1u || end > material.GetSize()) + return false; + if (!read_keys(material, uint64_t {first} * step + selector->offset, step, count)) return false; - if (!read_keys(material, uint64_t {first} * material.Stride() + selector->offset, - material.Stride(), count)) return false; } else if (!indirect.selector_mask.IsEmpty()) { uint32_t mask = 0; uint32_t count = 0; diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp index eedc6b77..3c476834 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp @@ -1469,7 +1469,18 @@ private: const auto* step = phi->Arg(initial ^ 1u).Resolve().TryInstruction(); if (!zero.IsImmediate() || zero.GetType() != Type::U32 || zero.U32() != 0u || step == nullptr || step->GetOpcode() != ValueOpcode::IAdd32 || - step->Parent() != phi->PhiBlock(initial ^ 1u)) continue; + step->Parent() == nullptr) continue; + // Structurization can forward the native increment through a synthetic continue block. + const auto* incoming = phi->PhiBlock(initial ^ 1u); + const auto* next = phi->Parent(); + for (size_t edges = 0; incoming != step->Parent() && edges < m_program.blocks.size(); ++edges) { + if (incoming == nullptr || incoming->terminator.kind != CFG::TerminatorKind::Branch || + incoming->terminator.true_block != next || incoming->ImmPredecessors().size() != 1u) + break; + next = incoming; + incoming = incoming->ImmPredecessors()[0]; + } + if (incoming != step->Parent()) continue; uint32_t increment = 0; if ((step->Arg(0).Resolve() == key && ImmediateU32(step->Arg(1), increment) && increment == 1u) || @@ -1507,6 +1518,30 @@ private: }); } + bool MatchScalarMaterialKey(Value key, DescriptorSource::IndirectDescriptor& indirect, + DescriptorSource& material_source) { + const auto* read = key.Resolve().TryInstruction(); + if (read == nullptr || read->GetOpcode() != ValueOpcode::ReadConstBuffer) return false; + uint32_t memory_index = 0; + const auto* memory = ScalarReadMemory(*read, memory_index); + Value index; + uint32_t offset = 0, stride = 0; + if (memory == nullptr || memory->kind != ResourceKind::ScalarBuffer || + memory->offset > INT32_MAX || !MemoryIndexBelongsTo(memory_index, *read) || + !MatchTableOffset(read->Arg(1), index, offset, stride) || + ((offset | stride | memory->offset) & 3u) != 0u || + uint64_t {offset} + memory->offset > UINT32_MAX) return false; + const auto* bound = BoundedLoop(index, read->Parent()); + if (bound == nullptr || bound->GetOpcode() != ValueOpcode::ULessThan32 || + !MakeRuntimeTableSource(*read, material_source)) return false; + indirect.selector.emplace(DescriptorSource::IndirectDescriptor::SelectorRead { + .source = InternSource(material_source), .stride = stride, + .offset = offset + memory->offset}); + indirect.selector_first = Value(0u); + indirect.key_count = bound->Arg(1); + return true; + } + void FoldBoundedLoopSelectors() { for (auto* header: m_program.blocks) { for (auto& induction: *header) { @@ -1742,6 +1777,10 @@ private: } if ((table_offset & 3u) != 0u || (bitscan && table_offset > UINT32_MAX - (32u * 32u - 1u))) return false; + } else if (!plan.table_indexed && + MatchScalarMaterialKey(key, indirect, material_source)) { + // The host enumerates the guarded scalar loop; its key and T# payload stay on the GPU. + plan.retain_reads = true; } else { // A bounded V# supplies the complete image table. Leave every GPU selector // and descriptor read in the shader; the host only translates table bytes. diff --git a/tests/ResourceTrackingTests.cpp b/tests/ResourceTrackingTests.cpp index d82a18d9..06b4315a 100644 --- a/tests/ResourceTrackingTests.cpp +++ b/tests/ResourceTrackingTests.cpp @@ -742,7 +742,9 @@ void TestBoundedComputeImageLoop() { enum class Variant { Bounded, Plain, Nonzero, TrueEdge, WrongGuard, EntryBypass, ExitBypass, GuardBlock, WrongStep, DivergentBound, DivergentKey, Disjunction, WrongPolarity, - IncrementBypass, PreviousBound, Masked, MaskedWrongGuard, MaskResurrection, StatusOverwrite, + IncrementBypass, PreviousBound, Forwarded, ForwardedChain, ForwardedJoin, + ForwardedBranch, ForwardedIncrementBypass, + Masked, MaskedWrongGuard, MaskResurrection, StatusOverwrite, GuardedDiamond, GuardedDiamondBypass }; const auto make_plan = [](Variant variant) { @@ -759,16 +761,21 @@ void TestBoundedComputeImageLoop() { auto *guard = masked ? fixture.AddBlock() : header; auto *increment = masked ? fixture.AddBlock() : latch; auto *final_exit = variant == Variant::PreviousBound ? fixture.AddBlock() : exit; + const bool forwarded = variant >= Variant::Forwarded && variant <= Variant::ForwardedIncrementBypass; + auto *forward = forwarded ? fixture.AddBlock() : increment; + auto *incoming = variant == Variant::ForwardedChain ? fixture.AddBlock() : forward; + auto *detour = variant == Variant::ForwardedJoin ? fixture.AddBlock() : nullptr; entry->AddBranch(header); if (!masked) { header->AddBranch(exit); header->AddBranch(body); body->AddBranch(latch); - latch->AddBranch(header); + latch->AddBranch(forwarded ? forward : header); if (variant == Variant::PreviousBound) latch->AddBranch(final_exit); if (variant == Variant::EntryBypass) entry->AddBranch(body); if (variant == Variant::ExitBypass) exit->AddBranch(body); - if (variant == Variant::IncrementBypass || variant == Variant::PreviousBound) + if (variant == Variant::IncrementBypass || variant == Variant::PreviousBound || + variant == Variant::ForwardedIncrementBypass) exit->AddBranch(latch); fixture.program.blocks[0]->terminator = { .kind = variant == Variant::EntryBypass @@ -789,16 +796,16 @@ void TestBoundedComputeImageLoop() { .kind = variant == Variant::PreviousBound ? CFG::TerminatorKind::ConditionalBranch : CFG::TerminatorKind::Branch, - .true_block = fixture.program.blocks[1u], + .true_block = forwarded ? forward : fixture.program.blocks[1u], .false_block = final_exit}; fixture.program.blocks[4]->terminator = { .kind = final_exit != exit || variant == Variant::ExitBypass || - variant == Variant::IncrementBypass + variant == Variant::IncrementBypass || variant == Variant::ForwardedIncrementBypass ? CFG::TerminatorKind::Branch : CFG::TerminatorKind::Return, .true_block = fixture.program.blocks[final_exit != exit || - variant == Variant::IncrementBypass + variant == Variant::IncrementBypass || variant == Variant::ForwardedIncrementBypass ? 3u : 2u]}; if (final_exit != exit) @@ -822,6 +829,29 @@ void TestBoundedComputeImageLoop() { count}, 0, compare); const auto initial_active = fixture.Emit(ValueOpcode::INotEqual32, {local, Value(0u)}, 0, entry); + if (forwarded) { + forward->AddBranch(incoming == forward ? header : incoming); + forward->terminator = {.kind = CFG::TerminatorKind::Branch, + .true_block = incoming == forward ? header : incoming}; + if (incoming != forward) { + incoming->AddBranch(header); + incoming->terminator = {.kind = CFG::TerminatorKind::Branch, .true_block = header}; + } + if (variant == Variant::ForwardedBranch) { + forward->AddBranch(exit); + forward->condition = initial_active; + forward->terminator = {.kind = CFG::TerminatorKind::ConditionalBranch, + .true_block = header, .false_block = exit}; + } + if (detour != nullptr) { + latch->AddBranch(detour); + latch->condition = initial_active; + latch->terminator = {.kind = CFG::TerminatorKind::ConditionalBranch, + .true_block = forward, .false_block = detour}; + detour->AddBranch(forward); + detour->terminator = {.kind = CFG::TerminatorKind::Branch, .true_block = forward}; + } + } if (masked) { auto *active_phi = diamond ? nullptr : &header->AppendNewInst( ValueOpcode::Phi, {}, static_cast(Type::U1)); @@ -934,7 +964,9 @@ void TestBoundedComputeImageLoop() { {key, Value(variant == Variant::WrongStep ? 2u : 1u)}, 0, increment); phi.AddPhiOperand(entry, variant == Variant::DivergentKey ? local : Value(0u)); - phi.AddPhiOperand(increment, step); + phi.AddPhiOperand(forwarded ? incoming : increment, step); + Check(!forwarded || step.ResolveInstruction()->Parent() != phi.PhiBlock(1), + "forwarded loop fixture did not separate the increment from its Phi incoming edge"); fixture.block = variant == Variant::GuardBlock ? header : body; const auto table = fixture.Address(fixture.UserData(0), fixture.UserData(1)); @@ -975,7 +1007,11 @@ void TestBoundedComputeImageLoop() { make_plan(Variant::TrueEdge); make_plan(Variant::Masked); make_plan(Variant::GuardedDiamond); + make_plan(Variant::Forwarded); + make_plan(Variant::ForwardedChain); for (const auto variant : {Variant::IncrementBypass, Variant::PreviousBound, + Variant::ForwardedJoin, Variant::ForwardedBranch, + Variant::ForwardedIncrementBypass, Variant::MaskedWrongGuard, Variant::MaskResurrection, Variant::StatusOverwrite, Variant::GuardedDiamondBypass}) { diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index 8d0df51a..b4ee2771 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -13857,6 +13857,156 @@ void TestNativeScalarAtomicPayloadStaysOnGpu() { "native resource materialization read the mutable flag or lost its atomic descriptor"); } +void TestBoundedScalarMaterialImageKeys() { + using namespace ShaderRecompiler::IR; + // SAROS 9fba2edffc549531: min(header count,64), scalar rows of 160 bytes, + // and separate float/uint image fields at 0x7c/0x94 in the same image heap. + std::vector shader { + 0xf4080106u, 0xfa0000a0u, 0xf4080806u, 0xfa0000b0u, + 0xbf8cc07fu, 0xf4201550u, 0xfa000024u, 0xbf8cc07fu, + EncodeSop2(0x07, 85, 85, 0xc0), EncodeSMovB32(86, 128), + EncodeSMovB32(87, 126), + }; + const auto loop = shader.size(); + shader.insert(shader.end(), {0x7da2aaf9u, 0x86860056u, EncodeSopp(0x08), + 0x931cff56u, 0x000000a0u, 0xf42006c2u, 0x38000094u, + 0xf4200682u, 0x3800007cu, 0xbf8cc07fu, 0x8f58851bu, 0x8f3c851au, + 0xf42c1304u, 0xb0000000u, 0xf42c0d04u, 0x78000000u, 0xbf8cc07fu, + 0xf0000110u, 0x00130106u, 0xf0000110u, 0x000d0206u, + EncodeVop1(0x01, 3, 78), // Preserve a T# dimension word as native scalar payload. + EncodeMubuf0(0x1c, 0, false), EncodeMubuf1(1, 0, 0), + EncodeMubuf0(0x1c, 4, false), EncodeMubuf1(2, 0, 0), + EncodeMubuf0(0x1c, 8, false), EncodeMubuf1(3, 0, 0), + EncodeSop2(0x02, 86, 86, 129), EncodeSMovB32(126, 87), + EncodeSopp(0x02)}); + shader.back() = EncodeSopp(0x02, static_cast(loop - shader.size())); + shader[loop + 2u] = EncodeSopp(0x08, shader.size() - loop - 3u); + shader.push_back(EncodeSopp(0x01)); + std::array user_data{}; + user_data[8] = 0x100000u; + user_data[9] = 32u << 16u; + user_data[10] = 198656u; + user_data[11] = 0x5204u; + user_data[12] = 0x1000u; + auto options = MakeCompileOptions(ShaderType::Compute); + options.user_data = user_data; + auto translated = ShaderRecompiler::TranslateProgram(shader, options); + auto plan = ExtractResourcePlan(translated.program); + Check(plan.info.images.size() == 2 && plan.capture_specialization_reads, + "scalar material fields lost their distinct image roots or clean read capture"); + for (const auto &image : plan.info.images) { + const auto &source = plan.descriptor_sources[image.source].indirect_descriptor; + Check(source && source->selector && source->selector->stride == 160u && + (source->selector->offset == 0x7cu || source->selector->offset == 0x94u) && + source->table_stride == 32u && !source->table_scalar && + source->key_count.ResolveInstruction()->GetOpcode() == ValueOpcode::UMin32, + "scalar material image selection fell back to the whole heap"); + } + Check(std::ranges::any_of(translated.program.blocks, [&](const auto *block) { + return std::ranges::any_of(*block, [&](const auto &inst) { + if (inst.GetOpcode() != ValueOpcode::ReadConstBuffer) return false; + const auto &memory = translated.program.memory_info[inst.template Flags().index]; + return memory.component_count == 8u && memory.component_index == 2u && + !memory.planning_only && inst.HasUses(); + }); + }), "scalar material projection removed a live T# dimension payload"); + struct Memory { + uint32_t count = 2; + uint32_t stride = 160; + uint32_t records = 409; + uint64_t unavailable = 0; + bool clean = true; + bool mixed = false; + bool same_key = false; + uint32_t keys = 0; + uint32_t images = 0; + } memory; + const auto read = +[](void *data, uint64_t address, std::span words) { + auto &m = *static_cast(data); + if (!m.clean || address == m.unavailable) return false; + if (words.size() == 1 && address >= 0x10a0u && address < 0x10c0u) { + const uint32_t descriptors[] = {0x2000u, m.stride << 16u, m.records, 0x5204u, + 0x4000u, 16u << 16u, 8u, 0x4dfacu}; + words[0] = descriptors[(address - 0x10a0u) / 4u]; + return true; + } + if (address == 0x4024u && words.size() == 1) { words[0] = m.count; return true; } + for (const uint32_t row : {0u, 1u}) { + for (const uint32_t field : {0x7cu, 0x94u}) { + if (address != 0x2000u + row * 160u + field || words.size() != 1) continue; + ++m.keys; + words[0] = (row == 0 || m.same_key ? 5582u : 5567u) + (field == 0x94u ? 6u : 0u); + return true; + } + } + for (const uint32_t key : {5567u, 5582u, 5573u, 5588u}) { + if (address != 0x100000u + key * 32u || words.size() != 8) continue; + ++m.images; + std::ranges::fill(words, 0u); + words[0] = key; + const bool integer = key == 5573u || key == 5588u; + words[1] = static_cast(integer && !(m.mixed && key == 5573u) + ? Prospero::BufferFormat::k32UInt : Prospero::BufferFormat::k32Float) << 20u; + words[3] = DstSel(4, 5, 6, 7) | + (static_cast(Prospero::ImageType::kColor3D) << 28u); + return true; + } + return false; + }; + const SrtRuntime runtime{.user_data = user_data, .userdata = &memory, + .read_specialization_memory = read}; + ResourceSnapshot snapshot; + ResourceSpecialization specialization; + for (const uint32_t stride : {160u, 16u}) { + memory = {.stride = stride, .records = stride == 160u ? 409u : 20u}; + Check(MaterializeResources(plan, runtime, snapshot, specialization) && + memory.keys == 4 && memory.images == 4 && snapshot.images.size() == 4, + "scalar material selection used V# stride as its instruction stride or scanned unused keys"); + for (uint32_t root = 0; root < plan.info.images.size(); ++root) { + const auto &selector = *plan.descriptor_sources[plan.info.images[root].source].indirect_descriptor->selector; + const uint32_t first = selector.offset == 0x94u ? 5573u : 5567u; + const auto offset = specialization.images[root].indirect_mapping_offset; + Check(snapshot.flattened_srt[offset] == 2u && + snapshot.flattened_srt[offset + 1u] == first && + snapshot.flattened_srt[offset + 3u] == first + 15u, + "scalar material image mapping changed descriptor-index keys into byte offsets"); + } + } + for (const bool same_key : {false, true}) { + memory = {.count = same_key ? 2u : 1u, .same_key = same_key}; + Check(MaterializeResources(plan, runtime, snapshot, specialization) && + memory.keys == (same_key ? 4u : 2u) && memory.images == 2 && + snapshot.images.size() == 2 && + std::ranges::all_of(specialization.images, [](const auto &image) { + return image.indirect_root == ImageResource::NoIndirectImage; + }), "single-candidate scalar material loop retained an indirect image mapping"); + } + memory = {.count = 0}; + Check(MaterializeResources(plan, runtime, snapshot, specialization) && + memory.keys == 0 && memory.images == 0 && snapshot.images.size() == 2, + "empty scalar material loop retained image probes"); + memory = {.clean = false}; + Check(!MaterializeResources(plan, runtime, snapshot, specialization), + "dirty scalar material proof was accepted"); + memory = {.unavailable = 0x2094u}; + Check(!MaterializeResources(plan, runtime, snapshot, specialization), + "unavailable selected material key was accepted"); + memory = {.stride = 1, .records = 311}; + Check(!MaterializeResources(plan, runtime, snapshot, specialization) && memory.keys == 0, + "scalar material byte range escaped V# bounds"); + memory = {.mixed = true}; + Check(!MaterializeResources(plan, runtime, snapshot, specialization), + "selected material fields bypassed image compatibility"); + memory = {}; + Check(MaterializeResources(plan, runtime, snapshot, specialization), + "native scalar material image selection did not recover after rejected candidates"); + ApplyResourceSpecialization(translated.program, specialization); + Check(translated.program.info.images.size() == 4 && + translated.program.info.images[0].indirect_resources.size() == 2 && + translated.program.info.images[1].indirect_resources.size() == 2, + "native image reads lost their distinct material field dispatches"); +} + void TestTypedDescriptorRealCarryAndScalarLoads() { const uint32_t carry_shader[] = { EncodeSop1(0x1f, 0, 0), // s_getpc_b64 s[0:1] @@ -15331,6 +15481,7 @@ int main() { TestUniformSelectedWritableDescriptor(); TestUniformSelectedDescriptorLoadAddress(); TestNativeScalarAtomicPayloadStaysOnGpu(); + TestBoundedScalarMaterialImageKeys(); TestImmutableDescriptorPredicate(); TestTypedDescriptorRealCarryAndScalarLoads(); TestSrtWalkerRealSmemTranslation();