diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp index 33044114..09aaee92 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp @@ -849,7 +849,7 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) { return; } const auto key = ctx.Def(handle->Arg(0)); - if (state.flattened_srt_variable == 0 || image.indirect_search_iterations == 0u || + if (state.flattened_srt_variable == 0 || image.indirect_resources.size() < 2u) { ctx.Fail(inst, "has no indirect image runtime mapping"); return; diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp index ad3ea70f..e6d9e122 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp @@ -44,6 +44,16 @@ uint32_t EmitIndirectResourceIndex(EmitterState& state, uint32_t key, uint32_t m }; const auto mapping = ConstantU32(state, mapping_offset); const auto count = LoadMapping(mapping); + if (search_iterations == 0u) { + const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), key, + ConstantU32(state, 2u)); + const auto valid = Binary(state, spv::OpULessThan, TypeBool(state), index, count); + const auto safe = Select(state, TypeU32(state), valid, index, ConstantU32(state, 0u)); + const auto entry = Binary(state, spv::OpIAdd, TypeU32(state), safe, + ConstantU32(state, mapping_offset + 1u)); + return Select(state, TypeU32(state), valid, LoadMapping(entry), + ConstantU32(state, default_resource)); + } const auto entry_at = [&](uint32_t index) { return Binary(state, spv::OpIAdd, TypeU32(state), mapping, Binary(state, spv::OpIAdd, TypeU32(state), diff --git a/src/graphics/shader/recompiler/ir/ShaderIR.h b/src/graphics/shader/recompiler/ir/ShaderIR.h index dd03c88f..3629d52a 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIR.h +++ b/src/graphics/shader/recompiler/ir/ShaderIR.h @@ -489,17 +489,16 @@ struct DescriptorSource { uint32_t source = UINT32_MAX; uint32_t stride = 0; uint32_t offset = 0; - uint32_t immediate = 0; - uint32_t shift = 0; - uint32_t bits = UINT32_MAX; bool operator==(const SelectorRead& other) const = default; }; - std::vector selectors; + std::optional selector; uint32_t table_source = 0; uint32_t table_offset = 0; uint32_t table_immediate = 0; uint32_t table_stride = 0; + uint32_t table_record_bytes = 0; + bool table_scalar = false; uint32_t workgroup_axis = UINT32_MAX; Value key_count; Value selector_first; diff --git a/src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp b/src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp index a9067dbd..226d5260 100644 --- a/src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp +++ b/src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp @@ -101,7 +101,7 @@ SharedMemoryResources CollectMemoryResources(const Program& program, std::vector bool UsesFlattenedSrt(const Program& program) { const auto uses_mapping = [](const auto& resource) { - return resource.indirect_search_iterations != 0u; + return resource.indirect_root != UINT32_MAX; }; return std::ranges::any_of(program.blocks, [](const Block* block) { return std::ranges::any_of(*block, [](const Inst& inst) { diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp index 4e0f38a0..1860c6b5 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp @@ -249,8 +249,24 @@ bool MaterializeIndirectDescriptor(const ResourcePlan& p uint32_t maximum_resources, Normalize&& normalize) { const auto descriptor_bytes = dword_count * sizeof(uint32_t); const auto& sources = indirect.sources; - const auto* selector = indirect.selectors.empty() ? nullptr : &indirect.selectors.front(); + const auto* selector = indirect.selector ? &*indirect.selector : nullptr; auto& keys = program.material_keys; + const auto children_begin = descriptors.size(); + const auto root_resource = specializations[resource_index]; + const auto intern_candidate = [&](const DescriptorValue& candidate, uint32_t& ordinal) { + ordinal = 0u; + if (candidate == descriptors[resource_index]) return true; + const auto found = std::find(descriptors.begin() + children_begin, descriptors.end(), candidate); + ordinal = static_cast(found - descriptors.begin() - children_begin + 1u); + if (found == descriptors.end()) { + if (descriptors.size() >= maximum_resources) return false; + descriptors.push_back(candidate); + auto child = root_resource; + child.indirect_root = resource_index; + specializations.push_back(child); + } + return true; + }; const auto read_keys = [&](const ShaderBufferResource& material, uint64_t first, uint64_t step, uint64_t count) { keys.resize(count); @@ -284,6 +300,48 @@ bool MaterializeIndirectDescriptor(const ResourcePlan& p } else { return false; } + if (selector == nullptr && table_value.dword_count == 4u && + indirect.workgroup_axis == UINT32_MAX) { + if (!indirect.table_scalar && table.Format() == Prospero::BufferFormat::kInvalid) { + descriptors[resource_index] = {.dword_count = dword_count}; + return true; + } + if (table_size > UINT32_MAX || + (indirect.table_record_bytes != 0u && + (table.Type() != 0u || table.SwizzleEnabled() || table.AddTid() || + table.OutOfBounds() != 0u || (table_base & 3u) != 0u || + (table.Stride() & 3u) != 0u || table.Stride() < indirect.table_record_bytes))) + return false; + // Descriptor bytes are refreshed together. The GPU selects the DWORD address; + // no material selector or unused table field is interpreted on the CPU. + keys.resize(table_size / 4u); + if (!ReadScalarTable(table_base, table_size, 0u, runtime, keys)) return false; + const auto mapping_offset = snapshot.flattened_srt.size(); + descriptors[resource_index] = {.dword_count = dword_count}; + snapshot.flattened_srt.resize(mapping_offset + 2u, 0u); + for (uint32_t word = 0; word < keys.size(); ++word) { + DescriptorValue candidate {.dword_count = dword_count}; + std::copy_n(keys.begin() + word, std::min(dword_count, keys.size() - word), + candidate.dwords.begin()); + if (!normalize(candidate)) return false; + uint32_t ordinal; + if (!intern_candidate(candidate, ordinal)) return false; + if (ordinal == 0u) continue; + snapshot.flattened_srt.resize(mapping_offset + 2u + word, 0u); + snapshot.flattened_srt[mapping_offset + 1u + word] = ordinal; + } + if (descriptors.size() == children_begin) { + snapshot.flattened_srt.resize(mapping_offset); + } else { + snapshot.flattened_srt[mapping_offset] = + static_cast(snapshot.flattened_srt.size() - mapping_offset - 1u); + auto& root = specializations[resource_index]; + root.indirect_root = resource_index; + root.indirect_mapping_offset = static_cast(mapping_offset); + root.indirect_search_iterations = 0u; + } + return true; + } if (selector == nullptr) { uint32_t key_count = 0; if (indirect.workgroup_axis != UINT32_MAX) { @@ -307,8 +365,6 @@ bool MaterializeIndirectDescriptor(const ResourcePlan& p ShaderBufferResource material; uint32_t first = 0, count = 0; if (!DecodeBufferDescriptor(material_value, material) || material.Type() != 0u || - ((selector->shift != 0u || selector->bits != UINT32_MAX) && - (material.Base48() & 3u) != 0u) || material.SwizzleEnabled() || material.AddTid() || material.OutOfBounds() != 0u || uint64_t {selector->offset} + 4u > material.Stride() || @@ -345,53 +401,12 @@ bool MaterializeIndirectDescriptor(const ResourcePlan& p keys.push_back(key); mask &= mask - 1u; } - } else { - ShaderBufferResource material; - if (!DecodeBufferDescriptor(material_value, material) || table_value.dword_count != 4u) { - return false; - } - // The first aligned offset includes the immediate added after shader U32 arithmetic. - const auto step = std::max(4u, - std::gcd(selector->stride, uint64_t {1} << 32u)); - const uint64_t first = (selector->offset % step & ~uint64_t {3}) + - (selector->immediate & ~3u); - const auto size = material.GetSize(); - const auto limit = std::min(first + (uint64_t {1} << 32u) - step, size >= 4u ? size - 4u : 0u); - const auto probe_count = size >= 4u && first <= limit ? (limit - first) / step + 1u : 0u; - if (probe_count > MaxIndirectDescriptorProbes) { - return false; - } - if (!read_keys(material, first, step, probe_count)) return false; - if (selector->shift == 0u) keys.push_back(0u); - } - for (uint32_t step = 0; step < indirect.selectors.size(); ++step) { - const auto& read = indirect.selectors[step]; - if (step != 0u) { - DescriptorValue value; - ShaderBufferResource material; - if (!clean.EvaluateDescriptor(read.source, value) || !DecodeBufferDescriptor(value, material)) - return false; - // Distinct keys can alias after U32 arithmetic and DWORD alignment. - for (auto& key: keys) key = (key * read.stride + read.offset) & ~3u; - std::ranges::sort(keys); - keys.erase(std::unique(keys.begin(), keys.end()), keys.end()); - for (auto& key: keys) { - const uint64_t offset = uint64_t {key} + (read.immediate & ~3u); - if (!ReadScalarTable(material.Base48(), material.GetSize(), offset, runtime, {&key, 1})) - return false; - } - } - for (auto& key: keys) key = (key >> read.shift) & read.bits; - if (step == 0u && (read.shift != 0u || read.bits != UINT32_MAX)) - keys.push_back(0u); // An out-of-range root material load returns zero. - std::ranges::sort(keys); - keys.erase(std::unique(keys.begin(), keys.end()), keys.end()); - } + } else return false; + std::ranges::sort(keys); + keys.erase(std::unique(keys.begin(), keys.end()), keys.end()); } - const auto children_begin = descriptors.size(); const auto mapping_offset = snapshot.flattened_srt.size(); - const auto root_resource = specializations[resource_index]; const auto key_count = sources.empty() ? keys.size() : sources.size(); snapshot.flattened_srt.resize(mapping_offset + 1u + key_count * 2u); snapshot.flattened_srt[mapping_offset] = static_cast(key_count); @@ -414,19 +429,8 @@ bool MaterializeIndirectDescriptor(const ResourcePlan& p uint32_t ordinal = 0; if (entry == 0) { descriptors[resource_index] = candidate; - } else if (descriptors[resource_index] != candidate) { - const auto found = - std::find(descriptors.begin() + children_begin, descriptors.end(), candidate); - ordinal = static_cast(found - descriptors.begin() - children_begin + 1u); - if (found == descriptors.end()) { - if (descriptors.size() >= maximum_resources) { - return false; - } - descriptors.push_back(candidate); - auto child = root_resource; - child.indirect_root = resource_index; - specializations.push_back(child); - } + } else if (!intern_candidate(candidate, ordinal)) { + return false; } snapshot.flattened_srt[mapping_offset + 1u + entry * 2u] = key; snapshot.flattened_srt[mapping_offset + 2u + entry * 2u] = ordinal; @@ -584,9 +588,9 @@ static bool BuildResourceSpecialization(const ResourcePlan& program, ResourceSna const auto key_count = root.indirect_mapping_offset < snapshot.flattened_srt.size() ? snapshot.flattened_srt[root.indirect_mapping_offset] : 0u; - if (root.indirect_search_iterations == 0u || key_count < 2u || + if (key_count == 0u || static_cast(root.indirect_mapping_offset) + 1u + - static_cast(key_count) * 2u > + static_cast(key_count) * (root.indirect_search_iterations == 0u ? 1u : 2u) > snapshot.flattened_srt.size()) { return SpecializationFail("indirect image specialization has an invalid key mapping"); } @@ -1028,9 +1032,9 @@ ResourcePlan ExtractResourcePlan(const Program& program) { if (!source.indirect_descriptor.has_value()) continue; const auto& indirect = *source.indirect_descriptor; plan.requires_specialization_memory = true; - capture_indirect_reads |= !indirect.selectors.empty() || !indirect.sources.empty(); - for (const auto& read: indirect.selectors) - MarkCleanFlatSlots(plan, Source(plan, read.source), plan.clean_flat_slots); + capture_indirect_reads = true; + if (indirect.selector) + MarkCleanFlatSlots(plan, Source(plan, indirect.selector->source), plan.clean_flat_slots); MarkCleanFlatSlots(plan, nullptr, plan.clean_flat_slots, indirect.selector_mask); MarkCleanFlatSlots(plan, nullptr, plan.clean_flat_slots, indirect.selector_first); MarkCleanFlatSlots(plan, nullptr, plan.clean_flat_slots, indirect.key_count); diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp index 3964280a..9ceb5735 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include @@ -374,11 +373,68 @@ public: } } for (const auto& plan: m_indirect_descriptors) { + auto key = plan.key; + if (plan.table_read != nullptr) { + const auto* anchor = plan.table_indexed ? plan.handle : plan.table_read; + auto* block = anchor->Parent(); + const auto where = std::ranges::find_if(block->Instructions(), + [&](const Inst& inst) { return &inst == anchor; }); + const auto emit = [&](ValueOpcode op, std::initializer_list args) { + return Value(&*block->PrependNewInst(where, op, args)); + }; + const auto& read = *plan.table_read; + const auto& memory = m_program.memory_info[read.Flags().index]; + Value offset; + Value valid(true); + if (plan.table_indexed) { + const auto* table = read.Arg(0).ResolveInstruction(); + const auto stride = emit(ValueOpcode::BitFieldUExtract, + {table->Arg(1), Value(16u), Value(14u)}); + const auto format = emit(ValueOpcode::BitFieldUExtract, + {table->Arg(3), Value(12u), Value(7u)}); + offset = emit(ValueOpcode::IAdd32, + {emit(ValueOpcode::IMul32, {read.Arg(1), stride}), Value(memory.offset)}); + valid = emit(ValueOpcode::LogicalAnd, + {emit(ValueOpcode::ULessThan32, {read.Arg(1), table->Arg(2)}), + emit(ValueOpcode::INotEqual32, {format, Value(0u)})}); + } else { + const auto dynamic = emit(ValueOpcode::BitwiseAnd32, {read.Arg(1), Value(~3u)}); + offset = emit(ValueOpcode::IAdd32, {dynamic, Value(memory.offset & ~3u)}); + valid = emit(ValueOpcode::ULessThanEqual32, {dynamic, offset}); + } + key = emit(ValueOpcode::SelectU32, {valid, offset, Value(UINT32_MAX)}); + if (!plan.table_indexed) { + // NativeDescriptorSource proved this read reaches the image. Preserve its + // SSA Phi edges while retaining descriptor DWORDs for other consumers. + std::map projected; + const auto project = [&](auto&& self, Value value) -> Value { + value = value.Resolve(); + const auto* phi = value.TryInstruction(); + if (phi == plan.table_read) return key; + if (phi == nullptr || phi->GetOpcode() != ValueOpcode::Phi) return Value(UINT32_MAX); + if (const auto found = projected.find(phi); found != projected.end()) return found->second; + auto* parent = phi->Parent(); + const auto position = std::ranges::find_if(parent->Instructions(), + [&](const Inst& inst) { return &inst == phi; }); + auto& selected = *parent->PrependNewInst(position, ValueOpcode::Phi); + selected.SetFlags(Type::U32); + projected.emplace(phi, Value(&selected)); + for (uint32_t arm = 0; arm < phi->NumArgs(); ++arm) + selected.AddPhiOperand(phi->PhiBlock(arm), self(self, phi->Arg(arm))); + return Value(&selected); + }; + key = project(project, plan.handle->Arg(0)); + } + plan.handle->SetArg(0, key); + for (uint32_t word = 1; word < plan.handle->NumArgs(); ++word) + plan.handle->SetArg(word, Value(0u)); + continue; + } if (plan.handle->GetOpcode() == ValueOpcode::GetBufferResource) { plan.handle->SetArg(0, plan.key); for (uint32_t word = 1; word < 4u; ++word) plan.handle->SetArg(word, Value(0u)); } else { - if (plan.reads[0] == nullptr) plan.handle->SetArg(0, plan.key); + if (plan.reads[0] == nullptr) plan.handle->SetArg(0, key); for (uint32_t dword = 0; dword < 4u; dword++) { plan.handle->SetArg(dword + 1u, plan.roots[dword + 4u]); } @@ -439,6 +495,8 @@ private: std::array reads {}; bool split_offsets = false; bool retain_reads = false; + bool table_indexed = false; + const Inst* table_read = nullptr; }; [[noreturn]] void Fail(uint32_t pc, const std::string& reason) const { @@ -836,7 +894,7 @@ private: uint32_t InternSource(const DescriptorSource& descriptor) { for (uint32_t candidate = 0; candidate < m_sources.size(); candidate++) { - const auto& current = m_sources[candidate]; + auto& current = m_sources[candidate]; if (current.dword_count != descriptor.dword_count || current.indirect_descriptor.has_value() != descriptor.indirect_descriptor.has_value()) { continue; @@ -844,7 +902,7 @@ private: if (current.indirect_descriptor.has_value()) { const auto& a = *current.indirect_descriptor; const auto& b = *descriptor.indirect_descriptor; - if (a.selectors != b.selectors || a.table_source != b.table_source || + if (a.selector != b.selector || a.table_source != b.table_source || a.table_offset != b.table_offset || a.table_immediate != b.table_immediate || a.table_stride != b.table_stride || a.workgroup_axis != b.workgroup_axis || a.sources != b.sources || @@ -861,6 +919,12 @@ private: same = same && EquivalentValue(m_program, current.dwords[i], descriptor.dwords[i]); } if (same) { + if (current.indirect_descriptor) { + current.indirect_descriptor->table_scalar |= descriptor.indirect_descriptor->table_scalar; + current.indirect_descriptor->table_record_bytes = std::max( + current.indirect_descriptor->table_record_bytes, + descriptor.indirect_descriptor->table_record_bytes); + } return candidate; } } @@ -950,54 +1014,6 @@ private: return ValidateSource(descriptor, bad_dword); } - Value MaterialOffset(Value value, uint32_t& stride, uint32_t& offset) const { - stride = 1u; - offset = 0u; - value = value.Resolve(); - while (const auto* inst = value.TryInstruction()) { - if (inst->NumArgs() != 2u) break; - uint32_t constant = 0; - Value remaining; - if (inst->GetOpcode() == ValueOpcode::ShiftLeftLogical32 && - ImmediateU32(inst->Arg(1), constant) && constant < 32u) { - stride *= 1u << constant; - remaining = inst->Arg(0); - } else if (inst->GetOpcode() == ValueOpcode::IAdd32 || - inst->GetOpcode() == ValueOpcode::IMul32) { - if (ImmediateU32(inst->Arg(0), constant)) remaining = inst->Arg(1); - else if (ImmediateU32(inst->Arg(1), constant)) remaining = inst->Arg(0); - else break; - if (inst->GetOpcode() == ValueOpcode::IAdd32) offset += stride * constant; - else stride *= constant; - } else { - break; - } - value = remaining.Resolve(); - } - // Treat the remaining GPU-derived scalar as any U32 value. - return stride != 0u && value.GetType() == Type::U32 && !value.IsImmediate() ? value : Value {}; - } - - Value MaterialKey(Value key, DescriptorSource::IndirectDescriptor::SelectorRead& selector) const { - selector.shift = 0u; - selector.bits = UINT32_MAX; - const auto* inst = key.Resolve().TryInstruction(); - uint32_t immediate; - if (inst != nullptr && inst->GetOpcode() == ValueOpcode::BitwiseAnd32) { - if (ImmediateU32(inst->Arg(1), immediate)) key = inst->Arg(0); - else if (ImmediateU32(inst->Arg(0), immediate)) key = inst->Arg(1); - else return key; - selector.bits = immediate; - inst = key.Resolve().TryInstruction(); - } - if (inst != nullptr && inst->GetOpcode() == ValueOpcode::ShiftRightLogical32 && - ImmediateU32(inst->Arg(1), immediate) && immediate != 0u && immediate < 32u) { - selector.shift = immediate; - key = inst->Arg(0); - } - return key.Resolve(); - } - enum class LaneQuantifier { Any, All }; struct EdgePredicate { Value condition; @@ -1444,7 +1460,7 @@ private: static_cast(UINT32_MAX) + 1u) return false; const auto mask = BoundedSetBitMask(offset.index, active); if (mask.IsEmpty()) return false; - indirect.selectors.push_back({.source = InternSource(material_source), + indirect.selector.emplace(DescriptorSource::IndirectDescriptor::SelectorRead{.source = InternSource(material_source), .stride = static_cast(offset.stride), .offset = static_cast(offset.offset + memory.offset)}); indirect.key_count = Value(32u); @@ -1609,6 +1625,7 @@ private: DescriptorSource& table_source, Value& key, uint32_t& table_offset, uint32_t& table_stride, uint32_t& table_immediate) { Inst* table_handle = nullptr; + Value table_guard; for (uint32_t dword = 0; dword < handle.NumArgs(); ++dword) { auto* read = descriptor.dwords[dword].Resolve().TryInstruction(); if (read == nullptr) { @@ -1616,6 +1633,30 @@ private: } uint32_t memory_index = 0; const auto* memory = ScalarReadMemory(*read, memory_index); + uint32_t component = 0; + const bool indexed = memory == nullptr; + if (indexed) { + if (handle.NumArgs() != 8u) return false; + const auto* selected = UniformizedMaterialValue(descriptor.dwords[dword], handle); + if (selected == nullptr) return false; + const auto* extract = selected->Arg(1).Resolve().TryInstruction(); + if (extract == nullptr || extract->GetOpcode() != ValueOpcode::CompositeExtractU32x4 || + !ImmediateU32(extract->Arg(1), component) || component >= 4u) return false; + read = extract->Arg(0).Resolve().TryInstruction(); + if (read == nullptr || read->GetOpcode() != ValueOpcode::LoadBufferU32x4) return false; + memory_index = read->Flags().index; + if (memory_index >= m_program.memory_info.size()) return false; + memory = &m_program.memory_info[memory_index]; + uint32_t offset = 1u, scalar = 1u; + if (memory->kind != ResourceKind::Buffer || memory->typed || memory->formatted || + !memory->idxen || memory->offen || memory->data_bits != 32u || + memory->data_dwords != 4u || + !ImmediateU32(read->Arg(2), offset) || offset != 0u || + !ImmediateU32(read->Arg(3), scalar) || scalar != 0u || + !EquivalentValue(m_program, read->Arg(4), selected->Arg(0)) || + (dword != 0u && !EquivalentValue(m_program, table_guard, read->Arg(4)))) return false; + table_guard = read->Arg(4); + } if (memory == nullptr || memory->offset > INT32_MAX || (memory->offset & 3u) != 0u || !MemoryIndexBelongsTo(memory_index, *read)) { return false; @@ -1631,23 +1672,31 @@ private: (memory->kind == ResourceKind::ScalarAddress && read->Parent() != handle.Parent()) || (table_handle != nullptr && !EquivalentValue(m_program, Value(table_handle), Value(current_handle))) || - !MatchTableOffset(read->Arg(1), current_key, offset, stride)) { + (!indexed && !MatchTableOffset(read->Arg(1), current_key, offset, stride))) { return false; } + if (indexed) { + current_key = read->Arg(1); + offset = component * 4u; + } if (dword == 0u) { + if (component != 0u) return false; + plan.table_indexed = indexed; key = current_key; table_offset = offset; table_immediate = memory->offset; table_stride = stride; - } else if (table_stride != stride || !EquivalentValue(m_program, key, current_key) || + } else if (plan.table_indexed != indexed || table_stride != stride || + !EquivalentValue(m_program, key, current_key) || uint64_t {table_offset} + table_immediate + dword * sizeof(uint32_t) != uint64_t {offset} + memory->offset) { return false; } - plan.split_offsets |= offset != table_offset; + plan.split_offsets |= !indexed && offset != table_offset; table_handle = current_handle; if (handle.NumArgs() == 8u) { - plan.retain_reads |= !UsesOnlyImageDescriptors(*read); + if (memory->kind == ResourceKind::ScalarAddress) + plan.retain_reads |= !UsesOnlyImageDescriptors(*read); } else if (std::ranges::any_of(read->Uses(), [](const Use& use) { return use.user->GetOpcode() != ValueOpcode::GetBufferResource || std::ranges::any_of(use.user->Uses(), [](const Use& consumer) { @@ -1666,36 +1715,6 @@ private: return true; } - bool MatchUniformizedBufferKey(Value key, const Inst& image, - DescriptorSource::IndirectDescriptor& indirect, - DescriptorSource& material_source) { - const auto* selected = UniformizedMaterialValue(key, image); - if (selected == nullptr) return false; - DescriptorSource::IndirectDescriptor::SelectorRead selector; - const auto value = MaterialKey(selected->Arg(1), selector); - if (selector.shift == 0u && selector.bits == UINT32_MAX) return false; - const auto* loaded = value.TryInstruction(); - if (loaded == nullptr || loaded->GetOpcode() != ValueOpcode::SelectU32 || - !EquivalentValue(m_program, selected->Arg(0), loaded->Arg(0))) return false; - const auto* read = loaded->Arg(1).Resolve().TryInstruction(); - if (read == nullptr || read->GetOpcode() != ValueOpcode::LoadBufferU32) return false; - const auto& memory = m_program.memory_info[read->Flags().index]; - uint32_t offset = 1u, scalar = 1u; - if (memory.kind != ResourceKind::Buffer || memory.typed || memory.formatted || - !memory.idxen || memory.offen || - !ImmediateU32(read->Arg(2), offset) || offset != 0u || - !ImmediateU32(read->Arg(3), scalar) || scalar != 0u || - !EquivalentValue(m_program, selected->Arg(0), read->Arg(4)) || - !MakeRuntimeTableSource(*read, material_source)) return false; - // GPU index arithmetic may wrap. Every record in the bounded V# is a candidate. - selector.source = InternSource(material_source); - selector.offset = memory.offset; - indirect.selectors.push_back(selector); - indirect.selector_first = Value(0u); - indirect.key_count = material_source.dwords[2]; - return true; - } - bool TryMakeIndirectImage(Inst& handle, const DescriptorSource& descriptor, IndirectDescriptorPlan& plan) { if (handle.GetOpcode() != ValueOpcode::GetImageResource || handle.NumArgs() != 8u) return false; @@ -1712,7 +1731,7 @@ private: indirect.table_immediate = table_immediate; indirect.table_stride = table_stride; indirect.workgroup_axis = WorkgroupAxis(key); - if (indirect.workgroup_axis != UINT32_MAX) { + if (indirect.workgroup_axis != UINT32_MAX && table_source.dword_count == 2u) { // The descriptor also supplies dimensions to shader arithmetic. Keep its reads; // only the image handle is projected onto the bounded workgroup key. plan.retain_reads = true; @@ -1734,39 +1753,20 @@ private: } if ((table_offset & 3u) != 0u || (bitscan && table_offset > UINT32_MAX - (32u * 32u - 1u))) return false; - } else if (!MatchUniformizedBufferKey(key, handle, indirect, material_source)) { - std::vector visited; - auto selected = key; - for (;;) { - DescriptorSource::IndirectDescriptor::SelectorRead selector; - const auto* read = MaterialKey(selected, selector).TryInstruction(); - uint32_t memory_index = 0; - const auto* memory = read != nullptr ? ScalarReadMemory(*read, memory_index) : nullptr; - DescriptorSource source; - if (memory == nullptr || memory->kind != ResourceKind::ScalarBuffer || - memory->offset > INT32_MAX || !MemoryIndexBelongsTo(memory_index, *read) || - !MakeRuntimeTableSource(*read, source)) break; - if (std::ranges::find(visited, read) != visited.end()) return false; - visited.push_back(read); - selected = MaterialOffset(read->Arg(1), selector.stride, selector.offset); - if (selected.IsEmpty()) return false; - selector.source = InternSource(source); - selector.immediate = memory->offset; - indirect.selectors.push_back(selector); - material_source = source; - } - if (indirect.selectors.empty()) return false; - std::ranges::reverse(indirect.selectors); - } - // Separate shader additions may only be merged into a contiguous descriptor - // when the selector cannot wrap between those additions. - if (plan.split_offsets && indirect.workgroup_axis == UINT32_MAX && - table_source.dword_count == 4u) { - const auto& selector = indirect.selectors.back(); - const auto maximum_key = (UINT32_MAX >> selector.shift) & selector.bits; - if (uint64_t {maximum_key} * table_stride + table_offset + table_immediate + 28u > - UINT32_MAX) return false; + } else { + // A bounded V# supplies the complete image table. Leave every GPU selector + // and descriptor read in the shader; the host only translates table bytes. + if (plan.split_offsets) return false; + indirect.table_stride = 0u; + indirect.table_offset = 0u; + indirect.table_immediate = 0u; + indirect.workgroup_axis = UINT32_MAX; + plan.table_read = plan.reads[0]; + if (plan.table_indexed) indirect.table_record_bytes = table_immediate + 32u; + else indirect.table_scalar = true; + plan.retain_reads = true; } + if (plan.retain_reads) plan.reads.fill(nullptr); indirect.table_source = InternSource(table_source); DescriptorSource image_source; @@ -1836,7 +1836,7 @@ private: indirect.table_stride = table_stride; if (!MakeRuntimeTableSource(*read, material_source) || !BoundedSelectedIndex(read->Arg(1), read->Arg(4), indirect)) return false; - indirect.selectors.push_back({.source = InternSource(material_source), + indirect.selector.emplace(DescriptorSource::IndirectDescriptor::SelectorRead{.source = InternSource(material_source), .offset = memory.offset + lane * 4u}); indirect.table_source = InternSource(table_source); DescriptorSource source; diff --git a/tests/ResourceTrackingTests.cpp b/tests/ResourceTrackingTests.cpp index d84a2826..8a483c3d 100644 --- a/tests/ResourceTrackingTests.cpp +++ b/tests/ResourceTrackingTests.cpp @@ -206,10 +206,7 @@ bool ReadLinearTestMemory(void *userdata, uint64_t address, std::span } std::unique_ptr -MakeIndirectImageFixture(bool malformed, uint32_t material_immediate = 4, - bool memory_backed_material = false, uint32_t member_offset = 0, - uint32_t material_stride = 224, uint32_t selector_bits = UINT32_MAX, - uint32_t table_stride = 32, uint32_t table_offset = 0, +MakeIndirectImageFixture(bool malformed, uint32_t table_stride = 32, uint32_t table_offset = 0, uint32_t table_shader_offset = 0, bool nested = false) { auto fixture = std::make_unique(); std::array material_words; @@ -218,43 +215,14 @@ MakeIndirectImageFixture(bool malformed, uint32_t material_immediate = 4, material_words[dword] = fixture->UserData(dword); heap_words[dword] = fixture->UserData(dword + 4u); } - if (memory_backed_material) { - const auto pointer_address = - fixture->Address(fixture->UserData(9), fixture->UserData(10), 0x10b0); - MemoryInfo pointer_word; - pointer_word.kind = ResourceKind::ScalarAddress; - const auto pointer = - fixture->Emit(ValueOpcode::LoadAddressU32, - {pointer_address, Value(0u), Value(0u), Value(true)}, - fixture->AddMemory(pointer_word, 0x10b0)); - const auto address = fixture->Address(pointer, Value(0u), 0x10c0); - MemoryInfo descriptor_word; - descriptor_word.kind = ResourceKind::ScalarAddress; - material_words[0] = - fixture->Emit(ValueOpcode::LoadAddressU32, - {address, Value(0u), Value(0u), Value(true)}, - fixture->AddMemory(descriptor_word, 0x10c0)); - } const auto material = fixture->Buffer(material_words, 0x10d8); const auto heap = fixture->Buffer(heap_words, 0x10d8); - if (memory_backed_material) { - MemoryInfo shared_buffer; - shared_buffer.kind = ResourceKind::Buffer; - const auto load = - fixture->Emit(ValueOpcode::LoadBufferU32, - {material, Value(0u), Value(0u), Value(0u), Value(true)}, - fixture->AddMemory(shared_buffer, 0x10d8)); - fixture->Emit(ValueOpcode::ReferenceU32, {load}); - } const auto invocation = fixture->Emit( ValueOpcode::GetBuiltin, {Value(static_cast(StageInputKind::GlobalInvocationId)), Value(0u)}); const auto selector = fixture->Emit(ValueOpcode::ReadFirstLane, {invocation, Value(true)}); - const auto record = - fixture->Emit(ValueOpcode::IMul32, {selector, Value(material_stride)}); - auto member = member_offset == 0 ? record : - fixture->Emit(ValueOpcode::IAdd32, {record, Value(member_offset)}); + Value member; if (nested) { std::array root_words; for (uint32_t word = 0; word < 4u; ++word) root_words[word] = fixture->UserData(12u + word); @@ -266,16 +234,16 @@ MakeIndirectImageFixture(bool malformed, uint32_t material_immediate = 4, fixture->AddMemory(root_memory, 0x1000)); const auto masked = fixture->Emit(ValueOpcode::BitwiseAnd32, {pointer, Value(0xfffffff1u)}); member = fixture->Emit(ValueOpcode::IAdd32, {masked, Value(16u)}); + } else { + member = fixture->Emit(ValueOpcode::IMul32, {selector, Value(224u)}); } fixture->Emit(ValueOpcode::ReferenceU32, {member}); MemoryInfo material_scalar; material_scalar.kind = ResourceKind::ScalarBuffer; - material_scalar.offset = material_immediate; - auto key = + material_scalar.offset = 4u; + const auto key = fixture->Emit(ValueOpcode::ReadConstBuffer, {material, member}, fixture->AddMemory(material_scalar, 0x10d8)); - if (selector_bits != UINT32_MAX) - key = fixture->Emit(ValueOpcode::BitwiseAnd32, {key, Value(selector_bits)}); auto heap_offset = fixture->Emit(ValueOpcode::IMul32, {key, Value(table_stride)}); if (table_shader_offset != 0u) heap_offset = fixture->Emit(ValueOpcode::IAdd32, {heap_offset, Value(table_shader_offset)}); @@ -312,384 +280,225 @@ MakeIndirectImageFixture(bool malformed, uint32_t material_immediate = 4, } void TestInvariantIndirectImageMaterialization() { - auto fixture = MakeIndirectImageFixture(false); + auto fixture = MakeIndirectImageFixture(false, 48u, 0u, 16u, true); fixture->PlanAndTrack(); - auto resource_plan = ExtractResourcePlan(fixture->program); + auto plan = ExtractResourcePlan(fixture->program); EliminateDeadCode(fixture->program.blocks); ValidateProgram(fixture->program, true); - - Check(fixture->program.info.buffers.size() == 1 && - fixture->program.info.images.size() == 1 && - std::ranges::any_of(*fixture->block, [](const Inst &inst) { - return inst.GetOpcode() == ValueOpcode::ReadConstBuffer; - }), - "indirect image key was not retained as a scalar-buffer read"); - const auto source = fixture->program.info.images[0].source; - Check(source < fixture->program.descriptor_sources.size() && - fixture->program.descriptor_sources[source] - .indirect_descriptor.has_value(), - "indirect image source was not retained for runtime proof"); - const auto image_handle = - std::ranges::find_if(*fixture->block, [](const Inst &inst) { - return inst.GetOpcode() == ValueOpcode::GetImageResource; - }); - Check(image_handle != fixture->block->end() && - image_handle->Arg(0).ResolveInstruction() != nullptr && - image_handle->Arg(0).ResolveInstruction()->GetOpcode() == - ValueOpcode::ReadConstBuffer, - "indirect image handle discarded the live material key"); - - std::array user_data{0x1000u, 224u << 16u, 2u, 0u, 0x2000u, - 16u << 16u, 4u, 0u, 7u}; + const auto &source = *plan.descriptor_sources[plan.info.images[0].source].indirect_descriptor; + Check(!source.selector && fixture->program.info.buffers.size() == 3u, + "bounded image table retained CPU selector traversal or lost live loads"); + std::array descriptor{}; + descriptor[0] = 0x20u; + descriptor[1] = static_cast(Libs::Graphics::Prospero::BufferFormat::k32_32_32_32Float) << 20u; + descriptor[2] = 3u | (3u << 14u); + descriptor[3] = Libs::Graphics::DstSel(4, 5, 6, 7) | + (static_cast(Libs::Graphics::Prospero::ImageType::kColor2D) << 28u); + std::array data{0x3000u, 16u << 16u, 0x200000u, 0u, + 0x2000u, 48u << 16u, 3u, 0u}; + std::copy_n(data.begin(), 4u, data.begin() + 12u); LinearTestMemory memory; - std::array image_descriptor{}; - image_descriptor[0] = 0x20u; - image_descriptor[1] = - static_cast( - Libs::Graphics::Prospero::BufferFormat::k32_32_32_32Float) - << 20u; - image_descriptor[2] = 3u | (3u << 14u); - image_descriptor[3] = - Libs::Graphics::DstSel(4, 5, 6, 7) | - (static_cast(Libs::Graphics::Prospero::ImageType::kColor2D) - << 28u); - for (uint32_t dword = 0; dword < image_descriptor.size(); dword++) { - memory.words[(0x2000u - memory.base) / 4u + dword] = - image_descriptor[dword]; - memory.words[(0x2020u - memory.base) / 4u + dword] = - image_descriptor[dword]; - } - memory.words[(0x2020u - memory.base) / 4u] ^= 1u; - - SrtRuntime runtime{.user_data = user_data, - .userdata = &memory, + memory.fail_address = 0x3000u; // GPU-only material backing must never be read on the CPU. + memory.watched_address = 0x2000u; + const auto put = [&](uint32_t offset, uint32_t identity) { + auto value = descriptor; + value[0] += identity; + std::copy(value.begin(), value.end(), memory.words.begin() + (0x1000u + offset) / 4u); + }; + put(16u, 0u); + put(64u, 1u); + SrtRuntime runtime{.user_data = data, .userdata = &memory, .read_specialization_memory = ReadLinearTestMemory}; ResourceSnapshot snapshot; ResourceSpecialization specialization; - Check(MaterializeResources(resource_plan, runtime, snapshot, specialization) && - snapshot.images.size() == 1 && - std::equal(image_descriptor.begin(), image_descriptor.end(), - snapshot.images[0].dwords.begin()), - "invariant indirect image table did not materialize"); - - user_data[5] = 0u; - user_data[6] = 19u; + Check(MaterializeResources(plan, runtime, snapshot, specialization) && + snapshot.images.size() == 3u && memory.watched_reads == 1u && + memory.watched_dwords == 36u && memory.reads == 1u, + "image table was not read once independently of its GPU selector chain"); + const auto root = specialization.images[0]; + const auto mapping = root.indirect_mapping_offset; + Check(root.indirect_search_iterations == 0u && snapshot.flattened_srt[mapping] == 17u && + snapshot.flattened_srt[mapping + 1u] == 0u && + snapshot.flattened_srt[mapping + 5u] == 1u && + snapshot.flattened_srt[mapping + 17u] == 2u && + snapshot.specialization_reads == std::vector>{{0x2000u, 144u}}, + "dense image mapping lost its null slot, trimmed length or coherent read dependency"); + const auto original_specialization = specialization; + std::fill_n(memory.words.begin() + 0x1040u / 4u, 8u, 0u); + put(112u, 1u); + Check(MaterializeResources(plan, runtime, snapshot, specialization) && + specialization == original_specialization && memory.watched_reads == 2u && + snapshot.flattened_srt[mapping] == 29u, + "moving a table entry changed shader specialization or reused stale mapping bytes"); + put(112u, 7u); + Check(MaterializeResources(plan, runtime, snapshot, specialization) && + snapshot.images[2].dwords[0] == descriptor[0] + 7u, + "changed table descriptor was not refreshed"); + // Scalar reads align each DWORD and return zero beyond the V# byte size. + data[5] = 0u; + data[6] = 19u; + std::fill(memory.words.begin() + 0x1000u / 4u, memory.words.end(), 0u); + put(0u, 0u); memory.fail_address = 0x2010u; - memory.watched_address = 0x2000u; - Check(MaterializeResources(resource_plan, runtime, snapshot, specialization) && - memory.watched_dwords == 4u && - std::equal(image_descriptor.begin(), image_descriptor.end(), - snapshot.images[0].dwords.begin()), - "partial scalar-buffer descriptor read crossed bounds instead of zeroing its tail"); + Check(MaterializeResources(plan, runtime, snapshot, specialization) && + memory.watched_dwords == 4u && snapshot.images.size() == 2u && + snapshot.images[1].dwords == descriptor, + "partial descriptor prefix did not zero its out-of-bounds DWORDs"); memory.fail_address = 0x2008u; - Check(!MaterializeResources(resource_plan, runtime, snapshot, specialization), - "unreadable memory inside the descriptor prefix was accepted"); - user_data[5] = 16u << 16u; - user_data[6] = 4u; - memory.watched_address = UINT64_MAX; - - memory.fail_address = 0x1004u; - Check(!MaterializeResources(resource_plan, runtime, snapshot, - specialization), - "unreadable material-table selector was accepted"); + Check(!MaterializeResources(plan, runtime, snapshot, specialization), + "unreadable in-range table bytes were accepted"); memory.fail_address = UINT64_MAX; - - memory.words[(0x1000u - memory.base + 36u) / 4u] = 1u; - for (uint32_t dword = 0; dword < image_descriptor.size(); dword++) { - memory.words[(0x2000u - memory.base) / 4u + dword] = 0u; - memory.words[(0x2020u - memory.base) / 4u + dword] = 0u; + for (uint32_t size : {0u, 3u, 4u}) { + data[6] = size; + Check(MaterializeResources(plan, runtime, snapshot, specialization) && + snapshot.images.size() == 1u && + specialization.images[0].indirect_root == ImageResource::NoIndirectImage && + std::ranges::all_of(snapshot.images[0].dwords, [](uint32_t word) { return word == 0u; }), + "empty image table did not specialize as a safe null image"); } - memory.words[(0x2000u - memory.base) / 4u + 1u] = image_descriptor[1]; - memory.words[(0x2000u - memory.base) / 4u + 3u] = image_descriptor[3]; - memory.words[(0x2020u - memory.base) / 4u + 1u] = image_descriptor[1]; - memory.words[(0x2020u - memory.base) / 4u + 3u] = - image_descriptor[3] ^ (1u << 28u); - ResourceSnapshot null_snapshot; - ResourceSpecialization null_specialization; - Check(MaterializeResources(resource_plan, runtime, null_snapshot, - null_specialization) && - std::ranges::all_of(null_snapshot.images[0].dwords, - [](uint32_t dword) { return dword == 0u; }), - "stale typed null image descriptors were not canonicalized"); - - for (uint32_t dword = 0; dword < image_descriptor.size(); dword++) { - memory.words[(0x2000u - memory.base) / 4u + dword] = - image_descriptor[dword]; - memory.words[(0x2020u - memory.base) / 4u + dword] = - image_descriptor[dword]; + // The GPU computes these offsets; host materialization must not enumerate the key. + for (const auto [shader, immediate, key, expected] : + {std::array{16u, 0u, 0x55555555u, 0u}, + std::array{0u, 16u, 0x55555555u, UINT32_MAX}, + std::array{0xfffffff0u, 32u, 0u, UINT32_MAX}}) { + auto wrap = MakeIndirectImageFixture(false, 48u, immediate, shader); + wrap->PlanAndTrack(); + for (auto &inst : *wrap->block) { + if (inst.GetOpcode() == ValueOpcode::ReadConstBuffer) { + inst.ReplaceUsesWith(Value(key)); + break; + } + } + const auto handle = std::ranges::find_if(*wrap->block, [](const Inst &inst) { + return inst.GetOpcode() == ValueOpcode::GetImageResource; + }); + SrtWalker walker(wrap->program, runtime); + uint32_t offset = 0; + Check(walker.Evaluate(handle->Arg(0), offset) && offset == expected, + "GPU descriptor offset projection lost U32 wrap or scalar immediate carry"); } - memory.words[(0x2020u - memory.base) / 4u] ^= 1u; - memory.words[(0x1000u - memory.base + 36u) / 4u] = 1u; - ResourceSnapshot dynamic_snapshot; - ResourceSpecialization dynamic_specialization; - Check(MaterializeResources(resource_plan, runtime, dynamic_snapshot, - dynamic_specialization) && - dynamic_snapshot.images.size() == 2 && - dynamic_specialization.images.size() == 2, - "dynamic indirect image table did not materialize"); - const auto second_image = (0x2020u - memory.base) / 4u; - memory.words[second_image + 1u] |= 3u << 30u; - memory.words[second_image + 2u] = 3u << 14u; - memory.words[second_image + 3u] = - Libs::Graphics::DstSel(4, 5, 6, 7) | - (static_cast(Libs::Graphics::Prospero::ImageType::kCube) << 28u); - memory.words[second_image + 4u] = 11u; - ResourceSnapshot mixed_snapshot; - ResourceSpecialization mixed_specialization; - Check(MaterializeResources(resource_plan, runtime, mixed_snapshot, - mixed_specialization) && - mixed_snapshot.images.size() == 2 && - mixed_specialization.images.size() == 2 && - mixed_specialization.images[0].dimension == - Decoder::ImageDimension::Dim2D && - !mixed_specialization.images[0].cube && - mixed_specialization.images[1].dimension == - Decoder::ImageDimension::Dim2DArray && - mixed_specialization.images[1].cube && - std::equal(mixed_snapshot.images[1].dwords.begin(), - mixed_snapshot.images[1].dwords.end(), - memory.words.begin() + second_image), - "mixed 2D and cube candidates were rejected or discarded"); - memory.words[second_image + 1u] = - static_cast( - Libs::Graphics::Prospero::BufferFormat::k32_32_32_32UInt) - << 20u | - (3u << 30u); - Check(!MaterializeResources(resource_plan, runtime, mixed_snapshot, - mixed_specialization), - "indirect images with different numeric classes were accepted"); - for (const auto dword : {1u, 2u, 3u, 4u}) { - memory.words[second_image + dword] = image_descriptor[dword]; - } - ApplyResourceSpecialization(fixture->program, dynamic_specialization); - Check(fixture->program.info.images.size() == 2 && - fixture->program.info.images[0].indirect_root == 0 && - fixture->program.info.images[0].indirect_search_iterations != 0 && - fixture->program.info.images[0].indirect_resources.size() == 2 && - dynamic_snapshot.images.size() == 2, - "dynamic indirect image table was not specialized transactionally"); - const auto &mapping = dynamic_specialization.images[0]; - const auto key_count = dynamic_snapshot.flattened_srt[mapping.indirect_mapping_offset]; - Check(mapping.indirect_search_iterations == std::bit_width(key_count) && - mapping.indirect_mapping_offset + 1u + key_count * 2u == - dynamic_snapshot.flattened_srt.size(), - "indirect image mapping retained worst-case padding"); - - for (uint32_t dword = 0; dword < image_descriptor.size(); dword++) { - memory.words[(0x2000u - memory.base) / 4u + dword] = - image_descriptor[dword]; - memory.words[(0x2020u - memory.base) / 4u + dword] = - image_descriptor[dword]; - } - memory.words[(0x2000u - memory.base) / 4u] += 0x100u; - memory.words[(0x2020u - memory.base) / 4u] += 0x101u; - ResourceSnapshot rebound_snapshot; - ResourceSpecialization rebound_specialization; - Check(MaterializeResources(resource_plan, runtime, rebound_snapshot, - rebound_specialization) && - rebound_specialization == dynamic_specialization, - "stable indirect key mapping did not accept changed image addresses"); - memory.words[(0x2020u - memory.base) / 4u] = - memory.words[(0x2000u - memory.base) / 4u]; - Check(MaterializeResources(resource_plan, runtime, rebound_snapshot, - rebound_specialization) && - rebound_specialization != dynamic_specialization, - "collapsed indirect candidates did not select a new specialization"); - const auto collapsed_specialization = rebound_specialization; - ResourceSnapshot capacity_snapshot; - ResourceSpecialization capacity_specialization; - for (const uint32_t records : {1u, 3u}) { - user_data[2] = records; - Check(MaterializeResources(resource_plan, runtime, capacity_snapshot, - capacity_specialization), - "runtime indirect key mapping rejected a valid material-table size"); - } - user_data[2] = 2u; - memory.words[(0x2020u - memory.base) / 4u] = - memory.words[(0x2000u - memory.base) / 4u] + 1u; - memory.words[(0x2040u - memory.base) / 4u] = - memory.words[(0x2000u - memory.base) / 4u] + 2u; - for (uint32_t dword = 1; dword < image_descriptor.size(); dword++) { - memory.words[(0x2040u - memory.base) / 4u + dword] = - image_descriptor[dword]; - } - memory.words[(0x1000u - memory.base + 68u) / 4u] = 2u; - Check(MaterializeResources(resource_plan, runtime, rebound_snapshot, - rebound_specialization) && - rebound_specialization != collapsed_specialization, - "larger indirect candidate topology reused the old specialization"); - - auto memory_backed = MakeIndirectImageFixture(false, 4u, true); - memory_backed->PlanAndTrack(); - auto memory_backed_plan = ExtractResourcePlan(memory_backed->program); - EliminateDeadCode(memory_backed->program.blocks); - std::array memory_backed_user_data{0x1000u, 224u << 16u, 2u, 0u, - 0x2000u, 16u << 16u, 4u, 0u, - 7u, 0x3100u, 0u}; - memory.words[(0x3100u - memory.base) / 4u] = 0x3000u; - memory.words[(0x3000u - memory.base) / 4u] = 0x1000u; - memory.fail_address = 0x3100u; - SrtRuntime memory_backed_runtime{.user_data = memory_backed_user_data, - .userdata = &memory, - .read_specialization_memory = - ReadLinearTestMemory}; - Check(!MaterializeResources(memory_backed_plan, memory_backed_runtime, - snapshot, specialization), - "unreadable indirect table descriptor was accepted"); - memory.fail_address = UINT64_MAX; - - memory.watched_address = 0x3100u; - memory.watched_reads = 0; - Check(MaterializeResources(memory_backed_plan, memory_backed_runtime, - snapshot, specialization) && memory.watched_reads == 1u, - "descriptor, flattened SRT and indirect-table roots repeated a clean pointer read"); - const auto* buffer_storage = snapshot.buffers.data(); - const auto* image_storage = snapshot.images.data(); - const auto* flat_storage = snapshot.flattened_srt.data(); - const auto* user_data_storage = snapshot.user_data.data(); - const auto* buffer_specialization_storage = specialization.buffers.data(); - const auto* image_specialization_storage = specialization.images.data(); - const auto old_address = snapshot.images[0].dwords[0]; - memory.words[(0x2000u - memory.base) / 4u] += 0x100u; - memory.watched_reads = 0; - Check(MaterializeResources(memory_backed_plan, memory_backed_runtime, - snapshot, specialization) && memory.watched_reads == 1u && - snapshot.images[0].dwords[0] == old_address + 0x100u, - "cache refresh reused stale table contents or repeated its clean pointer read"); - Check(snapshot.buffers.data() == buffer_storage && snapshot.images.data() == image_storage && - snapshot.flattened_srt.data() == flat_storage && - snapshot.user_data.data() == user_data_storage && - specialization.buffers.data() == buffer_specialization_storage && - specialization.images.data() == image_specialization_storage, - "a same-shape refresh discarded the runtime output storage"); - auto malformed = MakeIndirectImageFixture(true); - CheckFatal([&] { malformed->PlanAndTrack(); }, "not a valid runtime value", - "malformed indirect image pattern was accepted"); - Check(!malformed->program.resource_tracking_complete && - malformed->program.info.images.empty() && - malformed->program.descriptor_sources.empty(), - "malformed indirect image pattern was partially accepted"); + "noncontiguous descriptor DWORDs entered the table plan"); + Check(!malformed->program.resource_tracking_complete && malformed->program.descriptor_sources.empty(), + "malformed image table tracking was not transactional"); +} - auto negative_immediate = MakeIndirectImageFixture(false, 0xfffffffcu); - CheckFatal([&] { negative_immediate->PlanAndTrack(); }, - "not a valid runtime value", - "negative scalar immediate entered the indirect image proof"); - - for (const auto [immediate, member, first, stride] : - {std::array{0u, 4u, 4u, 224u}, std::array{4u, 4u, 8u, 224u}, - std::array{36u, 0u, 36u, 224u}, std::array{1u, 3u, 0u, 224u}, - std::array{1u, 3u, 0u, 1u}, std::array{1u, 3u, 0u, 2u}}) { - auto split_offset = MakeIndirectImageFixture(false, immediate, false, member, stride); - split_offset->PlanAndTrack(); - const auto split_plan = ExtractResourcePlan(split_offset->program); - LinearTestMemory split_memory; - std::copy(image_descriptor.begin(), image_descriptor.end(), - split_memory.words.begin() + 0x1000u / 4u); - split_memory.fail_address = first == 36u ? 0x1004u : UINT64_MAX; - split_memory.watched_address = split_memory.base + first; - user_data[1] = stride << 16u; - user_data[2] = stride < 4u ? 8u / stride : 1u; - SrtRuntime split_runtime{.user_data = user_data, - .userdata = &split_memory, - .read_specialization_memory = ReadLinearTestMemory}; - Check(MaterializeResources(split_plan, split_runtime, snapshot, specialization) && - snapshot.images.size() == 1 && - snapshot.images[0].dwords == image_descriptor && - split_memory.watched_reads == 1u, - "indirect scalar offsets did not align and bound their components independently"); +void TestWaterfallImageTable() { + namespace CFG = Libs::Graphics::ShaderRecompiler::CFG; + const auto make = [](bool equality, bool scalar) { + auto fixture = std::make_unique(); + auto *entry = fixture->block; + auto *body = fixture->AddBlock(); + auto *exit = fixture->AddBlock(); + entry->AddBranch(body); + entry->AddBranch(exit); + body->AddBranch(exit); + std::array words; + for (uint32_t i = 0; i < 4u; ++i) words[i] = fixture->UserData(i); + const auto table = fixture->Buffer(words); + const auto lane_value = fixture->Emit(ValueOpcode::GetAttribute, {Value(0u), Value(0u)}); + const auto active = fixture->Emit(ValueOpcode::INotEqual32, {lane_value, Value(0u)}); + const auto index = fixture->UserData(4u); + std::array image_words; + auto guard = active; + for (uint32_t group = 0; group < 2u; ++group) { + MemoryInfo memory; + memory.kind = ResourceKind::Buffer; + memory.idxen = true; + memory.data_dwords = 4u; + memory.offset = 16u + group * 16u; + const auto loaded = fixture->Emit(ValueOpcode::LoadBufferU32x4, + {table, index, Value(0u), Value(0u), active}, fixture->AddMemory(memory, 0x850u)); + for (uint32_t component = 0; component < 4u; ++component) { + const auto word = group * 4u + component; + const auto local = fixture->Emit(ValueOpcode::SelectU32, + {active, fixture->Emit(ValueOpcode::CompositeExtractU32x4, + {loaded, Value(component)}), Value(0xdeadu)}); + image_words[word] = fixture->Emit(ValueOpcode::ReadLane, {local, Value(0u)}); + if (equality || word != 7u) + guard = fixture->Emit(ValueOpcode::LogicalAnd, + {guard, fixture->Emit(ValueOpcode::IEqual32, {image_words[word], local})}); + } + } + fixture->program.block_info[0].condition = guard; + fixture->program.block_info[0].terminator = { + .kind = CFG::TerminatorKind::ConditionalBranch, .true_block = 1u, .false_block = 2u}; + const auto sample = [&](Value image, Block *block) { + const auto sampler = fixture->Sampler({Value(0u), Value(0u), Value(0u), Value(0u)}); + MemoryInfo memory; + memory.kind = ResourceKind::Image; + memory.image_dimension = Decoder::ImageDimension::Dim2D; + const auto value = fixture->Emit(ValueOpcode::ImageSampleRaw, + {image, sampler, fixture->ImageAddress()}, fixture->AddMemory(memory, 0x92cu), block); + fixture->Emit(ValueOpcode::ReferenceU32, + {fixture->Emit(ValueOpcode::CompositeExtractU32x4, {value, Value(0u)}, 0, block)}, 0, block); + }; + if (scalar) { + const auto key = fixture->Emit(ValueOpcode::ReadFirstLane, {lane_value, active}); + const auto offset = fixture->Emit(ValueOpcode::IAdd32, + {fixture->Emit(ValueOpcode::IMul32, {key, Value(48u)}), Value(16u)}); + std::array scalar_words; + for (uint32_t word = 0; word < 8u; ++word) { + MemoryInfo memory; + memory.kind = ResourceKind::ScalarBuffer; + memory.offset = word * 4u; + scalar_words[word] = fixture->Emit(ValueOpcode::ReadConstBuffer, {table, offset}, + fixture->AddMemory(memory, 0x7bcu)); + } + sample(fixture->Image(scalar_words), entry); + } + const auto image = fixture->Emit(ValueOpcode::GetImageResource, + {image_words[0], image_words[1], image_words[2], image_words[3], + image_words[4], image_words[5], image_words[6], image_words[7]}, 0, body); + sample(image, body); + return fixture; + }; + auto fixture = make(true, true); + fixture->PlanAndTrack(); + auto plan = ExtractResourcePlan(fixture->program); + Check(plan.info.images.size() == 1u && + plan.descriptor_sources[plan.info.images[0].source].indirect_descriptor->table_record_bytes == 48u, + "scalar and waterfall uses did not share their native image table"); + std::array data{0x2000u, 48u << 16u, 2u, 0x5204u, 1u}; + LinearTestMemory memory; + const std::array descriptor{0x20u, 75u << 20u, 3u | (3u << 14u), + Libs::Graphics::DstSel(4, 5, 6, 7) | (9u << 28u), 0u, 0u, 0u, 0u}; + std::copy(descriptor.begin(), descriptor.end(), memory.words.begin() + 0x1010u / 4u); + SrtRuntime runtime{.user_data = data, .userdata = &memory, + .read_specialization_memory = ReadLinearTestMemory}; + ResourceSnapshot snapshot; + ResourceSpecialization specialization; + Check(MaterializeResources(plan, runtime, snapshot, specialization) && memory.reads == 1u && + snapshot.images.size() == 2u, + "shared scalar/waterfall table repeated its coherent bulk read"); + const auto handle = std::ranges::find_if(*fixture->program.blocks[1], [](const Inst &inst) { + return inst.GetOpcode() == ValueOpcode::GetImageResource; + }); + for (const auto [index, format, expected] : + {std::array{1u, 0x5204u, 64u}, std::array{2u, 0x5204u, UINT32_MAX}, + std::array{UINT32_MAX, 0x5204u, UINT32_MAX}, std::array{0u, 0u, UINT32_MAX}}) { + data[4] = index; + data[3] = format; + SrtWalker walker(fixture->program, runtime); + uint32_t offset = 0; + Check(walker.Evaluate(handle->Arg(0), offset) && offset == expected, + "VMEM table projection lost record bounds, format disable or multiplication carry"); } - - // The shader packs flags above a 15-bit cube-image key in the final DWORD of - // each 64-byte material record; the image occupies bytes 16..47 of its record. - auto packed = MakeIndirectImageFixture(false, 60u, false, 0u, 64u, 0x7fffu, 48u, 16u); - packed->PlanAndTrack(); - const auto packed_plan = ExtractResourcePlan(packed->program); - LinearTestMemory packed_memory; - std::array packed_data{0x1000u, 64u << 16u, 3u, 0u, - 0x2000u, 48u << 16u, 2u, 0u}; - for (uint32_t i = 0; i < 3u; ++i) - packed_memory.words[(i * 64u + 60u) / 4u] = (0xffffu << 15u) | (i & 1u); - for (uint32_t i = 0; i < 2u; ++i) { - auto descriptor = image_descriptor; - descriptor[0] += i; - std::copy(descriptor.begin(), descriptor.end(), - packed_memory.words.begin() + (0x1000u + i * 48u + 16u) / 4u); - } - SrtRuntime packed_runtime{.user_data = packed_data, .userdata = &packed_memory, - .read_specialization_memory = ReadLinearTestMemory}; - Check(MaterializeResources(packed_plan, packed_runtime, snapshot, specialization) && - snapshot.images.size() == 2u && snapshot.images[0].dwords == image_descriptor && - snapshot.images[1].dwords[0] == image_descriptor[0] + 1u && - snapshot.flattened_srt[specialization.images[0].indirect_mapping_offset] == 2u, - "packed material flags changed the table key or 48-byte descriptor addressing"); - packed_memory.words[60u / 4u] = 0xffff8002u; - packed_memory.fail_address = 0x2070u; - Check(MaterializeResources(packed_plan, packed_runtime, snapshot, specialization) && - snapshot.images.size() == 3u && - std::ranges::all_of(snapshot.images[2].dwords, [](uint32_t word) { return word == 0u; }), - "masked out-of-range descriptor did not return zero without reading past table bounds"); - packed_memory.fail_address = 0x205cu; - Check(!MaterializeResources(packed_plan, packed_runtime, snapshot, specialization), - "unreadable in-range DWORD in a 48-byte descriptor record was accepted"); - // 0x55555555 * 48 is 0xfffffff0: shader +16 wraps, scalar +16 does not. - for (const auto [shader_offset, immediate, key, image_count] : - {std::array{0u, 16u, 0x55555555u, 1u}, std::array{16u, 0u, 0x55555555u, 2u}, - std::array{0xfffffff0u, 32u, 0u, 1u}}) { - auto wrapping = MakeIndirectImageFixture(false, 60u, false, 0u, 64u, - UINT32_MAX, 48u, immediate, shader_offset); - wrapping->PlanAndTrack(); - const auto wrapping_plan = ExtractResourcePlan(wrapping->program); - LinearTestMemory wrapping_memory; - wrapping_memory.words[60u / 4u] = key; - std::copy(image_descriptor.begin(), image_descriptor.end(), - wrapping_memory.words.begin() + 0x1000u / 4u); - std::array data{0x1000u, 64u << 16u, 1u, 0u, - 0x2000u, 48u << 16u, 1u, 0u}; - SrtRuntime wrapped_runtime{.user_data = data, .userdata = &wrapping_memory, - .read_specialization_memory = ReadLinearTestMemory}; - Check(MaterializeResources(wrapping_plan, wrapped_runtime, snapshot, specialization) && - snapshot.images.size() == image_count && - std::ranges::all_of(snapshot.images[0].dwords, [](uint32_t word) { return word == 0u; }) && - (image_count == 1u || snapshot.images[1].dwords == image_descriptor), - "descriptor addressing conflated shader U32 wrap with scalar immediate addition"); - } - - auto nested = MakeIndirectImageFixture(false, 0u, false, 0u, 1u, - UINT32_MAX, 48u, 0u, 16u, true); - nested->PlanAndTrack(); - const auto nested_plan = ExtractResourcePlan(nested->program); - const auto &chain = *nested_plan.descriptor_sources[nested_plan.info.images[0].source].indirect_descriptor; - Check(chain.selectors.size() == 2u && chain.selectors[0].stride == 32u && - chain.selectors[0].bits == 0xfffffff1u && chain.selectors[1].offset == 16u && - nested->program.info.buffers.size() == 3u, - "nested material selectors lost their ordered reads or live descriptor field"); - LinearTestMemory nested_memory; - nested_memory.words[0] = 0u; - nested_memory.words[32u / 4u] = 1u; - nested_memory.words[64u / 4u] = 0xffffffffu; - nested_memory.words[0x2000u / 4u] = 0u; - nested_memory.words[0x2010u / 4u] = 1u; - nested_memory.fail_address = 0x3020u; - nested_memory.watched_address = 0x3010u; - for (uint32_t key = 0; key < 2u; ++key) { - auto descriptor = image_descriptor; - descriptor[0] += key; - std::copy(descriptor.begin(), descriptor.end(), - nested_memory.words.begin() + (0x1010u + key * 48u) / 4u); - } - std::array nested_data{0x3000u, 16u << 16u, 0x200000u, 0u, - 0x2000u, 48u << 16u, 2u, 0u, - 0, 0, 0, 0, 0x1000u, 32u << 16u, 3u, 0u}; - SrtRuntime nested_runtime{.user_data = nested_data, .userdata = &nested_memory, - .read_specialization_memory = ReadLinearTestMemory}; - Check(MaterializeResources(nested_plan, nested_runtime, snapshot, specialization) && - snapshot.images.size() == 2u && snapshot.images[0].dwords == image_descriptor && - snapshot.images[1].dwords[0] == image_descriptor[0] + 1u && - nested_memory.watched_reads == 1u, - "nested selector traversal scanned unrelated material fields or repeated aliased offsets"); - + auto disabled = make(true, false); + disabled->PlanAndTrack(); + const auto disabled_plan = ExtractResourcePlan(disabled->program); + data[3] = 0u; + memory.fail_address = 0x2000u; + Check(MaterializeResources(disabled_plan, runtime, snapshot, specialization) && + snapshot.images.size() == 1u && + specialization.images[0].indirect_root == ImageResource::NoIndirectImage, + "disabled VMEM descriptor table read its unavailable backing"); + Check(!MaterializeResources(plan, runtime, snapshot, specialization), + "VMEM format disable incorrectly disabled a shared scalar descriptor read"); + auto malformed = make(false, false); + CheckFatal([&] { malformed->PlanAndTrack(); }, "not a valid runtime value", + "waterfall without all descriptor equality witnesses was accepted"); } void TestGuardedDirectImageTable() { @@ -780,7 +589,7 @@ void TestGuardedDirectImageTable() { fixture.PlanAndTrack(); const auto source = fixture.program.info.images[0].source; const auto &indirect = fixture.program.descriptor_sources[source].indirect_descriptor; - Check(indirect && indirect->selectors.empty() && + Check(indirect && !indirect->selector.has_value() && indirect->table_offset == 344u && indirect->key_count.Resolve().IsImmediate() && indirect->key_count.Resolve().U32() == 32u && @@ -1078,7 +887,7 @@ void TestBoundedComputeImageLoop() { fixture.PlanAndTrack(); const auto source = fixture.program.info.images[0].source; const auto &indirect = fixture.program.descriptor_sources[source].indirect_descriptor; - Check(indirect && indirect->selectors.empty() && + Check(indirect && !indirect->selector.has_value() && indirect->table_offset == 0x6b0u && indirect->key_count.Resolve() == count.Resolve(), "bounded compute loop lost its runtime image count"); @@ -1433,8 +1242,8 @@ void TestUniformizedMaterialImageKeys() { fixture.PlanAndTrack(); const auto source = fixture.program.info.images[0].source; const auto &indirect = fixture.program.descriptor_sources[source].indirect_descriptor; - Check(indirect && indirect->selectors.front().stride == 0x90u && - indirect->selectors.front().offset == 0xc00u && + Check(indirect && indirect->selector->stride == 0x90u && + indirect->selector->offset == 0xc00u && indirect->table_offset == 0x20e0u && !indirect->selector_mask.IsEmpty() && indirect->key_count.Resolve().IsImmediate() && @@ -3708,6 +3517,7 @@ int main() { Run("gather LOD sampler validation", TestGatherLodSamplerValidation); Run("dynamic storage mips", TestDynamicStorageMipTracking); Run("invariant indirect images", TestInvariantIndirectImageMaterialization); + Run("waterfall image table", TestWaterfallImageTable); Run("guarded direct image table", TestGuardedDirectImageTable); Run("bounded compute image loop", TestBoundedComputeImageLoop); Run("uniformized material image keys", TestUniformizedMaterialImageKeys); diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 8047efae..9d8d4de0 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -33594,7 +33594,9 @@ void CheckIndirectImageKeySwitch() { program.descriptor_sources.resize(2); program.descriptor_sources[0].dword_count = 8; program.descriptor_sources[0].indirect_descriptor = - DescriptorSource::IndirectDescriptor{.selectors = {{.source = 0, .stride = 224, .offset = 12}}, + DescriptorSource::IndirectDescriptor{ + .selector = DescriptorSource::IndirectDescriptor::SelectorRead{ + .source = 0, .stride = 224, .offset = 12}, .table_source = 0, .table_stride = 32}; program.descriptor_sources[1].dword_count = 4; @@ -33644,6 +33646,19 @@ void CheckIndirectImageKeySwitch() { sampled_image != 0u && std::ranges::find(nonuniform, sampled_image) != nonuniform.end(), "the sample operand lacks its nonuniform decoration"); + program.info.images[0].indirect_search_iterations = 0; + program.binding_layout_complete = false; + AllocateBindings(program); + spirv = ShaderRecompiler::Spirv::EmitProgram(program, {.compute = &compute}); + ValidateSpirv("DenseIndirectImageTable", spirv); + Require(name, "dense image map disassembly", tools.Disassemble(spirv, &text), + "failed to disassemble dense image table shader"); + Require(name, "bounded dense image selection", + CountText(text, "OpULessThan") == 1 && + CountText(text, "OpImageSampleExplicitLod") == 1 && + text.find("SampledImageArrayNonUniformIndexing") != std::string::npos, + "dense image selection lost its bounds check or native nonuniform array"); + // Materialized children follow all native roots, including unrelated images. root.indirect_resources = {0u}; program.info.images = {root, candidate}; @@ -38051,15 +38066,6 @@ void CheckImageSamplerSpecialization() { options.user_data = user_data; options.input_info.compute = &compute; auto translated = ShaderRecompiler::TranslateProgram(code, options); - if (scalar) { - bool key_phi = false; - for (const auto *block : translated.program.blocks) - for (const auto &inst : *block) - key_phi |= inst.GetOpcode() == ValueOpcode::GetImageResource && - inst.Arg(0).Resolve().IsPhi(); - Require(name, "descriptor key reconvergence", key_phi, - "native crossed exits did not retain a descriptor key Phi"); - } auto plan = ExtractResourcePlan(translated.program); ResourceSnapshot snapshot; ResourceSpecialization specialization; @@ -38071,56 +38077,57 @@ void CheckImageSamplerSpecialization() { snapshot.images.size() == count + 1u, "GPU-selected material descriptors did not expand beyond native root capacity"); const auto offset = specialization.images[0].indirect_mapping_offset; - const auto ordinal = [&](u32 key) { - for (u32 i = 0; i < snapshot.flattened_srt[offset]; ++i) - if (snapshot.flattened_srt[offset + 1u + 2u * i] == key) - return snapshot.flattened_srt[offset + 2u + 2u * i]; - return UINT32_MAX; + const auto ordinal = [&](u32 byte_offset) { + const auto word = byte_offset >> 2u; + return word < snapshot.flattened_srt[offset] + ? snapshot.flattened_srt[offset + 1u + word] : 0u; }; - Require(name, "shift, wrap and whole-record read", - snapshot.flattened_srt[offset] == count + 2u && ordinal(1u) != UINT32_MAX && - ordinal(0x08000001u) == ordinal(1u) && ordinal(count) < snapshot.images.size() && - snapshot.images[ordinal(count)].dwords[0] == 0u && - std::ranges::count(snapshot.specialization_reads, - std::pair{material_base, (count + 3u) * 4u}) == 1, - "material keys lost shift/dedup, scalar wrap/OOB, or bulk strict-read semantics"); + Require(name, "dense table map and coherent snapshot", + specialization.images[0].indirect_search_iterations == 0u && + snapshot.flattened_srt[offset] == (count - 1u) * 8u + 1u && + ordinal(32u) != 0u && + ordinal(count * 32u) == 0u && snapshot.images[0].dwords[0] == 0u && + snapshot.specialization_reads == + std::vector>{{table_base, count * 32u}}, + "image table lost its dense offsets, explicit null or single coherent read"); const auto previous = specialization; memory[table_base / 4u + 8u] += 0x10000u; Require(name, "runtime descriptor refresh", MaterializeResources(plan, runtime, snapshot, specialization) && - specialization == previous && snapshot.images[1].dwords[0] == 0x11100u, + specialization == previous && snapshot.images[ordinal(32u)].dwords[0] == 0x11100u, "descriptor addresses were frozen or entered the shader specialization"); auto compiled = ShaderRecompiler::CompileProgram(std::move(translated), options, specialization); ValidateSpirv(name, compiled.spirv); if (!scalar) { user_data[4] += 1u; - Require(name, "misaligned material rejection", - !MaterializeResources(plan, runtime, snapshot, specialization), - "unaligned raw buffer base was treated as separately aligned scalar memory"); + Require(name, "GPU material reads remain on GPU", + MaterializeResources(plan, runtime, snapshot, specialization) && + specialization == previous && snapshot.specialization_reads == + std::vector>{{table_base, count * 32u}}, + "GPU material addressing reentered host descriptor selection"); user_data[4] -= 1u; } auto dirty = runtime; dirty.read_specialization_memory = +[](void*, uint64_t, std::span) { return false; }; - Require(name, "dirty material rejection", + Require(name, "unreadable image table rejection", !MaterializeResources(plan, dirty, snapshot, specialization), "GPU-selected texture materialization bypassed strict read provenance"); -#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS if (scalar) { - ExpectFatal("ScalarImageDescriptorNumericConsumer", [&] { - code.insert(code.begin() + table_read_end, EncodeVop1(0x01, 18, 12)); - code.pop_back(); - AppendBufferStoreDword(&code, 18, 0); - code[2] = EncodeSopp(0x08, static_cast(code.size() - 3u)); - AppendEnd(&code); - (void)ShaderRecompiler::TranslateProgram(code, options); - }); + code.insert(code.begin() + table_read_end, EncodeVop1(0x01, 18, 12)); + code.pop_back(); + AppendBufferStoreDword(&code, 18, 0); + code[2] = EncodeSopp(0x08, static_cast(code.size() - 3u)); + AppendEnd(&code); } else { - ExpectFatal("MaterialImageWithoutEntryWitness", [&] { - code[2] = EncodeSopp(0x00, 0); // S_NOP removes the original nonempty-EXEC witness. - (void)ShaderRecompiler::TranslateProgram(code, options); - }); + code[2] = EncodeSopp(0x00, 0); // GPU selectors do not need a host nonempty-EXEC witness. } -#endif + auto retained = ShaderRecompiler::TranslateProgram(code, options); + auto retained_plan = ExtractResourcePlan(retained.program); + Require(name, "retained descriptor consumers", + MaterializeResources(retained_plan, runtime, snapshot, specialization), + "retained GPU descriptor reads could not use the shared image table"); + compiled = ShaderRecompiler::CompileProgram(std::move(retained), options, specialization); + ValidateSpirv(name, compiled.spirv); } std::printf("[host] %-32s ok\n", "ImageSpecializationPipelineId");