From 429d0af62b6e17b30966bc4edc94d58bd3d123bf Mon Sep 17 00:00:00 2001 From: nmzik Date: Sat, 3 Oct 2026 23:04:51 +0400 Subject: [PATCH] shader: bind bounded GPU-selected buffer store destinations Co-authored-by: Ali Almohaya --- 3rdparty/cpuinfo/src/arm/mach/init.c | 2 +- .../shader/recompiler/ShaderRecompiler.cpp | 5 + .../backend/spirv/spirvEmitterImage.cpp | 54 +-- .../backend/spirv/spirvEmitterInternal.h | 2 + .../backend/spirv/spirvEmitterMemory.cpp | 48 ++- .../spirv/spirvEmitterMemoryHelpers.cpp | 45 +++ src/graphics/shader/recompiler/ir/ShaderIR.h | 12 +- .../recompiler/ir/passes/BindingLayout.cpp | 11 +- .../ir/passes/ResourceMaterialization.cpp | 234 ++++++++---- .../ir/passes/ResourceMaterialization.h | 3 + .../recompiler/ir/passes/ResourceTracking.cpp | 335 ++++++++++++++---- tests/KernelFileSystemTests.cpp | 2 +- tests/ResourceTrackingTests.cpp | 14 +- tests/ShaderRecompilerComputeTests.cpp | 121 ++++++- tests/shaderCfgTests.cpp | 107 ++++++ 15 files changed, 784 insertions(+), 211 deletions(-) diff --git a/3rdparty/cpuinfo/src/arm/mach/init.c b/3rdparty/cpuinfo/src/arm/mach/init.c index dbea578c..5428e9fb 100644 --- a/3rdparty/cpuinfo/src/arm/mach/init.c +++ b/3rdparty/cpuinfo/src/arm/mach/init.c @@ -14,7 +14,7 @@ #include #include -/* Polyfill recent CPUFAMILY_ARM_* values for older SDKs */ +/* Polyfill recent CPUFAMILY_ARM_* values for older platform headers */ #ifndef CPUFAMILY_ARM_MONSOON_MISTRAL #define CPUFAMILY_ARM_MONSOON_MISTRAL 0xE81E7EF6 #endif diff --git a/src/graphics/shader/recompiler/ShaderRecompiler.cpp b/src/graphics/shader/recompiler/ShaderRecompiler.cpp index 9e840f35..9b74c9a9 100644 --- a/src/graphics/shader/recompiler/ShaderRecompiler.cpp +++ b/src/graphics/shader/recompiler/ShaderRecompiler.cpp @@ -648,6 +648,11 @@ CompileResult CompileProgram(TranslateResult translated, const CompileOptions& o })) { continue; } + const auto resource = inst.Flags(); + first = resource < ir.info.buffers.size() && + ir.info.buffers[resource].indirect_root == resource + ? 1u + : 0u; } else if (op == IR::ValueOpcode::GetImageResource) { const auto resource = inst.Flags(); first = resource < ir.info.images.size() && diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp index 6cb0911b..17c34a0b 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp @@ -808,7 +808,7 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) { const auto* source = image.source < state.program.descriptor_sources.size() ? &state.program.descriptor_sources[image.source] : nullptr; - if (handle == nullptr || source == nullptr || !source->indirect_image.has_value() || + if (handle == nullptr || source == nullptr || !source->indirect_descriptor.has_value() || handle->NumArgs() == 0u) { ctx.Fail(inst, "has invalid indirect image key provenance"); return; @@ -819,56 +819,8 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) { ctx.Fail(inst, "has no indirect image runtime mapping"); return; } - const auto LoadMapping = [&](uint32_t index) { - const auto pointer = state.builder.AllocateId(); - state.builder.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state), - pointer, state.flattened_srt_variable, ConstantU32(state, 0), - index); - const auto value = state.builder.AllocateId(); - state.builder.AddFunction(spv::OpLoad, TypeU32(state), value, pointer); - return value; - }; - const auto mapping = ConstantU32(state, image.indirect_mapping_offset); - auto low = ConstantU32(state, 0u); - auto high = LoadMapping(mapping); - auto selected = ConstantU32(state, 0u); - for (uint32_t iteration = 0; iteration < image.indirect_search_iterations; - iteration++) { - const auto active = Binary(state, spv::OpULessThan, TypeBool(state), low, high); - const auto mid = Binary(state, spv::OpShiftRightLogical, TypeU32(state), - Binary(state, spv::OpIAdd, TypeU32(state), low, high), - ConstantU32(state, 1u)); - const auto probe = state.builder.AllocateId(); - state.builder.AddFunction(spv::OpSelect, TypeU32(state), probe, active, mid, - ConstantU32(state, 0u)); - const auto entry = Binary(state, spv::OpIAdd, TypeU32(state), mapping, - Binary(state, spv::OpIAdd, TypeU32(state), - Binary(state, spv::OpShiftLeftLogical, TypeU32(state), - probe, ConstantU32(state, 1u)), - ConstantU32(state, 1u))); - const auto mapped_key = LoadMapping(entry); - const auto candidate = LoadMapping( - Binary(state, spv::OpIAdd, TypeU32(state), entry, ConstantU32(state, 1u))); - const auto equal = Binary(state, spv::OpIEqual, TypeBool(state), mapped_key, key); - const auto match = Binary(state, spv::OpLogicalAnd, TypeBool(state), active, equal); - const auto next_selected = state.builder.AllocateId(); - state.builder.AddFunction(spv::OpSelect, TypeU32(state), next_selected, match, - candidate, selected); - selected = next_selected; - const auto less = Binary(state, spv::OpULessThan, TypeBool(state), mapped_key, key); - const auto take_upper = Binary(state, spv::OpLogicalAnd, TypeBool(state), active, less); - const auto take_lower = Binary(state, spv::OpLogicalAnd, TypeBool(state), active, - Unary(state, spv::OpLogicalNot, TypeBool(state), less)); - const auto next_low = state.builder.AllocateId(); - state.builder.AddFunction( - spv::OpSelect, TypeU32(state), next_low, take_upper, - Binary(state, spv::OpIAdd, TypeU32(state), mid, ConstantU32(state, 1u)), low); - low = next_low; - const auto next_high = state.builder.AllocateId(); - state.builder.AddFunction(spv::OpSelect, TypeU32(state), next_high, take_lower, mid, - high); - high = next_high; - } + const auto selected = EmitIndirectResourceIndex( + state, key, image.indirect_mapping_offset, image.indirect_search_iterations, 0u); const auto default_label = state.builder.AllocateId(); const auto merge_label = state.builder.AllocateId(); std::vector labels(image.indirect_resources.size() - 1u); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h index 46cc6227..3b0dce75 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h @@ -408,6 +408,8 @@ struct MemoryResourceAccess { spv::MemoryAccessMask memory_access = spv::MemoryAccessMaskNone; }; +uint32_t EmitIndirectResourceIndex(EmitterState& state, uint32_t key, uint32_t mapping_offset, + uint32_t search_iterations, uint32_t default_resource); MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::MemoryInfo& mem); MemoryResourceAccess PrepareStorageBufferResourceAccess(EmitterState& state, diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp index 323cfaab..10a7f492 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp @@ -549,9 +549,51 @@ void StoreWordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::Me } void StoreWord(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem) { - EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() { - const auto resource = PrepareMemoryResourceAccess(ctx.state, mem); - StoreWordPrepared(ctx, inst, mem, resource, ctx.Arg(inst, inst.NumArgs() - 2)); + auto& state = ctx.state; + EmitIfCondition(state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() { + const auto data = ctx.Arg(inst, inst.NumArgs() - 2); + const auto store = [&](uint32_t resource) { + if (mem.kind == IR::ResourceKind::Buffer) { + const auto& buffer = state.program.info.buffers[resource]; + if (buffer.packed_stride == 0u && + buffer.source < state.program.descriptor_sources.size() && + state.program.descriptor_sources[buffer.source].indirect_descriptor) return; + } + mem.resource = resource; + const auto access = PrepareMemoryResourceAccess(state, mem); + StoreWordPrepared(ctx, inst, mem, access, data); + }; + if (mem.kind != IR::ResourceKind::Buffer || + state.program.info.buffers[mem.resource].indirect_root != mem.resource) { + store(mem.resource); + return; + } + const auto& buffer = state.program.info.buffers[mem.resource]; + const auto* handle = inst.Arg(0).ResolveInstruction(); + if (handle == nullptr || buffer.indirect_resources.empty() || + state.flattened_srt_variable == 0 || buffer.indirect_search_iterations == 0u) { + ctx.Fail(inst, "has no indirect buffer runtime mapping"); + return; + } + const auto selected = EmitIndirectResourceIndex( + state, ctx.Arg(*handle, 0), buffer.indirect_mapping_offset, + buffer.indirect_search_iterations, UINT32_MAX); + const auto merge = state.builder.AllocateId(); + std::vector labels(buffer.indirect_resources.size()); + std::vector branches {spv::OpSwitch, selected, merge}; + for (uint32_t i = 0; i < labels.size(); ++i) { + labels[i] = state.builder.AllocateId(); + branches.push_back(i); + branches.push_back(labels[i]); + } + state.builder.AddFunction(spv::OpSelectionMerge, merge, spv::SelectionControlMaskNone); + state.builder.AddFunction(branches); + for (uint32_t i = 0; i < labels.size(); ++i) { + EmitLabel(state, labels[i]); + store(buffer.indirect_resources[i]); + state.builder.AddFunction(spv::OpBranch, merge); + } + EmitLabel(state, merge); }); } diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp index 39f0aaea..0406ad73 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp @@ -31,6 +31,51 @@ uint32_t EmitBinaryU32(EmitterState& state, spv::Op opcode, uint32_t lhs, uint32 return ret; } +uint32_t EmitIndirectResourceIndex(EmitterState& state, uint32_t key, uint32_t mapping_offset, + uint32_t search_iterations, uint32_t default_resource) { + const auto LoadMapping = [&](uint32_t index) { + const auto pointer = state.builder.AllocateId(); + state.builder.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state), + pointer, state.flattened_srt_variable, ConstantU32(state, 0), + index); + const auto value = state.builder.AllocateId(); + state.builder.AddFunction(spv::OpLoad, TypeU32(state), value, pointer); + return value; + }; + const auto mapping = ConstantU32(state, mapping_offset); + const auto count = LoadMapping(mapping); + const auto entry_at = [&](uint32_t index) { + return Binary(state, spv::OpIAdd, TypeU32(state), mapping, + Binary(state, spv::OpIAdd, TypeU32(state), + Binary(state, spv::OpShiftLeftLogical, TypeU32(state), index, + ConstantU32(state, 1u)), ConstantU32(state, 1u))); + }; + auto low = ConstantU32(state, 0u); + auto high = count; + for (uint32_t iteration = 0; iteration < search_iterations; ++iteration) { + const auto active = Binary(state, spv::OpULessThan, TypeBool(state), low, high); + const auto mid = Binary(state, spv::OpShiftRightLogical, TypeU32(state), + Binary(state, spv::OpIAdd, TypeU32(state), low, high), + ConstantU32(state, 1u)); + const auto probe = Select(state, TypeU32(state), active, mid, ConstantU32(state, 0u)); + const auto less = Binary(state, spv::OpULessThan, TypeBool(state), + LoadMapping(entry_at(probe)), key); + low = Select(state, TypeU32(state), + Binary(state, spv::OpLogicalAnd, TypeBool(state), active, less), + Binary(state, spv::OpIAdd, TypeU32(state), mid, ConstantU32(state, 1u)), low); + high = Select(state, TypeU32(state), + Binary(state, spv::OpLogicalAnd, TypeBool(state), active, + Unary(state, spv::OpLogicalNot, TypeBool(state), less)), mid, high); + } + const auto in_range = Binary(state, spv::OpULessThan, TypeBool(state), low, count); + const auto entry = entry_at(Select(state, TypeU32(state), in_range, low, ConstantU32(state, 0u))); + const auto match = Binary(state, spv::OpLogicalAnd, TypeBool(state), in_range, + Binary(state, spv::OpIEqual, TypeBool(state), LoadMapping(entry), key)); + const auto resource = LoadMapping(Binary(state, spv::OpIAdd, TypeU32(state), entry, + ConstantU32(state, 1u))); + return Select(state, TypeU32(state), match, resource, ConstantU32(state, default_resource)); +} + uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryInfo& mem) { if (mem.resource >= state.program.info.buffers.size()) { ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers, mem.resource, diff --git a/src/graphics/shader/recompiler/ir/ShaderIR.h b/src/graphics/shader/recompiler/ir/ShaderIR.h index 07cb7f6e..73bade90 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIR.h +++ b/src/graphics/shader/recompiler/ir/ShaderIR.h @@ -97,6 +97,7 @@ struct ExportInfo { struct BufferResource { static constexpr uint32_t NoImageAlias = UINT32_MAX; + static constexpr uint32_t NoIndirectBuffer = UINT32_MAX; uint32_t source = 0; uint32_t first_use_pc = 0; @@ -110,6 +111,10 @@ struct BufferResource { bool atomic = false; bool formatted = false; bool scalar = false; + uint32_t indirect_root = NoIndirectBuffer; + uint32_t indirect_mapping_offset = 0; + uint32_t indirect_search_iterations = 0; + std::vector indirect_resources; bool operator==(const BufferResource& other) const = default; }; @@ -472,22 +477,23 @@ struct BlockInfo { }; struct DescriptorSource { - struct IndirectImage { + struct IndirectDescriptor { uint32_t material_source = UINT32_MAX; uint32_t table_source = 0; uint32_t selector_stride = 0; uint32_t selector_offset = 0; uint32_t table_offset = 0; Value key_count; + Value selector_first; Value selector_mask; std::vector sources; - bool operator==(const IndirectImage& other) const = default; + bool operator==(const IndirectDescriptor& other) const = default; }; std::array dwords {}; uint32_t dword_count = 0; - std::optional indirect_image; + std::optional indirect_descriptor; bool operator==(const DescriptorSource& other) const = default; }; diff --git a/src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp b/src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp index ba4cb279..08aaed1f 100644 --- a/src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp +++ b/src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp @@ -74,6 +74,9 @@ bool CollectMemoryResources(const Program& program, std::vector& buffe } else if (memory.kind == ResourceKind::Buffer || memory.kind == ResourceKind::ScalarBuffer) { EXIT_IF(memory.resource >= program.info.buffers.size()); live_buffers.at(memory.resource) = true; + for (const auto child: program.info.buffers[memory.resource].indirect_resources) { + live_buffers.at(child) = true; + } } } } @@ -86,13 +89,15 @@ bool CollectMemoryResources(const Program& program, std::vector& buffe } bool UsesFlattenedSrt(const Program& program) { + const auto uses_mapping = [](const auto& resource) { + return resource.indirect_search_iterations != 0u; + }; return std::ranges::any_of(program.blocks, [](const Block* block) { return std::ranges::any_of(*block, [](const Inst& inst) { return inst.GetOpcode() == ValueOpcode::ReadConst; }); - }) || std::ranges::any_of(program.info.images, [](const ImageResource& image) { - return image.indirect_search_iterations != 0u; - }); + }) || std::ranges::any_of(program.info.buffers, uses_mapping) || + std::ranges::any_of(program.info.images, uses_mapping); } void AllocateBindings(Program& program, uint32_t push_data_start_dword) { diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp index 6cb56bec..88e48ca9 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp @@ -20,7 +20,7 @@ namespace Libs::Graphics::ShaderRecompiler::IR { namespace { constexpr uint64_t AddressMask = 0x0000ffffffffffffull; -constexpr uint64_t MaxIndirectImageProbes = 65536u; +constexpr uint64_t MaxIndirectDescriptorProbes = 65536u; bool SpecializationFail(std::string_view message) { std::fprintf(stderr, "shader resource specialization failed: %.*s\n", @@ -224,12 +224,29 @@ bool ReadScalarTable(uint64_t base, uint64_t size, uint64_t dynamic_offset, runtime.read_specialization_memory(runtime.userdata, address, prefix); } -bool MaterializeIndirectImage(const ResourcePlan& program, - const DescriptorSource::IndirectImage& indirect, - uint32_t image_index, - const SrtRuntime& runtime, SrtWalker& clean, - ResourceSnapshot& snapshot, - ResourceSpecialization& specialization) { +bool NormalizeIndirectStoreBuffer(DescriptorValue& value) { + ShaderBufferResource descriptor; + if (!DecodeBufferDescriptor(value, descriptor)) return false; + if (descriptor.Type() != 0u || descriptor.Format() == Prospero::BufferFormat::kInvalid || + descriptor.Base48() == 0u || descriptor.GetSize() == 0u) { + value.dwords.fill(0u); + } else if (descriptor.Stride() != 1u || descriptor.SwizzleEnabled() || descriptor.AddTid() || + descriptor.OutOfBounds() != 3u) { + return SpecializationFail("indirect buffer stores require linear byte bounds"); + } + return true; +} + +template +bool MaterializeIndirectDescriptor(const ResourcePlan& program, + const DescriptorSource::IndirectDescriptor& indirect, + uint32_t resource_index, uint32_t dword_count, + const SrtRuntime& runtime, SrtWalker& clean, + ResourceSnapshot& snapshot, + std::vector& descriptors, + std::vector& specializations, + uint32_t maximum_resources, Normalize&& normalize) { + const auto descriptor_bytes = dword_count * sizeof(uint32_t); const auto& sources = indirect.sources; auto& keys = program.material_keys; uint64_t table_base = 0; @@ -257,12 +274,39 @@ bool MaterializeIndirectImage(const ResourcePlan& program, const bool evaluated = clean.Evaluate(indirect.key_count, key_count); if (std::bit_cast(key_count) <= 0) key_count = 0; if (table_value.dword_count != 2u || !evaluated || - key_count > MaxIndirectImageProbes || - uint64_t {indirect.table_offset} + uint64_t {key_count} * 32u > UINT32_MAX + 1ull) { + key_count > MaxIndirectDescriptorProbes || + uint64_t {indirect.table_offset} + uint64_t {key_count} * descriptor_bytes > + UINT32_MAX + 1ull) { return false; } keys.resize(key_count); std::iota(keys.begin(), keys.end(), 0u); + } else if (!indirect.selector_first.IsEmpty()) { + ShaderBufferResource material; + uint32_t first = 0, count = 0; + if (!DecodeBufferDescriptor(material_value, material) || material.Type() != 0u || + material.SwizzleEnabled() || + material.AddTid() || material.OutOfBounds() != 0u || + uint64_t {indirect.selector_offset} + 4u > material.Stride() || + !clean.Evaluate(indirect.selector_first, first) || + !clean.Evaluate(indirect.key_count, count) || count > MaxIndirectDescriptorProbes || + uint64_t {first} + count > material.NumRecords()) + return false; + if (count != 0u && (uint64_t {first} + count - 1u) * material.Stride() + + indirect.selector_offset + 4u > + uint64_t {UINT32_MAX} + 1u) + return false; + keys.reserve(count); + for (uint64_t index = first; index < uint64_t {first} + count; ++index) { + uint32_t key = 0; + const auto offset = index * material.Stride() + indirect.selector_offset; + if (!ReadScalarTable(material.Base48(), material.GetSize(), offset, runtime, + {&key, 1})) + return false; + keys.push_back(key); + } + std::ranges::sort(keys); + keys.erase(std::unique(keys.begin(), keys.end()), keys.end()); } else if (!indirect.selector_mask.IsEmpty()) { uint32_t mask = 0; uint32_t count = 0; @@ -301,7 +345,7 @@ bool MaterializeIndirectImage(const ResourcePlan& program, const auto size = material.GetSize(); const auto limit = std::min(first + (uint64_t {1} << 32u) - step, size >= 4u ? size - 4u : 0u); const auto probe_count = size >= 4u && first <= limit ? (limit - first) / step + 1u : 0u; - if (probe_count > MaxIndirectImageProbes) { + if (probe_count > MaxIndirectDescriptorProbes) { return false; } keys.reserve(static_cast(probe_count) + 1u); @@ -318,53 +362,52 @@ bool MaterializeIndirectImage(const ResourcePlan& program, } } - const auto children_begin = snapshot.images.size(); + const auto children_begin = descriptors.size(); const auto mapping_offset = snapshot.flattened_srt.size(); - const auto root_image = specialization.images[image_index]; + const auto root_resource = specializations[resource_index]; const auto key_count = sources.empty() ? keys.size() : sources.size(); snapshot.flattened_srt.resize(mapping_offset + 1u + key_count * 2u); snapshot.flattened_srt[mapping_offset] = static_cast(key_count); for (uint32_t entry = 0; entry < key_count; ++entry) { const auto key = sources.empty() ? keys[entry] : entry; DescriptorValue candidate; - candidate.dword_count = 8u; + candidate.dword_count = dword_count; if (sources.empty()) { - const auto table_offset = (key << 5u) + indirect.table_offset; - if (!ReadScalarTable(table_base, table_size, table_offset, runtime, candidate.dwords)) { + const auto table_offset = + static_cast(key * descriptor_bytes) + indirect.table_offset; + if (!ReadScalarTable(table_base, table_size, table_offset, runtime, + std::span(candidate.dwords).first(dword_count))) { return false; } } else if (!clean.EvaluateDescriptor(sources[entry], candidate)) { return false; } - if (NullImageDescriptor(candidate) || - !ValidImageDescriptor(candidate, program.info.images[image_index].r128)) { - candidate.dwords.fill(0); - } + if (!normalize(candidate)) return false; uint32_t ordinal = 0; if (entry == 0) { - snapshot.images[image_index] = candidate; - } else if (snapshot.images[image_index] != candidate) { - const auto found = std::find(snapshot.images.begin() + children_begin, - snapshot.images.end(), candidate); - ordinal = static_cast(found - snapshot.images.begin() - children_begin + 1u); - if (found == snapshot.images.end()) { - if (snapshot.images.size() >= ShaderInfo::MaxImages) { + descriptors[resource_index] = candidate; + } else if (descriptors[resource_index] != candidate) { + const auto found = + std::find(descriptors.begin() + children_begin, descriptors.end(), candidate); + ordinal = static_cast(found - descriptors.begin() - children_begin + 1u); + if (found == descriptors.end()) { + if (descriptors.size() >= maximum_resources) { return false; } - snapshot.images.push_back(candidate); - auto child = root_image; - child.indirect_root = image_index; - specialization.images.push_back(child); + descriptors.push_back(candidate); + auto child = root_resource; + child.indirect_root = resource_index; + specializations.push_back(child); } } snapshot.flattened_srt[mapping_offset + 1u + entry * 2u] = key; snapshot.flattened_srt[mapping_offset + 2u + entry * 2u] = ordinal; } - if (snapshot.images.size() == children_begin) { + if (descriptors.size() == children_begin) { snapshot.flattened_srt.resize(mapping_offset); } else { - auto& root = specialization.images[image_index]; - root.indirect_root = image_index; + auto& root = specializations[resource_index]; + root.indirect_root = resource_index; root.indirect_mapping_offset = static_cast(mapping_offset); root.indirect_search_iterations = std::bit_width(key_count); } @@ -859,10 +902,13 @@ ResourcePlan ExtractResourcePlan(const Program& program) { for (const auto& source: program.descriptor_sources) { auto& target = plan.descriptor_sources.emplace_back(); target.dword_count = source.dword_count; - target.indirect_image = source.indirect_image; - if (target.indirect_image.has_value()) { - target.indirect_image->key_count = Clone(target.indirect_image->key_count); - target.indirect_image->selector_mask = Clone(target.indirect_image->selector_mask); + target.indirect_descriptor = source.indirect_descriptor; + if (target.indirect_descriptor.has_value()) { + target.indirect_descriptor->key_count = Clone(target.indirect_descriptor->key_count); + target.indirect_descriptor->selector_first = + Clone(target.indirect_descriptor->selector_first); + target.indirect_descriptor->selector_mask = + Clone(target.indirect_descriptor->selector_mask); } for (uint32_t dword = 0; dword < source.dword_count; dword++) { target.dwords[dword] = Clone(source.dwords[dword]); @@ -909,27 +955,26 @@ ResourcePlan ExtractResourcePlan(const Program& program) { plan.uniform_fill.values[i] = Clone(plan.uniform_fill.values[i]); } plan.clean_flat_slots.resize(plan.srt_reads.size()); - bool capture_image_reads = false; - for (const auto& image: plan.info.images) { - const auto* source = Source(plan, image.source); - if (source == nullptr || !source->indirect_image.has_value()) { - continue; - } + bool capture_indirect_reads = false; + for (const auto& source: plan.descriptor_sources) { + if (!source.indirect_descriptor.has_value()) continue; + const auto& indirect = *source.indirect_descriptor; plan.requires_specialization_memory = true; - capture_image_reads |= !source->indirect_image->selector_mask.IsEmpty() || - !source->indirect_image->sources.empty(); - MarkCleanFlatSlots(plan, Source(plan, source->indirect_image->material_source), - plan.clean_flat_slots, source->indirect_image->selector_mask); - if (source->indirect_image->sources.empty()) { - MarkCleanFlatSlots(plan, Source(plan, source->indirect_image->table_source), - plan.clean_flat_slots); + capture_indirect_reads |= !indirect.selector_mask.IsEmpty() || + !indirect.selector_first.IsEmpty() || !indirect.sources.empty(); + MarkCleanFlatSlots(plan, Source(plan, indirect.material_source), plan.clean_flat_slots, + indirect.selector_mask); + MarkCleanFlatSlots(plan, nullptr, plan.clean_flat_slots, indirect.selector_first); + MarkCleanFlatSlots(plan, nullptr, plan.clean_flat_slots, indirect.key_count); + if (indirect.sources.empty()) { + MarkCleanFlatSlots(plan, Source(plan, indirect.table_source), plan.clean_flat_slots); } else { - for (const auto candidate: source->indirect_image->sources) { + for (const auto candidate: indirect.sources) { MarkCleanFlatSlots(plan, Source(plan, candidate), plan.clean_flat_slots); } } } - plan.capture_specialization_reads = capture_image_reads || !plan.control_flow.empty(); + plan.capture_specialization_reads = capture_indirect_reads || !plan.control_flow.empty(); // Writable descriptor addresses must come from clean backing for the alias proof. for (const auto& buffer: plan.info.buffers) { if (buffer.written) MarkCleanFlatSlots(plan, Source(plan, buffer.source), plan.clean_flat_slots); @@ -937,7 +982,7 @@ ResourcePlan ExtractResourcePlan(const Program& program) { for (const auto& image: plan.info.images) { if (image.written) MarkCleanFlatSlots(plan, Source(plan, image.source), plan.clean_flat_slots); } - if (capture_image_reads) plan.resource_tracking_complete &= !program.has_address_writes; + if (capture_indirect_reads) plan.resource_tracking_complete &= !program.has_address_writes; return plan; } @@ -988,12 +1033,27 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime return true; }; snapshot.buffers.resize(program.info.buffers.size()); - specialization.buffers.clear(); - specialization.buffers.reserve(program.info.buffers.size()); - for (uint32_t i = 0; i < program.info.buffers.size(); ++i) { - if (!evaluate(program.info.buffers[i].source, snapshot.buffers[i], - program.info.buffers[i].written)) { - return false; + specialization.buffers.assign(program.info.buffers.size(), {}); + for (uint32_t i = 0; i < snapshot.buffers.size(); ++i) { + const auto base_index = + i < program.info.buffers.size() ? i : specialization.buffers[i].indirect_root; + if (base_index >= program.info.buffers.size()) return false; + const auto& base = program.info.buffers[base_index]; + if (i < program.info.buffers.size()) { + const auto* source = Source(program, base.source); + if (source == nullptr) return false; + if (source->indirect_descriptor.has_value()) { + snapshot.buffers[i] = {.dword_count = 4u}; + if (active.empty() || active[base.source]) { + if (!MaterializeIndirectDescriptor( + program, *source->indirect_descriptor, i, 4u, observed, clean, snapshot, + snapshot.buffers, specialization.buffers, ShaderInfo::MaxBuffers, + NormalizeIndirectStoreBuffer)) + return false; + } + } else if (!evaluate(base.source, snapshot.buffers[i], base.written)) { + return false; + } } auto& descriptor_value = snapshot.buffers[i]; ShaderBufferResource descriptor; @@ -1012,15 +1072,12 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime } else if (!swizzle) { packed_stride &= ~(3u << 16u); } - specialization.buffers.push_back({ - .packed_stride = packed_stride, - .descriptor_format = program.info.buffers[i].formatted - ? descriptor.Format() - : Prospero::BufferFormat::kInvalid, - .descriptor_swizzle = - program.info.buffers[i].formatted ? descriptor.DstSelXYZW() : DstSel(4, 5, 6, 7), - .zero_stride_oob = descriptor.OutOfBounds() == 0u && stride == 0u, - }); + auto& buffer = specialization.buffers[i]; + buffer.packed_stride = packed_stride; + buffer.descriptor_format = + base.formatted ? descriptor.Format() : Prospero::BufferFormat::kInvalid; + buffer.descriptor_swizzle = base.formatted ? descriptor.DstSelXYZW() : DstSel(4, 5, 6, 7); + buffer.zero_stride_oob = descriptor.OutOfBounds() == 0u && stride == 0u; } snapshot.images.resize(program.info.images.size()); specialization.images.resize(program.info.images.size()); @@ -1041,13 +1098,19 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime if (source == nullptr) { return false; } - if (source->indirect_image.has_value()) { + if (source->indirect_descriptor.has_value()) { snapshot.images[i] = {.dword_count = 8u}; if (!active.empty() && !active[image.source]) { continue; } - const auto& indirect = *source->indirect_image; - if (!MaterializeIndirectImage(program, indirect, i, observed, clean, snapshot, specialization)) { + const auto& indirect = *source->indirect_descriptor; + if (!MaterializeIndirectDescriptor( + program, indirect, i, 8u, observed, clean, snapshot, snapshot.images, + specialization.images, ShaderInfo::MaxImages, [&](DescriptorValue& value) { + if (NullImageDescriptor(value) || !ValidImageDescriptor(value, image.r128)) + value.dwords.fill(0u); + return true; + })) { return false; } } else { @@ -1082,14 +1145,33 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime void ApplyResourceSpecialization(Program& program, const ResourceSpecialization& specialization) { EXIT_IF(!program.resource_tracking_complete || program.shader_info_complete || program.binding_layout_complete); - EXIT_IF(program.info.buffers.size() != specialization.buffers.size() || + EXIT_IF(program.info.buffers.size() > specialization.buffers.size() || program.info.images.size() > specialization.images.size()); auto& buffers = program.info.buffers; - for (size_t index = 0; index < buffers.size(); index++) { - buffers[index].packed_stride = specialization.buffers[index].packed_stride; - buffers[index].descriptor_format = specialization.buffers[index].descriptor_format; - buffers[index].descriptor_swizzle = specialization.buffers[index].descriptor_swizzle; + const auto original_buffer_count = buffers.size(); + buffers.reserve(specialization.buffers.size()); + for (uint32_t index = 0; index < specialization.buffers.size(); ++index) { + const auto& source = specialization.buffers[index]; + if (index >= buffers.size()) { + EXIT_IF(source.indirect_root >= original_buffer_count); + buffers.push_back(buffers[source.indirect_root]); + } + auto& buffer = buffers[index]; + buffer.packed_stride = source.packed_stride; + buffer.descriptor_format = source.descriptor_format; + buffer.descriptor_swizzle = source.descriptor_swizzle; + buffer.indirect_root = source.indirect_root; + buffer.indirect_mapping_offset = source.indirect_mapping_offset; + buffer.indirect_search_iterations = source.indirect_search_iterations; + buffer.indirect_resources.clear(); + } + for (uint32_t index = 0; index < buffers.size(); ++index) { + const auto root = buffers[index].indirect_root; + if (root != BufferResource::NoIndirectBuffer) { + EXIT_IF(root >= buffers.size()); + buffers[root].indirect_resources.push_back(index); + } } auto& images = program.info.images; const auto original_image_count = images.size(); diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.h b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.h index 7f9e6125..6ad72cb3 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.h +++ b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.h @@ -13,6 +13,9 @@ struct ResourceSpecialization { Prospero::BufferFormat descriptor_format = Prospero::BufferFormat::kInvalid; uint32_t descriptor_swizzle = DstSel(4, 5, 6, 7); bool zero_stride_oob = false; + uint32_t indirect_root = BufferResource::NoIndirectBuffer; + uint32_t indirect_mapping_offset = 0; + uint32_t indirect_search_iterations = 0; bool operator==(const Buffer&) const = default; }; diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp index 33913c2d..b3f8e02b 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp @@ -130,6 +130,58 @@ public: return true; } + bool SelectsRange(const Inst& selected, Value active, const Inst& candidate, + const Inst& carried, const Inst& mask, + std::span> exit_conditions) { + // Inactive lanes retain a previously selected index. Initially every lane is + // active; the bounded assignment establishes the invariant on each backedge. + m_exclude_bound = false; + std::vector range_values; + const auto in_range = [&] { + const auto value = Unknown(Type::U32); + range_values.push_back(value); + return value; + }; + const auto contains = [&](const auto& self, uint32_t node) -> uint32_t { + if (std::ranges::find(range_values, node) != range_values.end()) return 1u; + const auto entry = m_nodes[node]; + if (entry.variable == UINT32_MAX) return 0u; + return Select(NodeFor(entry.variable, 0u, 1u), self(self, entry.high), + self(self, entry.low)); + }; + const auto state = [&](bool in_body) { + m_values[1].clear(); + const auto enabled = Unknown(Type::U1); + for (const auto& phi: *m_induction.Parent()) { + if (phi.GetOpcode() != ValueOpcode::Phi) continue; + m_values[1][&phi] = EquivalentValue(m_program, Value(const_cast(&phi)), + Value(const_cast(&mask))) + ? enabled : Unknown(phi.GetType()); + } + m_values[1][&carried] = Select(enabled, Unknown(Type::U32), in_range()); + if (in_body) m_values[1][&m_bound] = 1u; + m_values[1][&candidate] = Select(Evaluate(Value(const_cast(&m_bound)), true), + in_range(), Unknown(Type::U32)); + }; + for (uint32_t arm = 0; arm < carried.NumArgs(); ++arm) { + const auto initial = mask.Arg(arm).Resolve(); + if (initial.IsImmediate() && initial.GetType() == Type::U1 && initial.U1()) continue; + state(true); + const auto retained = contains(contains, Evaluate(carried.Arg(arm), true)); + if (Select(Evaluate(mask.Arg(arm), true), 1u, retained) != 1u) return false; + } + for (uint32_t arm = 0; arm < selected.NumArgs(); ++arm) { + state(false); + const auto value = Evaluate(selected.Arg(arm), true); + m_values[1][&selected] = value; + auto allowed = Evaluate(exit_conditions[arm].first, true); + if (!exit_conditions[arm].second) allowed = Select(allowed, 0u, 1u); + allowed = Select(allowed, Evaluate(active, true), 0u); + if (Select(allowed, contains(contains, value), 1u) != 1u) return false; + } + return true; + } + private: struct Node { uint32_t variable = UINT32_MAX; @@ -171,6 +223,10 @@ private: } uint32_t Compare(const Inst& inst, uint32_t left, uint32_t right) { + if (left == right) { + if (inst.GetOpcode() == ValueOpcode::IEqual32) return 1u; + if (inst.GetOpcode() == ValueOpcode::INotEqual32) return 0u; + } const auto key = std::tuple {inst.GetOpcode(), inst.Flags(), left, right}; if (const auto found = m_predicates.find(key); found != m_predicates.end()) return found->second; const auto variable = std::min(m_nodes[left].variable, m_nodes[right].variable); @@ -202,7 +258,7 @@ private: return node; } const auto* inst = value.TryInstruction(); - if (current && inst == &m_bound) return 0u; + if (current && m_exclude_bound && inst == &m_bound) return 0u; auto& values = m_values[current]; if (const auto found = values.find(inst); found != values.end()) { if (found->second == UINT32_MAX) found->second = Unknown(value.GetType()); @@ -268,6 +324,7 @@ private: const Inst& m_bound; const Block* m_incoming = nullptr; uint32_t m_variables = 0; + bool m_exclude_bound = true; std::vector m_nodes {{}, {}}; std::vector> m_literals; std::array, 2> m_values; @@ -296,7 +353,7 @@ public: } PlanScalarReads(); EliminateDeadCode(m_program.blocks); - PlanIndirectImages(); + PlanIndirectDescriptors(); for (auto* block: m_program.blocks) { for (auto& inst: *block) { Collect(inst); @@ -313,17 +370,21 @@ public: memory.sampler = patch.sampler; } } - for (const auto& plan: m_indirect_images) { + for (const auto& plan: m_indirect_descriptors) { plan.handle->SetArg(0, plan.key); - for (uint32_t dword = 0; dword < 4u; dword++) { - plan.handle->SetArg(dword + 1u, plan.roots[dword + 4u]); - } - for (uint32_t dword = 5u; dword < plan.roots.size(); dword++) { - plan.handle->SetArg(dword, plan.key); + if (plan.handle->GetOpcode() == ValueOpcode::GetBufferResource) { + for (uint32_t word = 1; word < 4u; ++word) plan.handle->SetArg(word, Value(0u)); + } else { + for (uint32_t dword = 0; dword < 4u; dword++) { + plan.handle->SetArg(dword + 1u, plan.roots[dword + 4u]); + } + for (uint32_t dword = 5u; dword < plan.roots.size(); dword++) { + plan.handle->SetArg(dword, plan.key); + } } if (plan.reads[0] != nullptr) { - for (const auto index: plan.memory) - m_program.memory_info[index].planning_only = true; + for (uint32_t word = 0; word < plan.handle->NumArgs(); ++word) + m_program.memory_info[plan.memory[word]].planning_only = true; } } m_program.descriptor_sources = std::move(m_sources); @@ -351,7 +412,7 @@ private: Value planning_handle; }; - struct IndirectImagePlan { + struct IndirectDescriptorPlan { Inst* handle = nullptr; uint32_t source = 0; Value key; @@ -755,16 +816,19 @@ private: for (uint32_t candidate = 0; candidate < m_sources.size(); candidate++) { const auto& current = m_sources[candidate]; if (current.dword_count != descriptor.dword_count || - current.indirect_image.has_value() != descriptor.indirect_image.has_value()) { + current.indirect_descriptor.has_value() != descriptor.indirect_descriptor.has_value()) { continue; } - if (current.indirect_image.has_value()) { - const auto& a = *current.indirect_image; - const auto& b = *descriptor.indirect_image; + if (current.indirect_descriptor.has_value()) { + const auto& a = *current.indirect_descriptor; + const auto& b = *descriptor.indirect_descriptor; if (a.material_source != b.material_source || a.table_source != b.table_source || a.selector_stride != b.selector_stride || a.selector_offset != b.selector_offset || a.table_offset != b.table_offset || a.sources != b.sources || !EquivalentValue(m_program, a.key_count, b.key_count) || + a.selector_first.IsEmpty() != b.selector_first.IsEmpty() || + (!a.selector_first.IsEmpty() && + !EquivalentValue(m_program, a.selector_first, b.selector_first)) || a.selector_mask.IsEmpty() != b.selector_mask.IsEmpty() || (!a.selector_mask.IsEmpty() && !EquivalentValue(m_program, a.selector_mask, b.selector_mask))) continue; @@ -928,7 +992,7 @@ private: return false; } - bool MatchTableOffset(Value value, Value& key, uint32_t& offset) const { + bool MatchTableOffset(Value value, Value& key, uint32_t& offset, uint32_t shift = 5u) const { offset = 0; for (;;) { const auto* inst = value.Resolve().TryInstruction(); @@ -937,7 +1001,7 @@ private: } uint32_t immediate; if (inst->GetOpcode() == ValueOpcode::ShiftLeftLogical32 && - ImmediateU32(inst->Arg(1), immediate) && immediate == 5u) { + ImmediateU32(inst->Arg(1), immediate) && immediate == shift) { key = inst->Arg(0).Resolve(); return key.GetType() == Type::U32; } @@ -1241,9 +1305,7 @@ private: return {}; } - bool MatchUniformizedMaterialKey(Value key, const Inst& image, - DescriptorSource::IndirectImage& indirect, - DescriptorSource& material_source) { + const Inst* UniformizedMaterialValue(Value key, const Inst& use, Value executing = {}) const { Value guard; Value local; const auto* first = key.Resolve().TryInstruction(); @@ -1251,7 +1313,9 @@ private: first->NumArgs() == 2u) { guard = first->Arg(1).Resolve(); // Empty EXEC selects lane zero, which may not have loaded a material key. - if (!HasActiveLane(guard, first->Parent())) return false; + // An executing store also witnesses nonempty EXEC at the lane read. + if (executing.IsEmpty() ? !HasActiveLane(guard, first->Parent()) + : !Implies(executing, guard)) return nullptr; local = first->Arg(0).Resolve(); const auto* phi = guard.TryInstruction(); if (phi != nullptr && phi->GetOpcode() == ValueOpcode::Phi && @@ -1265,13 +1329,21 @@ private: } } } else { - guard = PositiveLaneWitness(image.Parent()); - if (guard.IsEmpty()) return false; + guard = PositiveLaneWitness(use.Parent()); + if (guard.IsEmpty()) return nullptr; local = EqualLocalKey(guard, key); } const auto* selected = local.Resolve().TryInstruction(); if (selected == nullptr || selected->GetOpcode() != ValueOpcode::SelectU32 || - selected->NumArgs() != 3u || !Implies(guard, selected->Arg(0))) return false; + selected->NumArgs() != 3u || !Implies(guard, selected->Arg(0))) return nullptr; + return selected; + } + + bool MatchUniformizedMaterialKey(Value key, const Inst& image, + DescriptorSource::IndirectDescriptor& indirect, + DescriptorSource& material_source) { + const auto* selected = UniformizedMaterialValue(key, image); + if (selected == nullptr) return false; const auto active = selected->Arg(0).Resolve(); const auto* read = selected->Arg(1).Resolve().TryInstruction(); if (read == nullptr || read->GetOpcode() != ValueOpcode::LoadAddressU32 || @@ -1303,9 +1375,9 @@ private: return true; } - Value BoundedLoopCount(Value key, const Block* use) const { + const Inst* BoundedLoop(Value key, const Block* use) const { const auto* phi = key.Resolve().TryInstruction(); - if (m_shader_writes || phi == nullptr || phi->GetOpcode() != ValueOpcode::Phi || + if (phi == nullptr || phi->GetOpcode() != ValueOpcode::Phi || phi->GetType() != Type::U32 || phi->NumArgs() != 2u || m_program.blocks.size() != m_program.block_info.size()) return {}; const Block* increment_block = nullptr; @@ -1342,8 +1414,9 @@ private: }; for (const auto& use_of_key: phi->Uses()) { const auto* compare = use_of_key.user; - if (compare->GetOpcode() != ValueOpcode::SLessThan32 || use_of_key.operand != 0u || - !ValidateRuntimeValue(m_program, compare->Arg(1))) continue; + if ((compare->GetOpcode() != ValueOpcode::SLessThan32 && + compare->GetOpcode() != ValueOpcode::ULessThan32) || use_of_key.operand != 0u || + !ValidateRuntimeValue(m_program, compare->Arg(1), RuntimeValueType::Integer)) continue; if (!guarded_on_entry(use, [&](const EdgePredicate& edge) { return edge.positive && Implies(edge.condition, Value(use_of_key.user)); })) continue; @@ -1352,19 +1425,75 @@ private: // skipped image alone does not prevent signed induction wraparound. if (guarded_on_entry(increment_block, [&](const EdgePredicate& edge) { return proof.Excludes(edge.condition, edge.positive); - })) return compare->Arg(1); + })) return compare; } return {}; } - bool TryMakeIndirectImage(Inst& handle, IndirectImagePlan& plan) { - if (handle.GetOpcode() != ValueOpcode::GetImageResource || handle.NumArgs() != 8u) { - return false; + bool BoundedSelectedIndex(Value index, Value active, + DescriptorSource::IndirectDescriptor& indirect) const { + const auto* selected = index.Resolve().TryInstruction(); + if (selected == nullptr || selected->GetOpcode() != ValueOpcode::Phi || + selected->NumArgs() == 0u) return false; + std::vector values {selected}; + for (size_t i = 0; i < values.size(); ++i) { + const auto* value = values[i]; + if (value->GetOpcode() != ValueOpcode::Phi && + value->GetOpcode() != ValueOpcode::SelectU32) continue; + for (size_t arm = value->GetOpcode() == ValueOpcode::SelectU32 ? 1u : 0u; + arm < value->NumArgs(); ++arm) { + const auto* next = value->Arg(arm).Resolve().TryInstruction(); + if (next != nullptr && std::ranges::find(values, next) == values.end()) + values.push_back(next); + } } + for (const auto* candidate: values) { + if (candidate->GetOpcode() != ValueOpcode::IAdd32) continue; + for (uint32_t arg = 0; arg < 2u; ++arg) { + const auto* bound = BoundedLoop(candidate->Arg(arg), candidate->Parent()); + const auto first = candidate->Arg(arg ^ 1u); + if (bound == nullptr || !ValidateRuntimeValue(m_program, first, RuntimeValueType::Integer)) + continue; + const auto* induction = candidate->Arg(arg).ResolveInstruction(); + const Inst* carried = nullptr; + for (const auto* value: values) { + if (value->GetOpcode() == ValueOpcode::Phi && value->Parent() == induction->Parent()) { + if (carried != nullptr) { carried = nullptr; break; } + carried = value; + } + } + if (carried == nullptr || carried->NumArgs() != 2u || + carried->PhiBlock(0) != induction->PhiBlock(0) || + carried->PhiBlock(1) != induction->PhiBlock(1)) continue; + std::vector> exits; + for (size_t arm = 0; arm < selected->NumArgs(); ++arm) { + const auto edge = ConditionalEdge(selected->PhiBlock(arm), selected->Parent()); + if (!edge || edge->lanes != LaneQuantifier::All) break; + exits.emplace_back(edge->condition, edge->positive); + } + if (exits.size() != selected->NumArgs()) continue; + for (const auto& mask: *induction->Parent()) { + if (mask.GetOpcode() != ValueOpcode::Phi || mask.GetType() != Type::U1 || + mask.NumArgs() != 2u || mask.PhiBlock(0) != carried->PhiBlock(0) || + mask.PhiBlock(1) != carried->PhiBlock(1)) continue; + const auto initial = induction->Arg(0).Resolve().IsImmediate() ? 0u : 1u; + const auto enabled = mask.Arg(initial).Resolve(); + if (!enabled.IsImmediate() || enabled.GetType() != Type::U1 || !enabled.U1()) continue; + LoopBoundProof proof(m_program, *induction, *bound); + if (!proof.SelectsRange(*selected, active, *candidate, *carried, mask, exits)) continue; + indirect.selector_first = first; + indirect.key_count = bound->Arg(1); + return true; + } + } + } + return false; + } + + bool MatchDescriptorTable(Inst& handle, IndirectDescriptorPlan& plan, + DescriptorSource& table_source, Value& key, uint32_t& table_offset) { Inst* table_handle = nullptr; - Value key; - uint32_t table_offset = 0; - for (uint32_t dword = 0; dword < plan.reads.size(); ++dword) { + for (uint32_t dword = 0; dword < handle.NumArgs(); ++dword) { auto* read = handle.Arg(dword).Resolve().TryInstruction(); if (read == nullptr) { return false; @@ -1385,7 +1514,7 @@ private: (memory->kind == ResourceKind::ScalarAddress && read->Parent() != handle.Parent()) || (table_handle != nullptr && !EquivalentValue(m_program, Value(table_handle), Value(current_handle))) || - !MatchTableOffset(read->Arg(1), current_key, offset) || + !MatchTableOffset(read->Arg(1), current_key, offset, handle.NumArgs() == 4u ? 4u : 5u) || memory->offset > UINT32_MAX - offset) { return false; } @@ -1399,19 +1528,33 @@ private: } table_handle = current_handle; const std::array image_users {&handle}; - if (!UsesOnly(*read, image_users)) { + if (handle.NumArgs() == 8u ? !UsesOnly(*read, image_users) : + std::ranges::any_of(read->Uses(), [](const Use& use) { + return use.user->GetOpcode() != ValueOpcode::GetBufferResource || + std::ranges::any_of(use.user->Uses(), [](const Use& consumer) { + return consumer.user->GetOpcode() != ValueOpcode::StoreBufferU32; + }); + })) { return false; } plan.memory[dword] = memory_index; plan.reads[dword] = read; } - DescriptorSource table_source; if (!MakeRuntimeTableSource(*plan.reads[0], table_source)) { return false; } + return true; + } + + bool TryMakeIndirectImage(Inst& handle, IndirectDescriptorPlan& plan) { + if (handle.GetOpcode() != ValueOpcode::GetImageResource || handle.NumArgs() != 8u) return false; + DescriptorSource table_source; + Value key; + uint32_t table_offset = 0; + if (!MatchDescriptorTable(handle, plan, table_source, key, table_offset)) return false; DescriptorSource material_source; - DescriptorSource::IndirectImage indirect; + DescriptorSource::IndirectDescriptor indirect; indirect.table_offset = table_offset; if (table_source.dword_count == 2u) { const auto* selector = key.Resolve().TryInstruction(); @@ -1420,8 +1563,9 @@ private: NonzeroOnEntry(selector->Arg(0), handle.Parent()); if (bitscan) { indirect.key_count = Value(32u); - } else { - indirect.key_count = BoundedLoopCount(key, handle.Parent()); + } else if (!m_shader_writes) { + if (const auto* bound = BoundedLoop(key, handle.Parent())) + indirect.key_count = bound->Arg(1); } if (indirect.key_count.IsEmpty() && !MatchUniformizedMaterialKey(key, handle, indirect, material_source)) { @@ -1467,7 +1611,7 @@ private: image_source.dwords.begin()); std::copy_n(table_source.dwords.begin(), table_source.dword_count, image_source.dwords.begin() + 4u); - image_source.indirect_image = indirect; + image_source.indirect_descriptor = indirect; plan.handle = &handle; plan.source = InternSource(image_source); plan.key = key; @@ -1475,6 +1619,66 @@ private: return true; } + bool TryMakeIndirectBuffer(Inst& handle, IndirectDescriptorPlan& plan) { + if (handle.GetOpcode() != ValueOpcode::GetBufferResource || handle.NumArgs() != 4u || + handle.Uses().empty()) return false; + for (const auto& use: handle.Uses()) { + const auto& inst = *use.user; + if (inst.GetOpcode() != ValueOpcode::StoreBufferU32) return false; + const auto& memory = m_program.memory_info[inst.Flags().index]; + uint32_t scalar = 1u; + if (memory.kind != ResourceKind::Buffer || memory.typed || memory.formatted || + memory.idxen || !memory.offen || !ImmediateU32(inst.Arg(3), scalar) || scalar != 0u) + return false; + } + DescriptorSource table_source; + Value key; + uint32_t table_offset = 0; + if (!MatchDescriptorTable(handle, plan, table_source, key, table_offset) || + table_source.dword_count != 4u || table_offset != 0u) return false; + const auto* lane_read = key.Resolve().TryInstruction(); + if (lane_read == nullptr || lane_read->GetOpcode() != ValueOpcode::ReadFirstLane) + return false; + const Inst* selected = nullptr; + for (const auto& use: handle.Uses()) { + const auto* value = UniformizedMaterialValue( + key, *use.user, use.user->Arg(use.user->NumArgs() - 1u)); + if (value == nullptr || (selected != nullptr && selected != value)) return false; + selected = value; + } + const auto* read = selected->Arg(1).Resolve().TryInstruction(); + uint32_t lane = 0; + if (read != nullptr && read->GetOpcode() == ValueOpcode::CompositeExtractU32x2) { + if (!ImmediateU32(read->Arg(1), lane) || lane >= 2u) return false; + read = read->Arg(0).Resolve().TryInstruction(); + if (read == nullptr || read->GetOpcode() != ValueOpcode::LoadBufferU32x2) return false; + } else if (read == nullptr || read->GetOpcode() != ValueOpcode::LoadBufferU32) { + return false; + } + const auto& memory = m_program.memory_info[read->Flags().index]; + uint32_t offset = 1u, scalar = 1u; + if (memory.kind != ResourceKind::Buffer || memory.typed || memory.formatted || + !memory.idxen || memory.offen || memory.offset > UINT32_MAX - lane * 4u || + !ImmediateU32(read->Arg(2), offset) || offset != 0u || + !ImmediateU32(read->Arg(3), scalar) || scalar != 0u || + !EquivalentValue(m_program, selected->Arg(0), read->Arg(4))) return false; + DescriptorSource material_source; + DescriptorSource::IndirectDescriptor indirect; + if (!MakeRuntimeTableSource(*read, material_source) || + !BoundedSelectedIndex(read->Arg(1), read->Arg(4), indirect)) return false; + indirect.material_source = InternSource(material_source); + indirect.table_source = InternSource(table_source); + indirect.selector_offset = memory.offset + lane * 4u; + DescriptorSource source; + source.dword_count = 4u; + source.dwords.fill(Value(0u)); + source.indirect_descriptor = indirect; + plan.handle = &handle; + plan.source = InternSource(source); + plan.key = key; + return true; + } + uint32_t SelectorMaximum(Value value) const { value = value.Resolve(); if (value.IsImmediate() && value.GetType() == Type::U32) return value.U32(); @@ -1515,7 +1719,7 @@ private: return found; } - bool TryMakeFiniteImage(Inst& handle, IndirectImagePlan& plan) { + bool TryMakeFiniteImage(Inst& handle, IndirectDescriptorPlan& plan) { if (handle.GetOpcode() != ValueOpcode::GetImageResource || handle.NumArgs() != 8u) return false; if (std::ranges::any_of(handle.Uses(), [](const Use& use) { @@ -1584,8 +1788,8 @@ private: DescriptorSource image_source; image_source.dword_count = 8u; image_source.dwords.fill(Value(0u)); - image_source.indirect_image.emplace(DescriptorSource::IndirectImage{}); - auto& sources = image_source.indirect_image->sources; + image_source.indirect_descriptor.emplace(DescriptorSource::IndirectDescriptor{}); + auto& sources = image_source.indirect_descriptor->sources; for (auto& choice: choices) { if (choice.branch != nullptr) continue; const auto source = InternSource(choice.descriptor); @@ -1621,37 +1825,40 @@ private: return true; } - const IndirectImagePlan* FindIndirectImage(const Inst& handle) const { + const IndirectDescriptorPlan* FindIndirectDescriptor(const Inst& handle) const { const auto found = - std::find_if(m_indirect_images.begin(), m_indirect_images.end(), - [&](const IndirectImagePlan& plan) { + std::find_if(m_indirect_descriptors.begin(), m_indirect_descriptors.end(), + [&](const IndirectDescriptorPlan& plan) { return plan.handle == &handle; }); - return found == m_indirect_images.end() ? nullptr : &*found; + return found == m_indirect_descriptors.end() ? nullptr : &*found; } bool IsIndirectPlanningMemory(uint32_t index) const { - return std::any_of(m_indirect_images.begin(), m_indirect_images.end(), - [&](const IndirectImagePlan& plan) { + return std::any_of(m_indirect_descriptors.begin(), m_indirect_descriptors.end(), + [&](const IndirectDescriptorPlan& plan) { return plan.reads[0] != nullptr && - std::ranges::find(plan.memory, index) != plan.memory.end(); + std::find(plan.memory.begin(), plan.memory.begin() + plan.handle->NumArgs(), index) != + plan.memory.begin() + plan.handle->NumArgs(); }); } - void PlanIndirectImages() { + void PlanIndirectDescriptors() { for (auto* block: m_program.blocks) { for (auto& inst: *block) { - if (ImageOpcodeInfoOf(inst.GetOpcode()).access == ImageAccess::None || + if ((ImageOpcodeInfoOf(inst.GetOpcode()).access == ImageAccess::None && + inst.GetOpcode() != ValueOpcode::StoreBufferU32) || inst.NumArgs() == 0u) { continue; } auto* handle = inst.Arg(0).Resolve().TryInstruction(); - if (handle == nullptr || FindIndirectImage(*handle) != nullptr) { + if (handle == nullptr || FindIndirectDescriptor(*handle) != nullptr) { continue; } - IndirectImagePlan plan; - if (TryMakeIndirectImage(*handle, plan) || TryMakeFiniteImage(*handle, plan)) { - m_indirect_images.push_back(std::move(plan)); + IndirectDescriptorPlan plan; + if (TryMakeIndirectImage(*handle, plan) || TryMakeFiniteImage(*handle, plan) || + TryMakeIndirectBuffer(*handle, plan)) { + m_indirect_descriptors.push_back(std::move(plan)); } } } @@ -1858,7 +2065,11 @@ private: uint32_t resource = 0; if (buffer != BufferAccess::None) { - if (!GetHandle(inst.Arg(0), ValueOpcode::GetBufferResource, 4, flags.pc, + handle = inst.Arg(0).Resolve().TryInstruction(); + const auto* indirect = handle == nullptr ? nullptr : FindIndirectDescriptor(*handle); + if (indirect != nullptr) { + source = indirect->source; + } else if (!GetHandle(inst.Arg(0), ValueOpcode::GetBufferResource, 4, flags.pc, memory.resource * 4u, handle, source)) { if (memory.kind != ResourceKind::Buffer || !memory.SupportsIndirectBufferLoad(op)) { Fail(flags.pc, @@ -1914,7 +2125,7 @@ private: Fail(flags.pc, "image operation has invalid resource kind"); } handle = inst.Arg(0).Resolve().TryInstruction(); - const auto* indirect = handle != nullptr ? FindIndirectImage(*handle) : nullptr; + const auto* indirect = handle != nullptr ? FindIndirectDescriptor(*handle) : nullptr; if (indirect != nullptr) { source = indirect->source; } else { @@ -1963,7 +2174,7 @@ private: for (uint32_t image = 0; image < m_info.images.size(); image++) { const auto* image_source = Source(m_info.images[image].source); if (image_source == nullptr || image_source->dword_count != 8 || - image_source->indirect_image.has_value()) { + image_source->indirect_descriptor.has_value()) { continue; } bool alias = true; @@ -1991,7 +2202,7 @@ private: std::vector m_sources; std::vector m_handle_patches; std::vector m_memory_patches; - std::vector m_indirect_images; + std::vector m_indirect_descriptors; std::vector> m_descriptor_selections; bool m_shader_writes = false; }; diff --git a/tests/KernelFileSystemTests.cpp b/tests/KernelFileSystemTests.cpp index 9253c519..201c3996 100644 --- a/tests/KernelFileSystemTests.cpp +++ b/tests/KernelFileSystemTests.cpp @@ -675,7 +675,7 @@ void CheckAmprOrdering(Loader::SymbolDatabase &symbols, uint32_t file_id) { "initialize AMPR command buffer"); } }; - // SDK WaitCompare order: ==, unsigned >/<, !=, wrapped >=, signed >/<. + // WaitCompare order: ==, unsigned >/<, !=, wrapped >=, signed >/<. struct Comparison { uint8_t compare; uint64_t blocked, reference, released; }; constexpr std::array comparisons { Comparison{0, 1, 2, 2}, Comparison{1, 0x40000000000019c3, 0x40000000000019c3, diff --git a/tests/ResourceTrackingTests.cpp b/tests/ResourceTrackingTests.cpp index d6f9a1a6..98272fd8 100644 --- a/tests/ResourceTrackingTests.cpp +++ b/tests/ResourceTrackingTests.cpp @@ -306,7 +306,7 @@ void TestInvariantIndirectImageMaterialization() { const auto source = fixture->program.info.images[0].source; Check(source < fixture->program.descriptor_sources.size() && fixture->program.descriptor_sources[source] - .indirect_image.has_value(), + .indirect_descriptor.has_value(), "indirect image source was not retained for runtime proof"); const auto image_handle = std::ranges::find_if(*fixture->block, [](const Inst &inst) { @@ -668,7 +668,7 @@ void TestGuardedDirectImageTable() { fixture.AddMemory(memory, 0x128)); fixture.PlanAndTrack(); const auto source = fixture.program.info.images[0].source; - const auto &indirect = fixture.program.descriptor_sources[source].indirect_image; + const auto &indirect = fixture.program.descriptor_sources[source].indirect_descriptor; Check(indirect && indirect->material_source == UINT32_MAX && indirect->selector_stride == 0u && indirect->table_offset == 344u && indirect->key_count.Resolve().IsImmediate() && @@ -966,7 +966,7 @@ void TestBoundedComputeImageLoop() { fixture.AddMemory(sample, 0x29c)); fixture.PlanAndTrack(); const auto source = fixture.program.info.images[0].source; - const auto &indirect = fixture.program.descriptor_sources[source].indirect_image; + const auto &indirect = fixture.program.descriptor_sources[source].indirect_descriptor; Check(indirect && indirect->material_source == UINT32_MAX && indirect->table_offset == 0x6b0u && indirect->key_count.Resolve() == count.Resolve(), @@ -1321,7 +1321,7 @@ void TestUniformizedMaterialImageKeys() { fixture.AddMemory(output_memory, 0x1b00u), done); fixture.PlanAndTrack(); const auto source = fixture.program.info.images[0].source; - const auto &indirect = fixture.program.descriptor_sources[source].indirect_image; + const auto &indirect = fixture.program.descriptor_sources[source].indirect_descriptor; Check(indirect && indirect->selector_stride == 0x90u && indirect->selector_offset == 0xc00u && indirect->table_offset == 0x20e0u && @@ -1340,7 +1340,7 @@ void TestUniformizedMaterialImageKeys() { plan = make_plan(Variant::FirstLaneAndNot); Check(plan.requires_specialization_memory && plan.descriptor_sources[plan.info.images[0].source] - .indirect_image->selector_mask.Resolve().TryInstruction() != nullptr, + .indirect_descriptor->selector_mask.Resolve().TryInstruction() != nullptr, "uniformized image mask did not survive extraction"); LinearTestMemory memory; memory.words.resize(0x23000u / 4u); @@ -2577,7 +2577,7 @@ void TestFiniteImagePhiCycle() { fixture.AddMemory(memory, 4)); fixture.PlanAndTrack(); const auto source = fixture.program.info.images[0].source; - const auto &finite = fixture.program.descriptor_sources[source].indirect_image; + const auto &finite = fixture.program.descriptor_sources[source].indirect_descriptor; Check(finite && finite->sources.size() == 2, "cyclic image selection did not retain two complete descriptor candidates"); const auto *key = image.Instruction()->Arg(0).Instruction(); @@ -2659,7 +2659,7 @@ void TestFiniteImageBitScanSentinel() { } fixture.PlanAndTrack(); const auto source = fixture.program.info.images[0].source; - const auto &finite = fixture.program.descriptor_sources[source].indirect_image; + const auto &finite = fixture.program.descriptor_sources[source].indirect_descriptor; Check(finite && finite->sources.size() == 1 && image.Instruction()->Arg(0).Instruction()->NumPhiBlocks() == 2, "nonzero bit scan retained its impossible sentinel or removed a Phi edge"); diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 7d9512d7..87e9ceb0 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -9202,7 +9202,7 @@ public: rendering.width == width && rendering.height == height && rendering.num_layers == view_layers && rendering.num_color_attachments == 1, - "SDK dimension=0 slice range did not create a Vulkan 1D-array " + "dimension=0 slice range did not create a Vulkan 1D-array " "attachment view"); RenderExecutorTestAccess::ResetBindings(executor); @@ -30569,6 +30569,117 @@ TestCase ImageD16StoreUnpacksHalfPairs() { return test; } +void CheckIndirectBufferStore(VulkanHarness &vulkan) { + using namespace ShaderRecompiler::IR; + TestCase test; + test.name = "IndirectBufferStore"; + test.initial.assign(64, 0xdeadbeefu); + test.storage_buffer_range_dwords = 16; + test.storage_buffer_offsets = {12, 108}; + test.required_spirv = {"OpSwitch"}; + test.forbidden_spirv = {"PhysicalStorageBuffer"}; + + CompiledShader compiled; + auto &program = compiled.program; + program.stage = ShaderType::Compute; + program.wave_size = 32; + program.srt_plan_complete = true; + program.resource_tracking_complete = true; + program.shader_info_complete = true; + program.block_storage.push_back(std::make_unique()); + auto *block = program.block_storage.back().get(); + program.blocks.push_back(block); + program.block_info.push_back({.id = 0}); + + auto &lane = block->AppendNewInst(ValueOpcode::LaneId); + auto &selector = block->AppendNewInst(ValueOpcode::BitwiseAnd32, + {Value(&lane), Value(7u)}); + auto &key = block->AppendNewInst(ValueOpcode::IAdd32, + {Value(&selector), Value(8u)}); + auto &handle = block->AppendNewInst( + ValueOpcode::GetBufferResource, + {Value(&key), Value(0u), Value(0u), Value(0u)}); + handle.SetFlags(0u); + auto &offset = block->AppendNewInst(ValueOpcode::IMul32, + {Value(&lane), Value(4u)}); + auto &data = block->AppendNewInst(ValueOpcode::IAdd32, + {Value(&lane), Value(0x1000u)}); + auto &exec = block->AppendNewInst(ValueOpcode::INotEqual32, + {Value(&lane), Value(4u)}); + block->AppendNewInst(ValueOpcode::StoreBufferU32, + {Value(&handle), Value(0u), Value(&offset), Value(0u), + Value(&data), Value(&exec)}); + program.memory_info.push_back({.kind = ResourceKind::Buffer, .offen = true}); + + BufferResource root; + root.packed_stride = 1; + root.written = true; + root.indirect_root = 0; + root.indirect_mapping_offset = 4; + root.indirect_search_iterations = std::bit_width(3u); + root.indirect_resources = {0u, 1u}; + auto candidate = root; + candidate.indirect_search_iterations = 0; + candidate.indirect_resources.clear(); + program.descriptor_sources.resize(1); + program.descriptor_sources[0].indirect_descriptor.emplace(); + ShaderComputeInputInfo compute; + compute.wave_size = 32; + compute.host_subgroup_size = vulkan.SubgroupSize(); + compute.threads_num[0] = 32; + compute.threads_num[1] = compute.threads_num[2] = 1; + for (const u32 null_buffers : {0u, 1u, 2u}) { + program.info.buffers = {root, candidate}; + if (null_buffers != 0u) program.info.buffers[1].packed_stride = 0; + if (null_buffers == 2u) { + // Materialization deduplicates all null destinations into one unmapped root. + program.info.buffers.resize(1); + program.info.buffers[0].packed_stride = 0; + program.info.buffers[0].indirect_root = BufferResource::NoIndirectBuffer; + program.info.buffers[0].indirect_search_iterations = 0; + program.info.buffers[0].indirect_resources.clear(); + } + program.binding_layout_complete = false; + AllocateBindings(program); + const auto *buffers = FindBinding(program.bindings, DescriptorBindingKind::Buffers); + Require(test.name, "candidate bindings", + buffers != nullptr && buffers->resources == + (null_buffers == 2u ? std::vector{0u} : std::vector{0u, 1u}), + "indirect store lost a buffer binding"); + compiled.packed_user_data.resize(program.bindings.ShaderDataDwords()); + compiled.packed_user_data[program.bindings.memory_offset_dword] = 12u | (108u << 8u); + compiled.spirv = ShaderRecompiler::Spirv::EmitProgram(program, {.compute = &compute}); + ValidateSpirv(test.name, compiled.spirv); + if (null_buffers == 2u) { + test.required_spirv.clear(); + test.forbidden_spirv.push_back("OpSwitch"); + } + CheckSpirvText(test, compiled.spirv); + + for (const bool swapped : {false, true}) { + if (null_buffers != 0u && swapped) continue; + // Repeated keys select two distinct logical buffers within one native allocation. + // Keys below, between and above the live keys must not select either buffer. + compiled.resources.flattened_srt = {0u, 0u, 0u, 0u, 3u, + 10u, swapped ? 0u : 1u, + 12u, swapped ? 1u : 0u, + 14u, swapped ? 0u : 1u}; + test.expected = test.initial; + for (const u32 lane : {2u, 6u, 10u, 12u, 14u}) { + const u32 resource = (lane & 7u) == 4u ? swapped : !swapped; + if (null_buffers == 2u || (null_buffers == 1u && resource == 1u)) continue; + test.expected[test.storage_buffer_offsets[resource] / 4u + lane] = 0x1000u + lane; + } + auto output = vulkan.CreateStorageBuffer(test.name, test.initial, test.initial.size()); + vulkan.Dispatch(test, compiled, output); + const auto actual = vulkan.ReadBuffer(test.name, output, test.expected.size()); + vulkan.DestroyBuffer(&output); + CompareWords(test, "key selection, EXEC, bounds and null descriptors", test.expected, actual); + } + } + std::printf("[compute] %-32s ok\n", test.name); +} + void CheckIndirectImageKeySwitch() { constexpr const char *name = "IndirectImageKeySwitch"; constexpr uint32_t mapping_capacity = 1793u; @@ -30621,8 +30732,8 @@ void CheckIndirectImageKeySwitch() { program.descriptor_sources.resize(2); program.descriptor_sources[0].dword_count = 8; - program.descriptor_sources[0].indirect_image = - DescriptorSource::IndirectImage{0u, 0u, 224u, 12u, 0u}; + program.descriptor_sources[0].indirect_descriptor = + DescriptorSource::IndirectDescriptor{0u, 0u, 224u, 12u, 0u}; program.descriptor_sources[1].dword_count = 4; ImageResource root{}; @@ -30656,7 +30767,7 @@ void CheckIndirectImageKeySwitch() { text.find("OpSwitch") != std::string::npos && text.find("OpPhi") != std::string::npos && CountText(text, "OpImageSampleExplicitLod") == 2 && - CountText(text, "OpIEqual") == 11, + CountText(text, "OpIEqual") == 1, "dynamic image key did not use a compact two-sample switch"); program.memory_info[0].image_dimension = @@ -37292,6 +37403,7 @@ int main(int argc, char **argv) { } if (argc == 2 && std::strcmp(argv[1], "--indirect-buffer-only") == 0) { VulkanHarness vulkan; + CheckIndirectBufferStore(vulkan); RunCase(&vulkan, BufferLoadsGpuSelectedDescriptors()); RunCase(&vulkan, BufferLoadDwordx3GpuSelectedDescriptors()); RunCase(&vulkan, BufferLoadDwordx4SnapshotsOverlappingAddress()); @@ -37857,6 +37969,7 @@ int main(int argc, char **argv) { CheckTessellationPrograms(); CheckPixelParameterAliases(); CheckRectListShaders(); + CheckIndirectBufferStore(vulkan); CheckIndirectImageKeySwitch(); CheckWave64WholeWaveResults(); CheckPs5GameExampleImageClearRuntimeShape(); diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index 6d398245..422ef515 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -8785,6 +8785,112 @@ void TestNativeGuardedSamplerSource() { } } +void TestBoundedMaterialBufferStores(bool scalar_key) { + // Captured batch search and descriptor waterfall, with deformation arithmetic removed. + const uint32_t shader[] = { + 0xbfa00003u, 0xd7460002u, 0x04010a04u, 0xf4080101u, + 0xfa000080u, 0xbe8e0380u, 0xbe8f037eu, 0xbf8cc07fu, + 0xf4240302u, 0xfa000014u, 0xf4080201u, 0xfa000020u, + 0xbf880019u, 0x7e3602c1u, 0xbf8cc07fu, 0xbf0a0d0eu, + 0xbf800000u, 0x8506807eu, 0xbeea0306u, 0xbefe0306u, + 0xbf860011u, 0x816b0e0cu, 0x810e810eu, 0x936aff6bu, + 0x00000050u, 0x7e36026bu, 0x816aa46au, 0xf4240104u, + 0xd4000000u, 0xbf8cc07fu, 0xbeea0404u, 0x816a0405u, + 0x7d8604f9u, 0x06868405u, 0x7d88046au, 0x876a6a04u, + 0x8a7e6a06u, 0xbf89ffe6u, 0xbefe030fu, 0x7daa36c1u, + 0xbf880013u, 0xbf8cc07fu, 0xe0342030u, 0x8002171bu, + 0xf4080100u, 0xfa000020u, 0x7e020281u, 0x7ed40518u, + 0xbeeb037eu, 0x7da4306au, 0x8f6a846au, 0x7e020280u, + 0xf4280002u, 0xd4000000u, 0xbf8cc07fu, 0xe0701000u, + 0x80003405u, 0xbefe036bu, 0x7daa0280u, 0xbf89fff3u, + 0xbf810000u, + }; + std::array material{}; + std::array table{}; + std::array batch{}; + std::array table_root{}; + std::array batch_root{}; + const auto buffer = [](uint32_t* words, const void* data, uint32_t stride, + uint32_t count, uint32_t mode = 0u) { + const auto address = reinterpret_cast(data); + words[0] = static_cast(address); + words[1] = static_cast(address >> 32u) | (stride << 16u); + words[2] = count; + words[3] = 0x5004u | (mode << 28u); + }; + batch[5] = 1u; + batch[6] = 2u; + material[20 + 9] = 32u; + material[20 + 13] = 3u; + material[40 + 9] = 32u; + material[40 + 10] = 32u; + material[40 + 13] = 5u; + buffer(table.data() + 12u, reinterpret_cast(0x100000u), 1u, 128u, 3u); + buffer(table.data() + 20u, reinterpret_cast(0x200000u), 1u, 128u, 3u); + buffer(table_root.data() + 8u, table.data(), 16u, 16u); + buffer(batch_root.data() + 8u, material.data(), 80u, 8u); + buffer(batch_root.data() + 32u, batch.data(), 16u, 2u); + const auto table_address = reinterpret_cast(table_root.data()); + const auto batch_address = reinterpret_cast(batch_root.data()); + const std::array user_data{ + static_cast(table_address), static_cast(table_address >> 32u), + static_cast(batch_address), static_cast(batch_address >> 32u)}; + auto options = MakeCompileOptions(ShaderType::Compute); + options.wave_size = 32u; + options.user_data = user_data; + auto code = std::to_array(shader); + if (scalar_key) { + code[0xa8u / 4u] = EncodeMubuf0(0x0cu, 52u); + code[0xacu / 4u] = EncodeMubuf1(24u, 2u, 27u); + } + auto translated = ShaderRecompiler::TranslateProgram(code, options); + auto plan = ShaderRecompiler::IR::ExtractResourcePlan(translated.program); + ShaderRecompiler::IR::ResourceSnapshot snapshot; + ShaderRecompiler::IR::ResourceSpecialization specialization; + const ShaderRecompiler::IR::SrtRuntime runtime{ + .user_data = user_data, .read_memory = ReadHostTestMemory, + .read_specialization_memory = ReadHostTestMemory}; + const auto materialize = [&] { + return ShaderRecompiler::IR::MaterializeResources(plan, runtime, snapshot, specialization); + }; + Check(materialize(), "bounded material range failed to materialize"); + const auto& buffers = specialization.buffers; + const auto root = std::ranges::find_if(buffers, [](const auto& resource) { + return resource.indirect_root != ShaderRecompiler::IR::BufferResource::NoIndirectBuffer; + }); + Check(root != buffers.end() && snapshot.flattened_srt[root->indirect_mapping_offset] == 2u && + buffers.size() == plan.info.buffers.size() + 1u, + "bounded material range did not select exactly its two writable buffers"); + const auto root_index = static_cast(root - buffers.begin()); + const auto first_specialization = specialization; + table[12] += 0x1000u; + Check(materialize() && specialization == first_specialization, + "buffer address changes created a shader permutation"); + batch[6] = 0u; + Check(materialize() && specialization.buffers[root_index].indirect_root == + ShaderRecompiler::IR::BufferResource::NoIndirectBuffer && + std::ranges::all_of(snapshot.buffers[root_index].dwords, + [](uint32_t word) { return word == 0u; }) && + specialization.buffers.size() == plan.info.buffers.size(), + "empty material batch retained writable candidates"); + batch[6] = 2u; + batch[5] = 7u; + Check(!materialize(), "material batch exceeded the descriptor record count"); + batch[5] = 1u; + batch_root[11] |= 1u << 30u; + Check(!materialize(), "material selection accepted a non-buffer descriptor type"); + batch_root[11] &= ~(1u << 30u); + Check(materialize(), "restored material batch failed to materialize"); + auto result = ShaderRecompiler::CompileProgram(std::move(translated), options, specialization, 0u); + CheckSpirvBinaryValidates(result.spirv); +#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS + auto unbounded = code; + unbounded[0x3cu / 4u] = EncodeSopc(0x0bu, 14u, 13u); // s_cmp_le_u32 admits wraparound. + ExpectFatal([&] { ShaderRecompiler::TranslateProgram(unbounded, options); }, + "material selection accepted an inclusive induction bound"); +#endif +} + void TestNativeDescriptorProvenanceRejectsGpuSelection() { #if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS const std::array selected{ @@ -14383,6 +14489,7 @@ int main() { TestNestedSelectionPreservesDescriptorSources(); TestNativeScalarReadDescriptorPlanning(); TestNativeGuardedSamplerSource(); + for (const bool scalar_key : {false, true}) TestBoundedMaterialBufferStores(scalar_key); TestNativeDescriptorProvenanceRejectsGpuSelection(); TestCfgSiblingSharedExit(); TestNewShaderRecompilerCfgNestedTailEarlyExit();