shifathosn
/
KytyPS5
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

shader: bind bounded GPU-selected buffer store destinations 

Co-authored-by: Ali Almohaya <AliAlmohaya@gmail.com>
This commit is contained in:
nmzik 2026-10-03 23:04:51 +04:00
parent db17585277
commit 429d0af62b
15 changed files with 784 additions and 211 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
3rdparty/cpuinfo/src/arm/mach/init.c vendored
View File

@ -14,7 +14,7 @@
#include <cpuinfo/internal-api.h> #include <cpuinfo/internal-api.h>
#include <cpuinfo/log.h> #include <cpuinfo/log.h>
/* Polyfill recent CPUFAMILY_ARM_* values for older SDKs */ /* Polyfill recent CPUFAMILY_ARM_* values for older platform headers */
#ifndef CPUFAMILY_ARM_MONSOON_MISTRAL #ifndef CPUFAMILY_ARM_MONSOON_MISTRAL
#define CPUFAMILY_ARM_MONSOON_MISTRAL 0xE81E7EF6 #define CPUFAMILY_ARM_MONSOON_MISTRAL 0xE81E7EF6
#endif #endif

5
src/graphics/shader/recompiler/ShaderRecompiler.cpp
View File

@ -648,6 +648,11 @@ CompileResult CompileProgram(TranslateResult translated, const CompileOptions& o
})) { })) {
continue; continue;
} }
const auto resource = inst.Flags<uint32_t>();
first = resource < ir.info.buffers.size() &&
ir.info.buffers[resource].indirect_root == resource
? 1u
: 0u;
} else if (op == IR::ValueOpcode::GetImageResource) { } else if (op == IR::ValueOpcode::GetImageResource) {
const auto resource = inst.Flags<uint32_t>(); const auto resource = inst.Flags<uint32_t>();
first = resource < ir.info.images.size() && first = resource < ir.info.images.size() &&

54
src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp
View File

@ -808,7 +808,7 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
const auto* source = image.source < state.program.descriptor_sources.size() const auto* source = image.source < state.program.descriptor_sources.size()
? &state.program.descriptor_sources[image.source] ? &state.program.descriptor_sources[image.source]
: nullptr; : nullptr;
if (handle == nullptr || source == nullptr || !source->indirect_image.has_value() || if (handle == nullptr || source == nullptr || !source->indirect_descriptor.has_value() ||
handle->NumArgs() == 0u) { handle->NumArgs() == 0u) {
ctx.Fail(inst, "has invalid indirect image key provenance"); ctx.Fail(inst, "has invalid indirect image key provenance");
return; return;
@ -819,56 +819,8 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
ctx.Fail(inst, "has no indirect image runtime mapping"); ctx.Fail(inst, "has no indirect image runtime mapping");
return; return;
} }
const auto LoadMapping = [&](uint32_t index) { const auto selected = EmitIndirectResourceIndex(
const auto pointer = state.builder.AllocateId(); state, key, image.indirect_mapping_offset, image.indirect_search_iterations, 0u);
state.builder.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state),
pointer, state.flattened_srt_variable, ConstantU32(state, 0),
index);
const auto value = state.builder.AllocateId();
state.builder.AddFunction(spv::OpLoad, TypeU32(state), value, pointer);
return value;
};
const auto mapping = ConstantU32(state, image.indirect_mapping_offset);
auto low = ConstantU32(state, 0u);
auto high = LoadMapping(mapping);
auto selected = ConstantU32(state, 0u);
for (uint32_t iteration = 0; iteration < image.indirect_search_iterations;
iteration++) {
const auto active = Binary(state, spv::OpULessThan, TypeBool(state), low, high);
const auto mid = Binary(state, spv::OpShiftRightLogical, TypeU32(state),
Binary(state, spv::OpIAdd, TypeU32(state), low, high),
ConstantU32(state, 1u));
const auto probe = state.builder.AllocateId();
state.builder.AddFunction(spv::OpSelect, TypeU32(state), probe, active, mid,
ConstantU32(state, 0u));
const auto entry = Binary(state, spv::OpIAdd, TypeU32(state), mapping,
Binary(state, spv::OpIAdd, TypeU32(state),
Binary(state, spv::OpShiftLeftLogical, TypeU32(state),
probe, ConstantU32(state, 1u)),
ConstantU32(state, 1u)));
const auto mapped_key = LoadMapping(entry);
const auto candidate = LoadMapping(
Binary(state, spv::OpIAdd, TypeU32(state), entry, ConstantU32(state, 1u)));
const auto equal = Binary(state, spv::OpIEqual, TypeBool(state), mapped_key, key);
const auto match = Binary(state, spv::OpLogicalAnd, TypeBool(state), active, equal);
const auto next_selected = state.builder.AllocateId();
state.builder.AddFunction(spv::OpSelect, TypeU32(state), next_selected, match,
candidate, selected);
selected = next_selected;
const auto less = Binary(state, spv::OpULessThan, TypeBool(state), mapped_key, key);
const auto take_upper = Binary(state, spv::OpLogicalAnd, TypeBool(state), active, less);
const auto take_lower = Binary(state, spv::OpLogicalAnd, TypeBool(state), active,
Unary(state, spv::OpLogicalNot, TypeBool(state), less));
const auto next_low = state.builder.AllocateId();
state.builder.AddFunction(
spv::OpSelect, TypeU32(state), next_low, take_upper,
Binary(state, spv::OpIAdd, TypeU32(state), mid, ConstantU32(state, 1u)), low);
low = next_low;
const auto next_high = state.builder.AllocateId();
state.builder.AddFunction(spv::OpSelect, TypeU32(state), next_high, take_lower, mid,
high);
high = next_high;
}
const auto default_label = state.builder.AllocateId(); const auto default_label = state.builder.AllocateId();
const auto merge_label = state.builder.AllocateId(); const auto merge_label = state.builder.AllocateId();
std::vector<uint32_t> labels(image.indirect_resources.size() - 1u); std::vector<uint32_t> labels(image.indirect_resources.size() - 1u);

2
src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h
View File

@ -408,6 +408,8 @@ struct MemoryResourceAccess {
spv::MemoryAccessMask memory_access = spv::MemoryAccessMaskNone; spv::MemoryAccessMask memory_access = spv::MemoryAccessMaskNone;
}; };
uint32_t EmitIndirectResourceIndex(EmitterState& state, uint32_t key, uint32_t mapping_offset,
uint32_t search_iterations, uint32_t default_resource);
MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::MemoryInfo& mem); MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::MemoryInfo& mem);
MemoryResourceAccess PrepareStorageBufferResourceAccess(EmitterState& state, MemoryResourceAccess PrepareStorageBufferResourceAccess(EmitterState& state,

48
src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp
View File

@ -549,9 +549,51 @@ void StoreWordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::Me
} }
void StoreWord(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem) { void StoreWord(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem) {
EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() { auto& state = ctx.state;
const auto resource = PrepareMemoryResourceAccess(ctx.state, mem); EmitIfCondition(state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
StoreWordPrepared(ctx, inst, mem, resource, ctx.Arg(inst, inst.NumArgs() - 2)); const auto data = ctx.Arg(inst, inst.NumArgs() - 2);
const auto store = [&](uint32_t resource) {
if (mem.kind == IR::ResourceKind::Buffer) {
const auto& buffer = state.program.info.buffers[resource];
if (buffer.packed_stride == 0u &&
buffer.source < state.program.descriptor_sources.size() &&
state.program.descriptor_sources[buffer.source].indirect_descriptor) return;
}
mem.resource = resource;
const auto access = PrepareMemoryResourceAccess(state, mem);
StoreWordPrepared(ctx, inst, mem, access, data);
};
if (mem.kind != IR::ResourceKind::Buffer ||
state.program.info.buffers[mem.resource].indirect_root != mem.resource) {
store(mem.resource);
return;
}
const auto& buffer = state.program.info.buffers[mem.resource];
const auto* handle = inst.Arg(0).ResolveInstruction();
if (handle == nullptr || buffer.indirect_resources.empty() ||
state.flattened_srt_variable == 0 || buffer.indirect_search_iterations == 0u) {
ctx.Fail(inst, "has no indirect buffer runtime mapping");
return;
}
const auto selected = EmitIndirectResourceIndex(
state, ctx.Arg(*handle, 0), buffer.indirect_mapping_offset,
buffer.indirect_search_iterations, UINT32_MAX);
const auto merge = state.builder.AllocateId();
std::vector<uint32_t> labels(buffer.indirect_resources.size());
std::vector<uint32_t> branches {spv::OpSwitch, selected, merge};
for (uint32_t i = 0; i < labels.size(); ++i) {
labels[i] = state.builder.AllocateId();
branches.push_back(i);
branches.push_back(labels[i]);
}
state.builder.AddFunction(spv::OpSelectionMerge, merge, spv::SelectionControlMaskNone);
state.builder.AddFunction(branches);
for (uint32_t i = 0; i < labels.size(); ++i) {
EmitLabel(state, labels[i]);
store(buffer.indirect_resources[i]);
state.builder.AddFunction(spv::OpBranch, merge);
}
EmitLabel(state, merge);
}); });
} }

45
src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp
View File

@ -31,6 +31,51 @@ uint32_t EmitBinaryU32(EmitterState& state, spv::Op opcode, uint32_t lhs, uint32
return ret; return ret;
} }
uint32_t EmitIndirectResourceIndex(EmitterState& state, uint32_t key, uint32_t mapping_offset,
uint32_t search_iterations, uint32_t default_resource) {
const auto LoadMapping = [&](uint32_t index) {
const auto pointer = state.builder.AllocateId();
state.builder.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state),
pointer, state.flattened_srt_variable, ConstantU32(state, 0),
index);
const auto value = state.builder.AllocateId();
state.builder.AddFunction(spv::OpLoad, TypeU32(state), value, pointer);
return value;
};
const auto mapping = ConstantU32(state, mapping_offset);
const auto count = LoadMapping(mapping);
const auto entry_at = [&](uint32_t index) {
return Binary(state, spv::OpIAdd, TypeU32(state), mapping,
Binary(state, spv::OpIAdd, TypeU32(state),
Binary(state, spv::OpShiftLeftLogical, TypeU32(state), index,
ConstantU32(state, 1u)), ConstantU32(state, 1u)));
};
auto low = ConstantU32(state, 0u);
auto high = count;
for (uint32_t iteration = 0; iteration < search_iterations; ++iteration) {
const auto active = Binary(state, spv::OpULessThan, TypeBool(state), low, high);
const auto mid = Binary(state, spv::OpShiftRightLogical, TypeU32(state),
Binary(state, spv::OpIAdd, TypeU32(state), low, high),
ConstantU32(state, 1u));
const auto probe = Select(state, TypeU32(state), active, mid, ConstantU32(state, 0u));
const auto less = Binary(state, spv::OpULessThan, TypeBool(state),
LoadMapping(entry_at(probe)), key);
low = Select(state, TypeU32(state),
Binary(state, spv::OpLogicalAnd, TypeBool(state), active, less),
Binary(state, spv::OpIAdd, TypeU32(state), mid, ConstantU32(state, 1u)), low);
high = Select(state, TypeU32(state),
Binary(state, spv::OpLogicalAnd, TypeBool(state), active,
Unary(state, spv::OpLogicalNot, TypeBool(state), less)), mid, high);
}
const auto in_range = Binary(state, spv::OpULessThan, TypeBool(state), low, count);
const auto entry = entry_at(Select(state, TypeU32(state), in_range, low, ConstantU32(state, 0u)));
const auto match = Binary(state, spv::OpLogicalAnd, TypeBool(state), in_range,
Binary(state, spv::OpIEqual, TypeBool(state), LoadMapping(entry), key));
const auto resource = LoadMapping(Binary(state, spv::OpIAdd, TypeU32(state), entry,
ConstantU32(state, 1u)));
return Select(state, TypeU32(state), match, resource, ConstantU32(state, default_resource));
}
uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryInfo& mem) { uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryInfo& mem) {
if (mem.resource >= state.program.info.buffers.size()) { if (mem.resource >= state.program.info.buffers.size()) {
ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers, mem.resource, ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers, mem.resource,

12
src/graphics/shader/recompiler/ir/ShaderIR.h
View File

@ -97,6 +97,7 @@ struct ExportInfo {
struct BufferResource { struct BufferResource {
static constexpr uint32_t NoImageAlias = UINT32_MAX; static constexpr uint32_t NoImageAlias = UINT32_MAX;
static constexpr uint32_t NoIndirectBuffer = UINT32_MAX;
uint32_t source = 0; uint32_t source = 0;
uint32_t first_use_pc = 0; uint32_t first_use_pc = 0;
@ -110,6 +111,10 @@ struct BufferResource {
bool atomic = false; bool atomic = false;
bool formatted = false; bool formatted = false;
bool scalar = false; bool scalar = false;
uint32_t indirect_root = NoIndirectBuffer;
uint32_t indirect_mapping_offset = 0;
uint32_t indirect_search_iterations = 0;
std::vector<uint32_t> indirect_resources;
bool operator==(const BufferResource& other) const = default; bool operator==(const BufferResource& other) const = default;
}; };
@ -472,22 +477,23 @@ struct BlockInfo {
}; };
struct DescriptorSource { struct DescriptorSource {
struct IndirectImage { struct IndirectDescriptor {
uint32_t material_source = UINT32_MAX; uint32_t material_source = UINT32_MAX;
uint32_t table_source = 0; uint32_t table_source = 0;
uint32_t selector_stride = 0; uint32_t selector_stride = 0;
uint32_t selector_offset = 0; uint32_t selector_offset = 0;
uint32_t table_offset = 0; uint32_t table_offset = 0;
Value key_count; Value key_count;
Value selector_first;
Value selector_mask; Value selector_mask;
std::vector<uint32_t> sources; std::vector<uint32_t> sources;
bool operator==(const IndirectImage& other) const = default; bool operator==(const IndirectDescriptor& other) const = default;
}; };
std::array<Value, 8> dwords {}; std::array<Value, 8> dwords {};
uint32_t dword_count = 0; uint32_t dword_count = 0;
std::optional<IndirectImage> indirect_image; std::optional<IndirectDescriptor> indirect_descriptor;
bool operator==(const DescriptorSource& other) const = default; bool operator==(const DescriptorSource& other) const = default;
}; };

11
src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp
View File

@ -74,6 +74,9 @@ bool CollectMemoryResources(const Program& program, std::vector<uint32_t>& buffe
} else if (memory.kind == ResourceKind::Buffer || memory.kind == ResourceKind::ScalarBuffer) { } else if (memory.kind == ResourceKind::Buffer || memory.kind == ResourceKind::ScalarBuffer) {
EXIT_IF(memory.resource >= program.info.buffers.size()); EXIT_IF(memory.resource >= program.info.buffers.size());
live_buffers.at(memory.resource) = true; live_buffers.at(memory.resource) = true;
for (const auto child: program.info.buffers[memory.resource].indirect_resources) {
live_buffers.at(child) = true;
}
} }
} }
} }
@ -86,13 +89,15 @@ bool CollectMemoryResources(const Program& program, std::vector<uint32_t>& buffe
} }
bool UsesFlattenedSrt(const Program& program) { bool UsesFlattenedSrt(const Program& program) {
const auto uses_mapping = [](const auto& resource) {
return resource.indirect_search_iterations != 0u;
};
return std::ranges::any_of(program.blocks, [](const Block* block) { return std::ranges::any_of(program.blocks, [](const Block* block) {
return std::ranges::any_of(*block, [](const Inst& inst) { return std::ranges::any_of(*block, [](const Inst& inst) {
return inst.GetOpcode() == ValueOpcode::ReadConst; return inst.GetOpcode() == ValueOpcode::ReadConst;
}); });
}) || std::ranges::any_of(program.info.images, [](const ImageResource& image) { }) || std::ranges::any_of(program.info.buffers, uses_mapping) ||
return image.indirect_search_iterations != 0u; std::ranges::any_of(program.info.images, uses_mapping);
});
} }
void AllocateBindings(Program& program, uint32_t push_data_start_dword) { void AllocateBindings(Program& program, uint32_t push_data_start_dword) {

234
src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp
View File

@ -20,7 +20,7 @@ namespace Libs::Graphics::ShaderRecompiler::IR {
namespace { namespace {
constexpr uint64_t AddressMask = 0x0000ffffffffffffull; constexpr uint64_t AddressMask = 0x0000ffffffffffffull;
constexpr uint64_t MaxIndirectImageProbes = 65536u; constexpr uint64_t MaxIndirectDescriptorProbes = 65536u;
bool SpecializationFail(std::string_view message) { bool SpecializationFail(std::string_view message) {
std::fprintf(stderr, "shader resource specialization failed: %.*s\n", std::fprintf(stderr, "shader resource specialization failed: %.*s\n",
@ -224,12 +224,29 @@ bool ReadScalarTable(uint64_t base, uint64_t size, uint64_t dynamic_offset,
runtime.read_specialization_memory(runtime.userdata, address, prefix); runtime.read_specialization_memory(runtime.userdata, address, prefix);
} }
bool MaterializeIndirectImage(const ResourcePlan& program, bool NormalizeIndirectStoreBuffer(DescriptorValue& value) {
const DescriptorSource::IndirectImage& indirect, ShaderBufferResource descriptor;
uint32_t image_index, if (!DecodeBufferDescriptor(value, descriptor)) return false;
const SrtRuntime& runtime, SrtWalker& clean, if (descriptor.Type() != 0u || descriptor.Format() == Prospero::BufferFormat::kInvalid ||
ResourceSnapshot& snapshot, descriptor.Base48() == 0u || descriptor.GetSize() == 0u) {
ResourceSpecialization& specialization) { value.dwords.fill(0u);
} else if (descriptor.Stride() != 1u || descriptor.SwizzleEnabled() || descriptor.AddTid() ||
descriptor.OutOfBounds() != 3u) {
return SpecializationFail("indirect buffer stores require linear byte bounds");
}
return true;
}
template <typename Specialization, typename Normalize>
bool MaterializeIndirectDescriptor(const ResourcePlan& program,
const DescriptorSource::IndirectDescriptor& indirect,
uint32_t resource_index, uint32_t dword_count,
const SrtRuntime& runtime, SrtWalker& clean,
ResourceSnapshot& snapshot,
std::vector<DescriptorValue>& descriptors,
std::vector<Specialization>& specializations,
uint32_t maximum_resources, Normalize&& normalize) {
const auto descriptor_bytes = dword_count * sizeof(uint32_t);
const auto& sources = indirect.sources; const auto& sources = indirect.sources;
auto& keys = program.material_keys; auto& keys = program.material_keys;
uint64_t table_base = 0; uint64_t table_base = 0;
@ -257,12 +274,39 @@ bool MaterializeIndirectImage(const ResourcePlan& program,
const bool evaluated = clean.Evaluate(indirect.key_count, key_count); const bool evaluated = clean.Evaluate(indirect.key_count, key_count);
if (std::bit_cast<int32_t>(key_count) <= 0) key_count = 0; if (std::bit_cast<int32_t>(key_count) <= 0) key_count = 0;
if (table_value.dword_count != 2u || !evaluated || if (table_value.dword_count != 2u || !evaluated ||
key_count > MaxIndirectImageProbes || key_count > MaxIndirectDescriptorProbes ||
uint64_t {indirect.table_offset} + uint64_t {key_count} * 32u > UINT32_MAX + 1ull) { uint64_t {indirect.table_offset} + uint64_t {key_count} * descriptor_bytes >
UINT32_MAX + 1ull) {
return false; return false;
} }
keys.resize(key_count); keys.resize(key_count);
std::iota(keys.begin(), keys.end(), 0u); std::iota(keys.begin(), keys.end(), 0u);
} else if (!indirect.selector_first.IsEmpty()) {
ShaderBufferResource material;
uint32_t first = 0, count = 0;
if (!DecodeBufferDescriptor(material_value, material) || material.Type() != 0u ||
material.SwizzleEnabled() ||
material.AddTid() || material.OutOfBounds() != 0u ||
uint64_t {indirect.selector_offset} + 4u > material.Stride() ||
!clean.Evaluate(indirect.selector_first, first) ||
!clean.Evaluate(indirect.key_count, count) || count > MaxIndirectDescriptorProbes ||
uint64_t {first} + count > material.NumRecords())
return false;
if (count != 0u && (uint64_t {first} + count - 1u) * material.Stride() +
indirect.selector_offset + 4u >
uint64_t {UINT32_MAX} + 1u)
return false;
keys.reserve(count);
for (uint64_t index = first; index < uint64_t {first} + count; ++index) {
uint32_t key = 0;
const auto offset = index * material.Stride() + indirect.selector_offset;
if (!ReadScalarTable(material.Base48(), material.GetSize(), offset, runtime,
{&key, 1}))
return false;
keys.push_back(key);
}
std::ranges::sort(keys);
keys.erase(std::unique(keys.begin(), keys.end()), keys.end());
} else if (!indirect.selector_mask.IsEmpty()) { } else if (!indirect.selector_mask.IsEmpty()) {
uint32_t mask = 0; uint32_t mask = 0;
uint32_t count = 0; uint32_t count = 0;
@ -301,7 +345,7 @@ bool MaterializeIndirectImage(const ResourcePlan& program,
const auto size = material.GetSize(); const auto size = material.GetSize();
const auto limit = std::min(first + (uint64_t {1} << 32u) - step, size >= 4u ? size - 4u : 0u); const auto limit = std::min(first + (uint64_t {1} << 32u) - step, size >= 4u ? size - 4u : 0u);
const auto probe_count = size >= 4u && first <= limit ? (limit - first) / step + 1u : 0u; const auto probe_count = size >= 4u && first <= limit ? (limit - first) / step + 1u : 0u;
if (probe_count > MaxIndirectImageProbes) { if (probe_count > MaxIndirectDescriptorProbes) {
return false; return false;
} }
keys.reserve(static_cast<size_t>(probe_count) + 1u); keys.reserve(static_cast<size_t>(probe_count) + 1u);
@ -318,53 +362,52 @@ bool MaterializeIndirectImage(const ResourcePlan& program,
} }
} }
const auto children_begin = snapshot.images.size(); const auto children_begin = descriptors.size();
const auto mapping_offset = snapshot.flattened_srt.size(); const auto mapping_offset = snapshot.flattened_srt.size();
const auto root_image = specialization.images[image_index]; const auto root_resource = specializations[resource_index];
const auto key_count = sources.empty() ? keys.size() : sources.size(); const auto key_count = sources.empty() ? keys.size() : sources.size();
snapshot.flattened_srt.resize(mapping_offset + 1u + key_count * 2u); snapshot.flattened_srt.resize(mapping_offset + 1u + key_count * 2u);
snapshot.flattened_srt[mapping_offset] = static_cast<uint32_t>(key_count); snapshot.flattened_srt[mapping_offset] = static_cast<uint32_t>(key_count);
for (uint32_t entry = 0; entry < key_count; ++entry) { for (uint32_t entry = 0; entry < key_count; ++entry) {
const auto key = sources.empty() ? keys[entry] : entry; const auto key = sources.empty() ? keys[entry] : entry;
DescriptorValue candidate; DescriptorValue candidate;
candidate.dword_count = 8u; candidate.dword_count = dword_count;
if (sources.empty()) { if (sources.empty()) {
const auto table_offset = (key << 5u) + indirect.table_offset; const auto table_offset =
if (!ReadScalarTable(table_base, table_size, table_offset, runtime, candidate.dwords)) { static_cast<uint32_t>(key * descriptor_bytes) + indirect.table_offset;
if (!ReadScalarTable(table_base, table_size, table_offset, runtime,
std::span(candidate.dwords).first(dword_count))) {
return false; return false;
} }
} else if (!clean.EvaluateDescriptor(sources[entry], candidate)) { } else if (!clean.EvaluateDescriptor(sources[entry], candidate)) {
return false; return false;
} }
if (NullImageDescriptor(candidate) || if (!normalize(candidate)) return false;
!ValidImageDescriptor(candidate, program.info.images[image_index].r128)) {
candidate.dwords.fill(0);
}
uint32_t ordinal = 0; uint32_t ordinal = 0;
if (entry == 0) { if (entry == 0) {
snapshot.images[image_index] = candidate; descriptors[resource_index] = candidate;
} else if (snapshot.images[image_index] != candidate) { } else if (descriptors[resource_index] != candidate) {
const auto found = std::find(snapshot.images.begin() + children_begin, const auto found =
snapshot.images.end(), candidate); std::find(descriptors.begin() + children_begin, descriptors.end(), candidate);
ordinal = static_cast<uint32_t>(found - snapshot.images.begin() - children_begin + 1u); ordinal = static_cast<uint32_t>(found - descriptors.begin() - children_begin + 1u);
if (found == snapshot.images.end()) { if (found == descriptors.end()) {
if (snapshot.images.size() >= ShaderInfo::MaxImages) { if (descriptors.size() >= maximum_resources) {
return false; return false;
} }
snapshot.images.push_back(candidate); descriptors.push_back(candidate);
auto child = root_image; auto child = root_resource;
child.indirect_root = image_index; child.indirect_root = resource_index;
specialization.images.push_back(child); specializations.push_back(child);
} }
} }
snapshot.flattened_srt[mapping_offset + 1u + entry * 2u] = key; snapshot.flattened_srt[mapping_offset + 1u + entry * 2u] = key;
snapshot.flattened_srt[mapping_offset + 2u + entry * 2u] = ordinal; snapshot.flattened_srt[mapping_offset + 2u + entry * 2u] = ordinal;
} }
if (snapshot.images.size() == children_begin) { if (descriptors.size() == children_begin) {
snapshot.flattened_srt.resize(mapping_offset); snapshot.flattened_srt.resize(mapping_offset);
} else { } else {
auto& root = specialization.images[image_index]; auto& root = specializations[resource_index];
root.indirect_root = image_index; root.indirect_root = resource_index;
root.indirect_mapping_offset = static_cast<uint32_t>(mapping_offset); root.indirect_mapping_offset = static_cast<uint32_t>(mapping_offset);
root.indirect_search_iterations = std::bit_width(key_count); root.indirect_search_iterations = std::bit_width(key_count);
} }
@ -859,10 +902,13 @@ ResourcePlan ExtractResourcePlan(const Program& program) {
for (const auto& source: program.descriptor_sources) { for (const auto& source: program.descriptor_sources) {
auto& target = plan.descriptor_sources.emplace_back(); auto& target = plan.descriptor_sources.emplace_back();
target.dword_count = source.dword_count; target.dword_count = source.dword_count;
target.indirect_image = source.indirect_image; target.indirect_descriptor = source.indirect_descriptor;
if (target.indirect_image.has_value()) { if (target.indirect_descriptor.has_value()) {
target.indirect_image->key_count = Clone(target.indirect_image->key_count); target.indirect_descriptor->key_count = Clone(target.indirect_descriptor->key_count);
target.indirect_image->selector_mask = Clone(target.indirect_image->selector_mask); target.indirect_descriptor->selector_first =
Clone(target.indirect_descriptor->selector_first);
target.indirect_descriptor->selector_mask =
Clone(target.indirect_descriptor->selector_mask);
} }
for (uint32_t dword = 0; dword < source.dword_count; dword++) { for (uint32_t dword = 0; dword < source.dword_count; dword++) {
target.dwords[dword] = Clone(source.dwords[dword]); target.dwords[dword] = Clone(source.dwords[dword]);
@ -909,27 +955,26 @@ ResourcePlan ExtractResourcePlan(const Program& program) {
plan.uniform_fill.values[i] = Clone(plan.uniform_fill.values[i]); plan.uniform_fill.values[i] = Clone(plan.uniform_fill.values[i]);
} }
plan.clean_flat_slots.resize(plan.srt_reads.size()); plan.clean_flat_slots.resize(plan.srt_reads.size());
bool capture_image_reads = false; bool capture_indirect_reads = false;
for (const auto& image: plan.info.images) { for (const auto& source: plan.descriptor_sources) {
const auto* source = Source(plan, image.source); if (!source.indirect_descriptor.has_value()) continue;
if (source == nullptr || !source->indirect_image.has_value()) { const auto& indirect = *source.indirect_descriptor;
continue;
}
plan.requires_specialization_memory = true; plan.requires_specialization_memory = true;
capture_image_reads |= !source->indirect_image->selector_mask.IsEmpty() || capture_indirect_reads |= !indirect.selector_mask.IsEmpty() ||
!source->indirect_image->sources.empty(); !indirect.selector_first.IsEmpty() || !indirect.sources.empty();
MarkCleanFlatSlots(plan, Source(plan, source->indirect_image->material_source), MarkCleanFlatSlots(plan, Source(plan, indirect.material_source), plan.clean_flat_slots,
plan.clean_flat_slots, source->indirect_image->selector_mask); indirect.selector_mask);
if (source->indirect_image->sources.empty()) { MarkCleanFlatSlots(plan, nullptr, plan.clean_flat_slots, indirect.selector_first);
MarkCleanFlatSlots(plan, Source(plan, source->indirect_image->table_source), MarkCleanFlatSlots(plan, nullptr, plan.clean_flat_slots, indirect.key_count);
plan.clean_flat_slots); if (indirect.sources.empty()) {
MarkCleanFlatSlots(plan, Source(plan, indirect.table_source), plan.clean_flat_slots);
} else { } else {
for (const auto candidate: source->indirect_image->sources) { for (const auto candidate: indirect.sources) {
MarkCleanFlatSlots(plan, Source(plan, candidate), plan.clean_flat_slots); MarkCleanFlatSlots(plan, Source(plan, candidate), plan.clean_flat_slots);
} }
} }
} }
plan.capture_specialization_reads = capture_image_reads || !plan.control_flow.empty(); plan.capture_specialization_reads = capture_indirect_reads || !plan.control_flow.empty();
// Writable descriptor addresses must come from clean backing for the alias proof. // Writable descriptor addresses must come from clean backing for the alias proof.
for (const auto& buffer: plan.info.buffers) { for (const auto& buffer: plan.info.buffers) {
if (buffer.written) MarkCleanFlatSlots(plan, Source(plan, buffer.source), plan.clean_flat_slots); if (buffer.written) MarkCleanFlatSlots(plan, Source(plan, buffer.source), plan.clean_flat_slots);
@ -937,7 +982,7 @@ ResourcePlan ExtractResourcePlan(const Program& program) {
for (const auto& image: plan.info.images) { for (const auto& image: plan.info.images) {
if (image.written) MarkCleanFlatSlots(plan, Source(plan, image.source), plan.clean_flat_slots); if (image.written) MarkCleanFlatSlots(plan, Source(plan, image.source), plan.clean_flat_slots);
} }
if (capture_image_reads) plan.resource_tracking_complete &= !program.has_address_writes; if (capture_indirect_reads) plan.resource_tracking_complete &= !program.has_address_writes;
return plan; return plan;
} }
@ -988,12 +1033,27 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
return true; return true;
}; };
snapshot.buffers.resize(program.info.buffers.size()); snapshot.buffers.resize(program.info.buffers.size());
specialization.buffers.clear(); specialization.buffers.assign(program.info.buffers.size(), {});
specialization.buffers.reserve(program.info.buffers.size()); for (uint32_t i = 0; i < snapshot.buffers.size(); ++i) {
for (uint32_t i = 0; i < program.info.buffers.size(); ++i) { const auto base_index =
if (!evaluate(program.info.buffers[i].source, snapshot.buffers[i], i < program.info.buffers.size() ? i : specialization.buffers[i].indirect_root;
program.info.buffers[i].written)) { if (base_index >= program.info.buffers.size()) return false;
return false; const auto& base = program.info.buffers[base_index];
if (i < program.info.buffers.size()) {
const auto* source = Source(program, base.source);
if (source == nullptr) return false;
if (source->indirect_descriptor.has_value()) {
snapshot.buffers[i] = {.dword_count = 4u};
if (active.empty() || active[base.source]) {
if (!MaterializeIndirectDescriptor(
program, *source->indirect_descriptor, i, 4u, observed, clean, snapshot,
snapshot.buffers, specialization.buffers, ShaderInfo::MaxBuffers,
NormalizeIndirectStoreBuffer))
return false;
}
} else if (!evaluate(base.source, snapshot.buffers[i], base.written)) {
return false;
}
} }
auto& descriptor_value = snapshot.buffers[i]; auto& descriptor_value = snapshot.buffers[i];
ShaderBufferResource descriptor; ShaderBufferResource descriptor;
@ -1012,15 +1072,12 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
} else if (!swizzle) { } else if (!swizzle) {
packed_stride &= ~(3u << 16u); packed_stride &= ~(3u << 16u);
} }
specialization.buffers.push_back({ auto& buffer = specialization.buffers[i];
.packed_stride = packed_stride, buffer.packed_stride = packed_stride;
.descriptor_format = program.info.buffers[i].formatted buffer.descriptor_format =
? descriptor.Format() base.formatted ? descriptor.Format() : Prospero::BufferFormat::kInvalid;
: Prospero::BufferFormat::kInvalid, buffer.descriptor_swizzle = base.formatted ? descriptor.DstSelXYZW() : DstSel(4, 5, 6, 7);
.descriptor_swizzle = buffer.zero_stride_oob = descriptor.OutOfBounds() == 0u && stride == 0u;
program.info.buffers[i].formatted ? descriptor.DstSelXYZW() : DstSel(4, 5, 6, 7),
.zero_stride_oob = descriptor.OutOfBounds() == 0u && stride == 0u,
});
} }
snapshot.images.resize(program.info.images.size()); snapshot.images.resize(program.info.images.size());
specialization.images.resize(program.info.images.size()); specialization.images.resize(program.info.images.size());
@ -1041,13 +1098,19 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
if (source == nullptr) { if (source == nullptr) {
return false; return false;
} }
if (source->indirect_image.has_value()) { if (source->indirect_descriptor.has_value()) {
snapshot.images[i] = {.dword_count = 8u}; snapshot.images[i] = {.dword_count = 8u};
if (!active.empty() && !active[image.source]) { if (!active.empty() && !active[image.source]) {
continue; continue;
} }
const auto& indirect = *source->indirect_image; const auto& indirect = *source->indirect_descriptor;
if (!MaterializeIndirectImage(program, indirect, i, observed, clean, snapshot, specialization)) { if (!MaterializeIndirectDescriptor(
program, indirect, i, 8u, observed, clean, snapshot, snapshot.images,
specialization.images, ShaderInfo::MaxImages, [&](DescriptorValue& value) {
if (NullImageDescriptor(value) || !ValidImageDescriptor(value, image.r128))
value.dwords.fill(0u);
return true;
})) {
return false; return false;
} }
} else { } else {
@ -1082,14 +1145,33 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
void ApplyResourceSpecialization(Program& program, const ResourceSpecialization& specialization) { void ApplyResourceSpecialization(Program& program, const ResourceSpecialization& specialization) {
EXIT_IF(!program.resource_tracking_complete || program.shader_info_complete || EXIT_IF(!program.resource_tracking_complete || program.shader_info_complete ||
program.binding_layout_complete); program.binding_layout_complete);
EXIT_IF(program.info.buffers.size() != specialization.buffers.size() || EXIT_IF(program.info.buffers.size() > specialization.buffers.size() ||
program.info.images.size() > specialization.images.size()); program.info.images.size() > specialization.images.size());
auto& buffers = program.info.buffers; auto& buffers = program.info.buffers;
for (size_t index = 0; index < buffers.size(); index++) { const auto original_buffer_count = buffers.size();
buffers[index].packed_stride = specialization.buffers[index].packed_stride; buffers.reserve(specialization.buffers.size());
buffers[index].descriptor_format = specialization.buffers[index].descriptor_format; for (uint32_t index = 0; index < specialization.buffers.size(); ++index) {
buffers[index].descriptor_swizzle = specialization.buffers[index].descriptor_swizzle; const auto& source = specialization.buffers[index];
if (index >= buffers.size()) {
EXIT_IF(source.indirect_root >= original_buffer_count);
buffers.push_back(buffers[source.indirect_root]);
}
auto& buffer = buffers[index];
buffer.packed_stride = source.packed_stride;
buffer.descriptor_format = source.descriptor_format;
buffer.descriptor_swizzle = source.descriptor_swizzle;
buffer.indirect_root = source.indirect_root;
buffer.indirect_mapping_offset = source.indirect_mapping_offset;
buffer.indirect_search_iterations = source.indirect_search_iterations;
buffer.indirect_resources.clear();
}
for (uint32_t index = 0; index < buffers.size(); ++index) {
const auto root = buffers[index].indirect_root;
if (root != BufferResource::NoIndirectBuffer) {
EXIT_IF(root >= buffers.size());
buffers[root].indirect_resources.push_back(index);
}
} }
auto& images = program.info.images; auto& images = program.info.images;
const auto original_image_count = images.size(); const auto original_image_count = images.size();

3
src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.h
View File

@ -13,6 +13,9 @@ struct ResourceSpecialization {
Prospero::BufferFormat descriptor_format = Prospero::BufferFormat::kInvalid; Prospero::BufferFormat descriptor_format = Prospero::BufferFormat::kInvalid;
uint32_t descriptor_swizzle = DstSel(4, 5, 6, 7); uint32_t descriptor_swizzle = DstSel(4, 5, 6, 7);
bool zero_stride_oob = false; bool zero_stride_oob = false;
uint32_t indirect_root = BufferResource::NoIndirectBuffer;
uint32_t indirect_mapping_offset = 0;
uint32_t indirect_search_iterations = 0;
bool operator==(const Buffer&) const = default; bool operator==(const Buffer&) const = default;
}; };

335
src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp
View File

@ -130,6 +130,58 @@ public:
return true; return true;
} }
bool SelectsRange(const Inst& selected, Value active, const Inst& candidate,
const Inst& carried, const Inst& mask,
std::span<const std::pair<Value, bool>> exit_conditions) {
// Inactive lanes retain a previously selected index. Initially every lane is
// active; the bounded assignment establishes the invariant on each backedge.
m_exclude_bound = false;
std::vector<uint32_t> range_values;
const auto in_range = [&] {
const auto value = Unknown(Type::U32);
range_values.push_back(value);
return value;
};
const auto contains = [&](const auto& self, uint32_t node) -> uint32_t {
if (std::ranges::find(range_values, node) != range_values.end()) return 1u;
const auto entry = m_nodes[node];
if (entry.variable == UINT32_MAX) return 0u;
return Select(NodeFor(entry.variable, 0u, 1u), self(self, entry.high),
self(self, entry.low));
};
const auto state = [&](bool in_body) {
m_values[1].clear();
const auto enabled = Unknown(Type::U1);
for (const auto& phi: *m_induction.Parent()) {
if (phi.GetOpcode() != ValueOpcode::Phi) continue;
m_values[1][&phi] = EquivalentValue(m_program, Value(const_cast<Inst*>(&phi)),
Value(const_cast<Inst*>(&mask)))
? enabled : Unknown(phi.GetType());
}
m_values[1][&carried] = Select(enabled, Unknown(Type::U32), in_range());
if (in_body) m_values[1][&m_bound] = 1u;
m_values[1][&candidate] = Select(Evaluate(Value(const_cast<Inst*>(&m_bound)), true),
in_range(), Unknown(Type::U32));
};
for (uint32_t arm = 0; arm < carried.NumArgs(); ++arm) {
const auto initial = mask.Arg(arm).Resolve();
if (initial.IsImmediate() && initial.GetType() == Type::U1 && initial.U1()) continue;
state(true);
const auto retained = contains(contains, Evaluate(carried.Arg(arm), true));
if (Select(Evaluate(mask.Arg(arm), true), 1u, retained) != 1u) return false;
}
for (uint32_t arm = 0; arm < selected.NumArgs(); ++arm) {
state(false);
const auto value = Evaluate(selected.Arg(arm), true);
m_values[1][&selected] = value;
auto allowed = Evaluate(exit_conditions[arm].first, true);
if (!exit_conditions[arm].second) allowed = Select(allowed, 0u, 1u);
allowed = Select(allowed, Evaluate(active, true), 0u);
if (Select(allowed, contains(contains, value), 1u) != 1u) return false;
}
return true;
}
private: private:
struct Node { struct Node {
uint32_t variable = UINT32_MAX; uint32_t variable = UINT32_MAX;
@ -171,6 +223,10 @@ private:
} }
uint32_t Compare(const Inst& inst, uint32_t left, uint32_t right) { uint32_t Compare(const Inst& inst, uint32_t left, uint32_t right) {
if (left == right) {
if (inst.GetOpcode() == ValueOpcode::IEqual32) return 1u;
if (inst.GetOpcode() == ValueOpcode::INotEqual32) return 0u;
}
const auto key = std::tuple {inst.GetOpcode(), inst.Flags<uint64_t>(), left, right}; const auto key = std::tuple {inst.GetOpcode(), inst.Flags<uint64_t>(), left, right};
if (const auto found = m_predicates.find(key); found != m_predicates.end()) return found->second; if (const auto found = m_predicates.find(key); found != m_predicates.end()) return found->second;
const auto variable = std::min(m_nodes[left].variable, m_nodes[right].variable); const auto variable = std::min(m_nodes[left].variable, m_nodes[right].variable);
@ -202,7 +258,7 @@ private:
return node; return node;
} }
const auto* inst = value.TryInstruction(); const auto* inst = value.TryInstruction();
if (current && inst == &m_bound) return 0u; if (current && m_exclude_bound && inst == &m_bound) return 0u;
auto& values = m_values[current]; auto& values = m_values[current];
if (const auto found = values.find(inst); found != values.end()) { if (const auto found = values.find(inst); found != values.end()) {
if (found->second == UINT32_MAX) found->second = Unknown(value.GetType()); if (found->second == UINT32_MAX) found->second = Unknown(value.GetType());
@ -268,6 +324,7 @@ private:
const Inst& m_bound; const Inst& m_bound;
const Block* m_incoming = nullptr; const Block* m_incoming = nullptr;
uint32_t m_variables = 0; uint32_t m_variables = 0;
bool m_exclude_bound = true;
std::vector<Node> m_nodes {{}, {}}; std::vector<Node> m_nodes {{}, {}};
std::vector<std::pair<Value, uint32_t>> m_literals; std::vector<std::pair<Value, uint32_t>> m_literals;
std::array<std::map<const Inst*, uint32_t>, 2> m_values; std::array<std::map<const Inst*, uint32_t>, 2> m_values;
@ -296,7 +353,7 @@ public:
} }
PlanScalarReads(); PlanScalarReads();
EliminateDeadCode(m_program.blocks); EliminateDeadCode(m_program.blocks);
PlanIndirectImages(); PlanIndirectDescriptors();
for (auto* block: m_program.blocks) { for (auto* block: m_program.blocks) {
for (auto& inst: *block) { for (auto& inst: *block) {
Collect(inst); Collect(inst);
@ -313,17 +370,21 @@ public:
memory.sampler = patch.sampler; memory.sampler = patch.sampler;
} }
} }
for (const auto& plan: m_indirect_images) { for (const auto& plan: m_indirect_descriptors) {
plan.handle->SetArg(0, plan.key); plan.handle->SetArg(0, plan.key);
for (uint32_t dword = 0; dword < 4u; dword++) { if (plan.handle->GetOpcode() == ValueOpcode::GetBufferResource) {
plan.handle->SetArg(dword + 1u, plan.roots[dword + 4u]); for (uint32_t word = 1; word < 4u; ++word) plan.handle->SetArg(word, Value(0u));
} } else {
for (uint32_t dword = 5u; dword < plan.roots.size(); dword++) { for (uint32_t dword = 0; dword < 4u; dword++) {
plan.handle->SetArg(dword, plan.key); plan.handle->SetArg(dword + 1u, plan.roots[dword + 4u]);
}
for (uint32_t dword = 5u; dword < plan.roots.size(); dword++) {
plan.handle->SetArg(dword, plan.key);
}
} }
if (plan.reads[0] != nullptr) { if (plan.reads[0] != nullptr) {
for (const auto index: plan.memory) for (uint32_t word = 0; word < plan.handle->NumArgs(); ++word)
m_program.memory_info[index].planning_only = true; m_program.memory_info[plan.memory[word]].planning_only = true;
} }
} }
m_program.descriptor_sources = std::move(m_sources); m_program.descriptor_sources = std::move(m_sources);
@ -351,7 +412,7 @@ private:
Value planning_handle; Value planning_handle;
}; };
struct IndirectImagePlan { struct IndirectDescriptorPlan {
Inst* handle = nullptr; Inst* handle = nullptr;
uint32_t source = 0; uint32_t source = 0;
Value key; Value key;
@ -755,16 +816,19 @@ private:
for (uint32_t candidate = 0; candidate < m_sources.size(); candidate++) { for (uint32_t candidate = 0; candidate < m_sources.size(); candidate++) {
const auto& current = m_sources[candidate]; const auto& current = m_sources[candidate];
if (current.dword_count != descriptor.dword_count || if (current.dword_count != descriptor.dword_count ||
current.indirect_image.has_value() != descriptor.indirect_image.has_value()) { current.indirect_descriptor.has_value() != descriptor.indirect_descriptor.has_value()) {
continue; continue;
} }
if (current.indirect_image.has_value()) { if (current.indirect_descriptor.has_value()) {
const auto& a = *current.indirect_image; const auto& a = *current.indirect_descriptor;
const auto& b = *descriptor.indirect_image; const auto& b = *descriptor.indirect_descriptor;
if (a.material_source != b.material_source || a.table_source != b.table_source || if (a.material_source != b.material_source || a.table_source != b.table_source ||
a.selector_stride != b.selector_stride || a.selector_offset != b.selector_offset || a.selector_stride != b.selector_stride || a.selector_offset != b.selector_offset ||
a.table_offset != b.table_offset || a.sources != b.sources || a.table_offset != b.table_offset || a.sources != b.sources ||
!EquivalentValue(m_program, a.key_count, b.key_count) || !EquivalentValue(m_program, a.key_count, b.key_count) ||
a.selector_first.IsEmpty() != b.selector_first.IsEmpty() ||
(!a.selector_first.IsEmpty() &&
!EquivalentValue(m_program, a.selector_first, b.selector_first)) ||
a.selector_mask.IsEmpty() != b.selector_mask.IsEmpty() || a.selector_mask.IsEmpty() != b.selector_mask.IsEmpty() ||
(!a.selector_mask.IsEmpty() && (!a.selector_mask.IsEmpty() &&
!EquivalentValue(m_program, a.selector_mask, b.selector_mask))) continue; !EquivalentValue(m_program, a.selector_mask, b.selector_mask))) continue;
@ -928,7 +992,7 @@ private:
return false; return false;
} }
bool MatchTableOffset(Value value, Value& key, uint32_t& offset) const { bool MatchTableOffset(Value value, Value& key, uint32_t& offset, uint32_t shift = 5u) const {
offset = 0; offset = 0;
for (;;) { for (;;) {
const auto* inst = value.Resolve().TryInstruction(); const auto* inst = value.Resolve().TryInstruction();
@ -937,7 +1001,7 @@ private:
} }
uint32_t immediate; uint32_t immediate;
if (inst->GetOpcode() == ValueOpcode::ShiftLeftLogical32 && if (inst->GetOpcode() == ValueOpcode::ShiftLeftLogical32 &&
ImmediateU32(inst->Arg(1), immediate) && immediate == 5u) { ImmediateU32(inst->Arg(1), immediate) && immediate == shift) {
key = inst->Arg(0).Resolve(); key = inst->Arg(0).Resolve();
return key.GetType() == Type::U32; return key.GetType() == Type::U32;
} }
@ -1241,9 +1305,7 @@ private:
return {}; return {};
} }
bool MatchUniformizedMaterialKey(Value key, const Inst& image, const Inst* UniformizedMaterialValue(Value key, const Inst& use, Value executing = {}) const {
DescriptorSource::IndirectImage& indirect,
DescriptorSource& material_source) {
Value guard; Value guard;
Value local; Value local;
const auto* first = key.Resolve().TryInstruction(); const auto* first = key.Resolve().TryInstruction();
@ -1251,7 +1313,9 @@ private:
first->NumArgs() == 2u) { first->NumArgs() == 2u) {
guard = first->Arg(1).Resolve(); guard = first->Arg(1).Resolve();
// Empty EXEC selects lane zero, which may not have loaded a material key. // Empty EXEC selects lane zero, which may not have loaded a material key.
if (!HasActiveLane(guard, first->Parent())) return false; // An executing store also witnesses nonempty EXEC at the lane read.
if (executing.IsEmpty() ? !HasActiveLane(guard, first->Parent())
: !Implies(executing, guard)) return nullptr;
local = first->Arg(0).Resolve(); local = first->Arg(0).Resolve();
const auto* phi = guard.TryInstruction(); const auto* phi = guard.TryInstruction();
if (phi != nullptr && phi->GetOpcode() == ValueOpcode::Phi && if (phi != nullptr && phi->GetOpcode() == ValueOpcode::Phi &&
@ -1265,13 +1329,21 @@ private:
} }
} }
} else { } else {
guard = PositiveLaneWitness(image.Parent()); guard = PositiveLaneWitness(use.Parent());
if (guard.IsEmpty()) return false; if (guard.IsEmpty()) return nullptr;
local = EqualLocalKey(guard, key); local = EqualLocalKey(guard, key);
} }
const auto* selected = local.Resolve().TryInstruction(); const auto* selected = local.Resolve().TryInstruction();
if (selected == nullptr || selected->GetOpcode() != ValueOpcode::SelectU32 || if (selected == nullptr || selected->GetOpcode() != ValueOpcode::SelectU32 ||
selected->NumArgs() != 3u || !Implies(guard, selected->Arg(0))) return false; selected->NumArgs() != 3u || !Implies(guard, selected->Arg(0))) return nullptr;
return selected;
}
bool MatchUniformizedMaterialKey(Value key, const Inst& image,
DescriptorSource::IndirectDescriptor& indirect,
DescriptorSource& material_source) {
const auto* selected = UniformizedMaterialValue(key, image);
if (selected == nullptr) return false;
const auto active = selected->Arg(0).Resolve(); const auto active = selected->Arg(0).Resolve();
const auto* read = selected->Arg(1).Resolve().TryInstruction(); const auto* read = selected->Arg(1).Resolve().TryInstruction();
if (read == nullptr || read->GetOpcode() != ValueOpcode::LoadAddressU32 || if (read == nullptr || read->GetOpcode() != ValueOpcode::LoadAddressU32 ||
@ -1303,9 +1375,9 @@ private:
return true; return true;
} }
Value BoundedLoopCount(Value key, const Block* use) const { const Inst* BoundedLoop(Value key, const Block* use) const {
const auto* phi = key.Resolve().TryInstruction(); const auto* phi = key.Resolve().TryInstruction();
if (m_shader_writes || phi == nullptr || phi->GetOpcode() != ValueOpcode::Phi || if (phi == nullptr || phi->GetOpcode() != ValueOpcode::Phi ||
phi->GetType() != Type::U32 || phi->NumArgs() != 2u || phi->GetType() != Type::U32 || phi->NumArgs() != 2u ||
m_program.blocks.size() != m_program.block_info.size()) return {}; m_program.blocks.size() != m_program.block_info.size()) return {};
const Block* increment_block = nullptr; const Block* increment_block = nullptr;
@ -1342,8 +1414,9 @@ private:
}; };
for (const auto& use_of_key: phi->Uses()) { for (const auto& use_of_key: phi->Uses()) {
const auto* compare = use_of_key.user; const auto* compare = use_of_key.user;
if (compare->GetOpcode() != ValueOpcode::SLessThan32 || use_of_key.operand != 0u || if ((compare->GetOpcode() != ValueOpcode::SLessThan32 &&
!ValidateRuntimeValue(m_program, compare->Arg(1))) continue; compare->GetOpcode() != ValueOpcode::ULessThan32) || use_of_key.operand != 0u ||
!ValidateRuntimeValue(m_program, compare->Arg(1), RuntimeValueType::Integer)) continue;
if (!guarded_on_entry(use, [&](const EdgePredicate& edge) { if (!guarded_on_entry(use, [&](const EdgePredicate& edge) {
return edge.positive && Implies(edge.condition, Value(use_of_key.user)); return edge.positive && Implies(edge.condition, Value(use_of_key.user));
})) continue; })) continue;
@ -1352,19 +1425,75 @@ private:
// skipped image alone does not prevent signed induction wraparound. // skipped image alone does not prevent signed induction wraparound.
if (guarded_on_entry(increment_block, [&](const EdgePredicate& edge) { if (guarded_on_entry(increment_block, [&](const EdgePredicate& edge) {
return proof.Excludes(edge.condition, edge.positive); return proof.Excludes(edge.condition, edge.positive);
})) return compare->Arg(1); })) return compare;
} }
return {}; return {};
} }
bool TryMakeIndirectImage(Inst& handle, IndirectImagePlan& plan) { bool BoundedSelectedIndex(Value index, Value active,
if (handle.GetOpcode() != ValueOpcode::GetImageResource || handle.NumArgs() != 8u) { DescriptorSource::IndirectDescriptor& indirect) const {
return false; const auto* selected = index.Resolve().TryInstruction();
if (selected == nullptr || selected->GetOpcode() != ValueOpcode::Phi ||
selected->NumArgs() == 0u) return false;
std::vector<const Inst*> values {selected};
for (size_t i = 0; i < values.size(); ++i) {
const auto* value = values[i];
if (value->GetOpcode() != ValueOpcode::Phi &&
value->GetOpcode() != ValueOpcode::SelectU32) continue;
for (size_t arm = value->GetOpcode() == ValueOpcode::SelectU32 ? 1u : 0u;
arm < value->NumArgs(); ++arm) {
const auto* next = value->Arg(arm).Resolve().TryInstruction();
if (next != nullptr && std::ranges::find(values, next) == values.end())
values.push_back(next);
}
} }
for (const auto* candidate: values) {
if (candidate->GetOpcode() != ValueOpcode::IAdd32) continue;
for (uint32_t arg = 0; arg < 2u; ++arg) {
const auto* bound = BoundedLoop(candidate->Arg(arg), candidate->Parent());
const auto first = candidate->Arg(arg ^ 1u);
if (bound == nullptr || !ValidateRuntimeValue(m_program, first, RuntimeValueType::Integer))
continue;
const auto* induction = candidate->Arg(arg).ResolveInstruction();
const Inst* carried = nullptr;
for (const auto* value: values) {
if (value->GetOpcode() == ValueOpcode::Phi && value->Parent() == induction->Parent()) {
if (carried != nullptr) { carried = nullptr; break; }
carried = value;
}
}
if (carried == nullptr || carried->NumArgs() != 2u ||
carried->PhiBlock(0) != induction->PhiBlock(0) ||
carried->PhiBlock(1) != induction->PhiBlock(1)) continue;
std::vector<std::pair<Value, bool>> exits;
for (size_t arm = 0; arm < selected->NumArgs(); ++arm) {
const auto edge = ConditionalEdge(selected->PhiBlock(arm), selected->Parent());
if (!edge || edge->lanes != LaneQuantifier::All) break;
exits.emplace_back(edge->condition, edge->positive);
}
if (exits.size() != selected->NumArgs()) continue;
for (const auto& mask: *induction->Parent()) {
if (mask.GetOpcode() != ValueOpcode::Phi || mask.GetType() != Type::U1 ||
mask.NumArgs() != 2u || mask.PhiBlock(0) != carried->PhiBlock(0) ||
mask.PhiBlock(1) != carried->PhiBlock(1)) continue;
const auto initial = induction->Arg(0).Resolve().IsImmediate() ? 0u : 1u;
const auto enabled = mask.Arg(initial).Resolve();
if (!enabled.IsImmediate() || enabled.GetType() != Type::U1 || !enabled.U1()) continue;
LoopBoundProof proof(m_program, *induction, *bound);
if (!proof.SelectsRange(*selected, active, *candidate, *carried, mask, exits)) continue;
indirect.selector_first = first;
indirect.key_count = bound->Arg(1);
return true;
}
}
}
return false;
}
bool MatchDescriptorTable(Inst& handle, IndirectDescriptorPlan& plan,
DescriptorSource& table_source, Value& key, uint32_t& table_offset) {
Inst* table_handle = nullptr; Inst* table_handle = nullptr;
Value key; for (uint32_t dword = 0; dword < handle.NumArgs(); ++dword) {
uint32_t table_offset = 0;
for (uint32_t dword = 0; dword < plan.reads.size(); ++dword) {
auto* read = handle.Arg(dword).Resolve().TryInstruction(); auto* read = handle.Arg(dword).Resolve().TryInstruction();
if (read == nullptr) { if (read == nullptr) {
return false; return false;
@ -1385,7 +1514,7 @@ private:
(memory->kind == ResourceKind::ScalarAddress && read->Parent() != handle.Parent()) || (memory->kind == ResourceKind::ScalarAddress && read->Parent() != handle.Parent()) ||
(table_handle != nullptr && (table_handle != nullptr &&
!EquivalentValue(m_program, Value(table_handle), Value(current_handle))) || !EquivalentValue(m_program, Value(table_handle), Value(current_handle))) ||
!MatchTableOffset(read->Arg(1), current_key, offset) || !MatchTableOffset(read->Arg(1), current_key, offset, handle.NumArgs() == 4u ? 4u : 5u) ||
memory->offset > UINT32_MAX - offset) { memory->offset > UINT32_MAX - offset) {
return false; return false;
} }
@ -1399,19 +1528,33 @@ private:
} }
table_handle = current_handle; table_handle = current_handle;
const std::array<const Inst*, 1> image_users {&handle}; const std::array<const Inst*, 1> image_users {&handle};
if (!UsesOnly(*read, image_users)) { if (handle.NumArgs() == 8u ? !UsesOnly(*read, image_users) :
std::ranges::any_of(read->Uses(), [](const Use& use) {
return use.user->GetOpcode() != ValueOpcode::GetBufferResource ||
std::ranges::any_of(use.user->Uses(), [](const Use& consumer) {
return consumer.user->GetOpcode() != ValueOpcode::StoreBufferU32;
});
})) {
return false; return false;
} }
plan.memory[dword] = memory_index; plan.memory[dword] = memory_index;
plan.reads[dword] = read; plan.reads[dword] = read;
} }
DescriptorSource table_source;
if (!MakeRuntimeTableSource(*plan.reads[0], table_source)) { if (!MakeRuntimeTableSource(*plan.reads[0], table_source)) {
return false; return false;
} }
return true;
}
bool TryMakeIndirectImage(Inst& handle, IndirectDescriptorPlan& plan) {
if (handle.GetOpcode() != ValueOpcode::GetImageResource || handle.NumArgs() != 8u) return false;
DescriptorSource table_source;
Value key;
uint32_t table_offset = 0;
if (!MatchDescriptorTable(handle, plan, table_source, key, table_offset)) return false;
DescriptorSource material_source; DescriptorSource material_source;
DescriptorSource::IndirectImage indirect; DescriptorSource::IndirectDescriptor indirect;
indirect.table_offset = table_offset; indirect.table_offset = table_offset;
if (table_source.dword_count == 2u) { if (table_source.dword_count == 2u) {
const auto* selector = key.Resolve().TryInstruction(); const auto* selector = key.Resolve().TryInstruction();
@ -1420,8 +1563,9 @@ private:
NonzeroOnEntry(selector->Arg(0), handle.Parent()); NonzeroOnEntry(selector->Arg(0), handle.Parent());
if (bitscan) { if (bitscan) {
indirect.key_count = Value(32u); indirect.key_count = Value(32u);
} else { } else if (!m_shader_writes) {
indirect.key_count = BoundedLoopCount(key, handle.Parent()); if (const auto* bound = BoundedLoop(key, handle.Parent()))
indirect.key_count = bound->Arg(1);
} }
if (indirect.key_count.IsEmpty() && if (indirect.key_count.IsEmpty() &&
!MatchUniformizedMaterialKey(key, handle, indirect, material_source)) { !MatchUniformizedMaterialKey(key, handle, indirect, material_source)) {
@ -1467,7 +1611,7 @@ private:
image_source.dwords.begin()); image_source.dwords.begin());
std::copy_n(table_source.dwords.begin(), table_source.dword_count, std::copy_n(table_source.dwords.begin(), table_source.dword_count,
image_source.dwords.begin() + 4u); image_source.dwords.begin() + 4u);
image_source.indirect_image = indirect; image_source.indirect_descriptor = indirect;
plan.handle = &handle; plan.handle = &handle;
plan.source = InternSource(image_source); plan.source = InternSource(image_source);
plan.key = key; plan.key = key;
@ -1475,6 +1619,66 @@ private:
return true; return true;
} }
bool TryMakeIndirectBuffer(Inst& handle, IndirectDescriptorPlan& plan) {
if (handle.GetOpcode() != ValueOpcode::GetBufferResource || handle.NumArgs() != 4u ||
handle.Uses().empty()) return false;
for (const auto& use: handle.Uses()) {
const auto& inst = *use.user;
if (inst.GetOpcode() != ValueOpcode::StoreBufferU32) return false;
const auto& memory = m_program.memory_info[inst.Flags<MemoryFlags>().index];
uint32_t scalar = 1u;
if (memory.kind != ResourceKind::Buffer || memory.typed || memory.formatted ||
memory.idxen || !memory.offen || !ImmediateU32(inst.Arg(3), scalar) || scalar != 0u)
return false;
}
DescriptorSource table_source;
Value key;
uint32_t table_offset = 0;
if (!MatchDescriptorTable(handle, plan, table_source, key, table_offset) ||
table_source.dword_count != 4u || table_offset != 0u) return false;
const auto* lane_read = key.Resolve().TryInstruction();
if (lane_read == nullptr || lane_read->GetOpcode() != ValueOpcode::ReadFirstLane)
return false;
const Inst* selected = nullptr;
for (const auto& use: handle.Uses()) {
const auto* value = UniformizedMaterialValue(
key, *use.user, use.user->Arg(use.user->NumArgs() - 1u));
if (value == nullptr || (selected != nullptr && selected != value)) return false;
selected = value;
}
const auto* read = selected->Arg(1).Resolve().TryInstruction();
uint32_t lane = 0;
if (read != nullptr && read->GetOpcode() == ValueOpcode::CompositeExtractU32x2) {
if (!ImmediateU32(read->Arg(1), lane) || lane >= 2u) return false;
read = read->Arg(0).Resolve().TryInstruction();
if (read == nullptr || read->GetOpcode() != ValueOpcode::LoadBufferU32x2) return false;
} else if (read == nullptr || read->GetOpcode() != ValueOpcode::LoadBufferU32) {
return false;
}
const auto& memory = m_program.memory_info[read->Flags<MemoryFlags>().index];
uint32_t offset = 1u, scalar = 1u;
if (memory.kind != ResourceKind::Buffer || memory.typed || memory.formatted ||
!memory.idxen || memory.offen || memory.offset > UINT32_MAX - lane * 4u ||
!ImmediateU32(read->Arg(2), offset) || offset != 0u ||
!ImmediateU32(read->Arg(3), scalar) || scalar != 0u ||
!EquivalentValue(m_program, selected->Arg(0), read->Arg(4))) return false;
DescriptorSource material_source;
DescriptorSource::IndirectDescriptor indirect;
if (!MakeRuntimeTableSource(*read, material_source) ||
!BoundedSelectedIndex(read->Arg(1), read->Arg(4), indirect)) return false;
indirect.material_source = InternSource(material_source);
indirect.table_source = InternSource(table_source);
indirect.selector_offset = memory.offset + lane * 4u;
DescriptorSource source;
source.dword_count = 4u;
source.dwords.fill(Value(0u));
source.indirect_descriptor = indirect;
plan.handle = &handle;
plan.source = InternSource(source);
plan.key = key;
return true;
}
uint32_t SelectorMaximum(Value value) const { uint32_t SelectorMaximum(Value value) const {
value = value.Resolve(); value = value.Resolve();
if (value.IsImmediate() && value.GetType() == Type::U32) return value.U32(); if (value.IsImmediate() && value.GetType() == Type::U32) return value.U32();
@ -1515,7 +1719,7 @@ private:
return found; return found;
} }
bool TryMakeFiniteImage(Inst& handle, IndirectImagePlan& plan) { bool TryMakeFiniteImage(Inst& handle, IndirectDescriptorPlan& plan) {
if (handle.GetOpcode() != ValueOpcode::GetImageResource || handle.NumArgs() != 8u) if (handle.GetOpcode() != ValueOpcode::GetImageResource || handle.NumArgs() != 8u)
return false; return false;
if (std::ranges::any_of(handle.Uses(), [](const Use& use) { if (std::ranges::any_of(handle.Uses(), [](const Use& use) {
@ -1584,8 +1788,8 @@ private:
DescriptorSource image_source; DescriptorSource image_source;
image_source.dword_count = 8u; image_source.dword_count = 8u;
image_source.dwords.fill(Value(0u)); image_source.dwords.fill(Value(0u));
image_source.indirect_image.emplace(DescriptorSource::IndirectImage{}); image_source.indirect_descriptor.emplace(DescriptorSource::IndirectDescriptor{});
auto& sources = image_source.indirect_image->sources; auto& sources = image_source.indirect_descriptor->sources;
for (auto& choice: choices) { for (auto& choice: choices) {
if (choice.branch != nullptr) continue; if (choice.branch != nullptr) continue;
const auto source = InternSource(choice.descriptor); const auto source = InternSource(choice.descriptor);
@ -1621,37 +1825,40 @@ private:
return true; return true;
} }
const IndirectImagePlan* FindIndirectImage(const Inst& handle) const { const IndirectDescriptorPlan* FindIndirectDescriptor(const Inst& handle) const {
const auto found = const auto found =
std::find_if(m_indirect_images.begin(), m_indirect_images.end(), std::find_if(m_indirect_descriptors.begin(), m_indirect_descriptors.end(),
[&](const IndirectImagePlan& plan) { [&](const IndirectDescriptorPlan& plan) {
return plan.handle == &handle; return plan.handle == &handle;
}); });
return found == m_indirect_images.end() ? nullptr : &*found; return found == m_indirect_descriptors.end() ? nullptr : &*found;
} }
bool IsIndirectPlanningMemory(uint32_t index) const { bool IsIndirectPlanningMemory(uint32_t index) const {
return std::any_of(m_indirect_images.begin(), m_indirect_images.end(), return std::any_of(m_indirect_descriptors.begin(), m_indirect_descriptors.end(),
[&](const IndirectImagePlan& plan) { [&](const IndirectDescriptorPlan& plan) {
return plan.reads[0] != nullptr && return plan.reads[0] != nullptr &&
std::ranges::find(plan.memory, index) != plan.memory.end(); std::find(plan.memory.begin(), plan.memory.begin() + plan.handle->NumArgs(), index) !=
plan.memory.begin() + plan.handle->NumArgs();
}); });
} }
void PlanIndirectImages() { void PlanIndirectDescriptors() {
for (auto* block: m_program.blocks) { for (auto* block: m_program.blocks) {
for (auto& inst: *block) { for (auto& inst: *block) {
if (ImageOpcodeInfoOf(inst.GetOpcode()).access == ImageAccess::None || if ((ImageOpcodeInfoOf(inst.GetOpcode()).access == ImageAccess::None &&
inst.GetOpcode() != ValueOpcode::StoreBufferU32) ||
inst.NumArgs() == 0u) { inst.NumArgs() == 0u) {
continue; continue;
} }
auto* handle = inst.Arg(0).Resolve().TryInstruction(); auto* handle = inst.Arg(0).Resolve().TryInstruction();
if (handle == nullptr || FindIndirectImage(*handle) != nullptr) { if (handle == nullptr || FindIndirectDescriptor(*handle) != nullptr) {
continue; continue;
} }
IndirectImagePlan plan; IndirectDescriptorPlan plan;
if (TryMakeIndirectImage(*handle, plan) || TryMakeFiniteImage(*handle, plan)) { if (TryMakeIndirectImage(*handle, plan) || TryMakeFiniteImage(*handle, plan) ||
m_indirect_images.push_back(std::move(plan)); TryMakeIndirectBuffer(*handle, plan)) {
m_indirect_descriptors.push_back(std::move(plan));
} }
} }
} }
@ -1858,7 +2065,11 @@ private:
uint32_t resource = 0; uint32_t resource = 0;
if (buffer != BufferAccess::None) { if (buffer != BufferAccess::None) {
if (!GetHandle(inst.Arg(0), ValueOpcode::GetBufferResource, 4, flags.pc, handle = inst.Arg(0).Resolve().TryInstruction();
const auto* indirect = handle == nullptr ? nullptr : FindIndirectDescriptor(*handle);
if (indirect != nullptr) {
source = indirect->source;
} else if (!GetHandle(inst.Arg(0), ValueOpcode::GetBufferResource, 4, flags.pc,
memory.resource * 4u, handle, source)) { memory.resource * 4u, handle, source)) {
if (memory.kind != ResourceKind::Buffer || !memory.SupportsIndirectBufferLoad(op)) { if (memory.kind != ResourceKind::Buffer || !memory.SupportsIndirectBufferLoad(op)) {
Fail(flags.pc, Fail(flags.pc,
@ -1914,7 +2125,7 @@ private:
Fail(flags.pc, "image operation has invalid resource kind"); Fail(flags.pc, "image operation has invalid resource kind");
} }
handle = inst.Arg(0).Resolve().TryInstruction(); handle = inst.Arg(0).Resolve().TryInstruction();
const auto* indirect = handle != nullptr ? FindIndirectImage(*handle) : nullptr; const auto* indirect = handle != nullptr ? FindIndirectDescriptor(*handle) : nullptr;
if (indirect != nullptr) { if (indirect != nullptr) {
source = indirect->source; source = indirect->source;
} else { } else {
@ -1963,7 +2174,7 @@ private:
for (uint32_t image = 0; image < m_info.images.size(); image++) { for (uint32_t image = 0; image < m_info.images.size(); image++) {
const auto* image_source = Source(m_info.images[image].source); const auto* image_source = Source(m_info.images[image].source);
if (image_source == nullptr || image_source->dword_count != 8 || if (image_source == nullptr || image_source->dword_count != 8 ||
image_source->indirect_image.has_value()) { image_source->indirect_descriptor.has_value()) {
continue; continue;
} }
bool alias = true; bool alias = true;
@ -1991,7 +2202,7 @@ private:
std::vector<DescriptorSource> m_sources; std::vector<DescriptorSource> m_sources;
std::vector<HandlePatch> m_handle_patches; std::vector<HandlePatch> m_handle_patches;
std::vector<MemoryPatch> m_memory_patches; std::vector<MemoryPatch> m_memory_patches;
std::vector<IndirectImagePlan> m_indirect_images; std::vector<IndirectDescriptorPlan> m_indirect_descriptors;
std::vector<std::pair<const Inst*, Value>> m_descriptor_selections; std::vector<std::pair<const Inst*, Value>> m_descriptor_selections;
bool m_shader_writes = false; bool m_shader_writes = false;
}; };

2
tests/KernelFileSystemTests.cpp
View File

@ -675,7 +675,7 @@ void CheckAmprOrdering(Loader::SymbolDatabase &symbols, uint32_t file_id) {
"initialize AMPR command buffer"); "initialize AMPR command buffer");
} }
}; };
// SDK WaitCompare order: ==, unsigned >/<, !=, wrapped >=, signed >/<. // WaitCompare order: ==, unsigned >/<, !=, wrapped >=, signed >/<.
struct Comparison { uint8_t compare; uint64_t blocked, reference, released; }; struct Comparison { uint8_t compare; uint64_t blocked, reference, released; };
constexpr std::array comparisons { constexpr std::array comparisons {
Comparison{0, 1, 2, 2}, Comparison{1, 0x40000000000019c3, 0x40000000000019c3, Comparison{0, 1, 2, 2}, Comparison{1, 0x40000000000019c3, 0x40000000000019c3,

14
tests/ResourceTrackingTests.cpp
View File

@ -306,7 +306,7 @@ void TestInvariantIndirectImageMaterialization() {
const auto source = fixture->program.info.images[0].source; const auto source = fixture->program.info.images[0].source;
Check(source < fixture->program.descriptor_sources.size() && Check(source < fixture->program.descriptor_sources.size() &&
fixture->program.descriptor_sources[source] fixture->program.descriptor_sources[source]
.indirect_image.has_value(), .indirect_descriptor.has_value(),
"indirect image source was not retained for runtime proof"); "indirect image source was not retained for runtime proof");
const auto image_handle = const auto image_handle =
std::ranges::find_if(*fixture->block, [](const Inst &inst) { std::ranges::find_if(*fixture->block, [](const Inst &inst) {
@ -668,7 +668,7 @@ void TestGuardedDirectImageTable() {
fixture.AddMemory(memory, 0x128)); fixture.AddMemory(memory, 0x128));
fixture.PlanAndTrack(); fixture.PlanAndTrack();
const auto source = fixture.program.info.images[0].source; const auto source = fixture.program.info.images[0].source;
const auto &indirect = fixture.program.descriptor_sources[source].indirect_image; const auto &indirect = fixture.program.descriptor_sources[source].indirect_descriptor;
Check(indirect && indirect->material_source == UINT32_MAX && Check(indirect && indirect->material_source == UINT32_MAX &&
indirect->selector_stride == 0u && indirect->table_offset == 344u && indirect->selector_stride == 0u && indirect->table_offset == 344u &&
indirect->key_count.Resolve().IsImmediate() && indirect->key_count.Resolve().IsImmediate() &&
@ -966,7 +966,7 @@ void TestBoundedComputeImageLoop() {
fixture.AddMemory(sample, 0x29c)); fixture.AddMemory(sample, 0x29c));
fixture.PlanAndTrack(); fixture.PlanAndTrack();
const auto source = fixture.program.info.images[0].source; const auto source = fixture.program.info.images[0].source;
const auto &indirect = fixture.program.descriptor_sources[source].indirect_image; const auto &indirect = fixture.program.descriptor_sources[source].indirect_descriptor;
Check(indirect && indirect->material_source == UINT32_MAX && Check(indirect && indirect->material_source == UINT32_MAX &&
indirect->table_offset == 0x6b0u && indirect->table_offset == 0x6b0u &&
indirect->key_count.Resolve() == count.Resolve(), indirect->key_count.Resolve() == count.Resolve(),
@ -1321,7 +1321,7 @@ void TestUniformizedMaterialImageKeys() {
fixture.AddMemory(output_memory, 0x1b00u), done); fixture.AddMemory(output_memory, 0x1b00u), done);
fixture.PlanAndTrack(); fixture.PlanAndTrack();
const auto source = fixture.program.info.images[0].source; const auto source = fixture.program.info.images[0].source;
const auto &indirect = fixture.program.descriptor_sources[source].indirect_image; const auto &indirect = fixture.program.descriptor_sources[source].indirect_descriptor;
Check(indirect && indirect->selector_stride == 0x90u && Check(indirect && indirect->selector_stride == 0x90u &&
indirect->selector_offset == 0xc00u && indirect->selector_offset == 0xc00u &&
indirect->table_offset == 0x20e0u && indirect->table_offset == 0x20e0u &&
@ -1340,7 +1340,7 @@ void TestUniformizedMaterialImageKeys() {
plan = make_plan(Variant::FirstLaneAndNot); plan = make_plan(Variant::FirstLaneAndNot);
Check(plan.requires_specialization_memory && Check(plan.requires_specialization_memory &&
plan.descriptor_sources[plan.info.images[0].source] plan.descriptor_sources[plan.info.images[0].source]
.indirect_image->selector_mask.Resolve().TryInstruction() != nullptr, .indirect_descriptor->selector_mask.Resolve().TryInstruction() != nullptr,
"uniformized image mask did not survive extraction"); "uniformized image mask did not survive extraction");
LinearTestMemory memory; LinearTestMemory memory;
memory.words.resize(0x23000u / 4u); memory.words.resize(0x23000u / 4u);
@ -2577,7 +2577,7 @@ void TestFiniteImagePhiCycle() {
fixture.AddMemory(memory, 4)); fixture.AddMemory(memory, 4));
fixture.PlanAndTrack(); fixture.PlanAndTrack();
const auto source = fixture.program.info.images[0].source; const auto source = fixture.program.info.images[0].source;
const auto &finite = fixture.program.descriptor_sources[source].indirect_image; const auto &finite = fixture.program.descriptor_sources[source].indirect_descriptor;
Check(finite && finite->sources.size() == 2, Check(finite && finite->sources.size() == 2,
"cyclic image selection did not retain two complete descriptor candidates"); "cyclic image selection did not retain two complete descriptor candidates");
const auto *key = image.Instruction()->Arg(0).Instruction(); const auto *key = image.Instruction()->Arg(0).Instruction();
@ -2659,7 +2659,7 @@ void TestFiniteImageBitScanSentinel() {
} }
fixture.PlanAndTrack(); fixture.PlanAndTrack();
const auto source = fixture.program.info.images[0].source; const auto source = fixture.program.info.images[0].source;
const auto &finite = fixture.program.descriptor_sources[source].indirect_image; const auto &finite = fixture.program.descriptor_sources[source].indirect_descriptor;
Check(finite && finite->sources.size() == 1 && Check(finite && finite->sources.size() == 1 &&
image.Instruction()->Arg(0).Instruction()->NumPhiBlocks() == 2, image.Instruction()->Arg(0).Instruction()->NumPhiBlocks() == 2,
"nonzero bit scan retained its impossible sentinel or removed a Phi edge"); "nonzero bit scan retained its impossible sentinel or removed a Phi edge");

121
tests/ShaderRecompilerComputeTests.cpp
View File

@ -9202,7 +9202,7 @@ public:
rendering.width == width && rendering.height == height && rendering.width == width && rendering.height == height &&
rendering.num_layers == view_layers && rendering.num_layers == view_layers &&
rendering.num_color_attachments == 1, rendering.num_color_attachments == 1,
"SDK dimension=0 slice range did not create a Vulkan 1D-array " "dimension=0 slice range did not create a Vulkan 1D-array "
"attachment view"); "attachment view");
RenderExecutorTestAccess::ResetBindings(executor); RenderExecutorTestAccess::ResetBindings(executor);
@ -30569,6 +30569,117 @@ TestCase ImageD16StoreUnpacksHalfPairs() {
return test; return test;
} }
void CheckIndirectBufferStore(VulkanHarness &vulkan) {
using namespace ShaderRecompiler::IR;
TestCase test;
test.name = "IndirectBufferStore";
test.initial.assign(64, 0xdeadbeefu);
test.storage_buffer_range_dwords = 16;
test.storage_buffer_offsets = {12, 108};
test.required_spirv = {"OpSwitch"};
test.forbidden_spirv = {"PhysicalStorageBuffer"};
CompiledShader compiled;
auto &program = compiled.program;
program.stage = ShaderType::Compute;
program.wave_size = 32;
program.srt_plan_complete = true;
program.resource_tracking_complete = true;
program.shader_info_complete = true;
program.block_storage.push_back(std::make_unique<Block>());
auto *block = program.block_storage.back().get();
program.blocks.push_back(block);
program.block_info.push_back({.id = 0});
auto &lane = block->AppendNewInst(ValueOpcode::LaneId);
auto &selector = block->AppendNewInst(ValueOpcode::BitwiseAnd32,
{Value(&lane), Value(7u)});
auto &key = block->AppendNewInst(ValueOpcode::IAdd32,
{Value(&selector), Value(8u)});
auto &handle = block->AppendNewInst(
ValueOpcode::GetBufferResource,
{Value(&key), Value(0u), Value(0u), Value(0u)});
handle.SetFlags<uint32_t>(0u);
auto &offset = block->AppendNewInst(ValueOpcode::IMul32,
{Value(&lane), Value(4u)});
auto &data = block->AppendNewInst(ValueOpcode::IAdd32,
{Value(&lane), Value(0x1000u)});
auto &exec = block->AppendNewInst(ValueOpcode::INotEqual32,
{Value(&lane), Value(4u)});
block->AppendNewInst(ValueOpcode::StoreBufferU32,
{Value(&handle), Value(0u), Value(&offset), Value(0u),
Value(&data), Value(&exec)});
program.memory_info.push_back({.kind = ResourceKind::Buffer, .offen = true});
BufferResource root;
root.packed_stride = 1;
root.written = true;
root.indirect_root = 0;
root.indirect_mapping_offset = 4;
root.indirect_search_iterations = std::bit_width(3u);
root.indirect_resources = {0u, 1u};
auto candidate = root;
candidate.indirect_search_iterations = 0;
candidate.indirect_resources.clear();
program.descriptor_sources.resize(1);
program.descriptor_sources[0].indirect_descriptor.emplace();
ShaderComputeInputInfo compute;
compute.wave_size = 32;
compute.host_subgroup_size = vulkan.SubgroupSize();
compute.threads_num[0] = 32;
compute.threads_num[1] = compute.threads_num[2] = 1;
for (const u32 null_buffers : {0u, 1u, 2u}) {
program.info.buffers = {root, candidate};
if (null_buffers != 0u) program.info.buffers[1].packed_stride = 0;
if (null_buffers == 2u) {
// Materialization deduplicates all null destinations into one unmapped root.
program.info.buffers.resize(1);
program.info.buffers[0].packed_stride = 0;
program.info.buffers[0].indirect_root = BufferResource::NoIndirectBuffer;
program.info.buffers[0].indirect_search_iterations = 0;
program.info.buffers[0].indirect_resources.clear();
}
program.binding_layout_complete = false;
AllocateBindings(program);
const auto *buffers = FindBinding(program.bindings, DescriptorBindingKind::Buffers);
Require(test.name, "candidate bindings",
buffers != nullptr && buffers->resources ==
(null_buffers == 2u ? std::vector<u32>{0u} : std::vector<u32>{0u, 1u}),
"indirect store lost a buffer binding");
compiled.packed_user_data.resize(program.bindings.ShaderDataDwords());
compiled.packed_user_data[program.bindings.memory_offset_dword] = 12u | (108u << 8u);
compiled.spirv = ShaderRecompiler::Spirv::EmitProgram(program, {.compute = &compute});
ValidateSpirv(test.name, compiled.spirv);
if (null_buffers == 2u) {
test.required_spirv.clear();
test.forbidden_spirv.push_back("OpSwitch");
}
CheckSpirvText(test, compiled.spirv);
for (const bool swapped : {false, true}) {
if (null_buffers != 0u && swapped) continue;
// Repeated keys select two distinct logical buffers within one native allocation.
// Keys below, between and above the live keys must not select either buffer.
compiled.resources.flattened_srt = {0u, 0u, 0u, 0u, 3u,
10u, swapped ? 0u : 1u,
12u, swapped ? 1u : 0u,
14u, swapped ? 0u : 1u};
test.expected = test.initial;
for (const u32 lane : {2u, 6u, 10u, 12u, 14u}) {
const u32 resource = (lane & 7u) == 4u ? swapped : !swapped;
if (null_buffers == 2u || (null_buffers == 1u && resource == 1u)) continue;
test.expected[test.storage_buffer_offsets[resource] / 4u + lane] = 0x1000u + lane;
}
auto output = vulkan.CreateStorageBuffer(test.name, test.initial, test.initial.size());
vulkan.Dispatch(test, compiled, output);
const auto actual = vulkan.ReadBuffer(test.name, output, test.expected.size());
vulkan.DestroyBuffer(&output);
CompareWords(test, "key selection, EXEC, bounds and null descriptors", test.expected, actual);
}
}
std::printf("[compute] %-32s ok\n", test.name);
}
void CheckIndirectImageKeySwitch() { void CheckIndirectImageKeySwitch() {
constexpr const char *name = "IndirectImageKeySwitch"; constexpr const char *name = "IndirectImageKeySwitch";
constexpr uint32_t mapping_capacity = 1793u; constexpr uint32_t mapping_capacity = 1793u;
@ -30621,8 +30732,8 @@ void CheckIndirectImageKeySwitch() {
program.descriptor_sources.resize(2); program.descriptor_sources.resize(2);
program.descriptor_sources[0].dword_count = 8; program.descriptor_sources[0].dword_count = 8;
program.descriptor_sources[0].indirect_image = program.descriptor_sources[0].indirect_descriptor =
DescriptorSource::IndirectImage{0u, 0u, 224u, 12u, 0u}; DescriptorSource::IndirectDescriptor{0u, 0u, 224u, 12u, 0u};
program.descriptor_sources[1].dword_count = 4; program.descriptor_sources[1].dword_count = 4;
ImageResource root{}; ImageResource root{};
@ -30656,7 +30767,7 @@ void CheckIndirectImageKeySwitch() {
text.find("OpSwitch") != std::string::npos && text.find("OpSwitch") != std::string::npos &&
text.find("OpPhi") != std::string::npos && text.find("OpPhi") != std::string::npos &&
CountText(text, "OpImageSampleExplicitLod") == 2 && CountText(text, "OpImageSampleExplicitLod") == 2 &&
CountText(text, "OpIEqual") == 11, CountText(text, "OpIEqual") == 1,
"dynamic image key did not use a compact two-sample switch"); "dynamic image key did not use a compact two-sample switch");
program.memory_info[0].image_dimension = program.memory_info[0].image_dimension =
@ -37292,6 +37403,7 @@ int main(int argc, char **argv) {
} }
if (argc == 2 && std::strcmp(argv[1], "--indirect-buffer-only") == 0) { if (argc == 2 && std::strcmp(argv[1], "--indirect-buffer-only") == 0) {
VulkanHarness vulkan; VulkanHarness vulkan;
CheckIndirectBufferStore(vulkan);
RunCase(&vulkan, BufferLoadsGpuSelectedDescriptors()); RunCase(&vulkan, BufferLoadsGpuSelectedDescriptors());
RunCase(&vulkan, BufferLoadDwordx3GpuSelectedDescriptors()); RunCase(&vulkan, BufferLoadDwordx3GpuSelectedDescriptors());
RunCase(&vulkan, BufferLoadDwordx4SnapshotsOverlappingAddress()); RunCase(&vulkan, BufferLoadDwordx4SnapshotsOverlappingAddress());
@ -37857,6 +37969,7 @@ int main(int argc, char **argv) {
CheckTessellationPrograms(); CheckTessellationPrograms();
CheckPixelParameterAliases(); CheckPixelParameterAliases();
CheckRectListShaders(); CheckRectListShaders();
CheckIndirectBufferStore(vulkan);
CheckIndirectImageKeySwitch(); CheckIndirectImageKeySwitch();
CheckWave64WholeWaveResults(); CheckWave64WholeWaveResults();
CheckPs5GameExampleImageClearRuntimeShape(); CheckPs5GameExampleImageClearRuntimeShape();

107
tests/shaderCfgTests.cpp
View File

@ -8785,6 +8785,112 @@ void TestNativeGuardedSamplerSource() {
} }
} }
void TestBoundedMaterialBufferStores(bool scalar_key) {
// Captured batch search and descriptor waterfall, with deformation arithmetic removed.
const uint32_t shader[] = {
0xbfa00003u, 0xd7460002u, 0x04010a04u, 0xf4080101u,
0xfa000080u, 0xbe8e0380u, 0xbe8f037eu, 0xbf8cc07fu,
0xf4240302u, 0xfa000014u, 0xf4080201u, 0xfa000020u,
0xbf880019u, 0x7e3602c1u, 0xbf8cc07fu, 0xbf0a0d0eu,
0xbf800000u, 0x8506807eu, 0xbeea0306u, 0xbefe0306u,
0xbf860011u, 0x816b0e0cu, 0x810e810eu, 0x936aff6bu,
0x00000050u, 0x7e36026bu, 0x816aa46au, 0xf4240104u,
0xd4000000u, 0xbf8cc07fu, 0xbeea0404u, 0x816a0405u,
0x7d8604f9u, 0x06868405u, 0x7d88046au, 0x876a6a04u,
0x8a7e6a06u, 0xbf89ffe6u, 0xbefe030fu, 0x7daa36c1u,
0xbf880013u, 0xbf8cc07fu, 0xe0342030u, 0x8002171bu,
0xf4080100u, 0xfa000020u, 0x7e020281u, 0x7ed40518u,
0xbeeb037eu, 0x7da4306au, 0x8f6a846au, 0x7e020280u,
0xf4280002u, 0xd4000000u, 0xbf8cc07fu, 0xe0701000u,
0x80003405u, 0xbefe036bu, 0x7daa0280u, 0xbf89fff3u,
0xbf810000u,
};
std::array<uint32_t, 8 * 20> material{};
std::array<uint32_t, 16 * 4> table{};
std::array<uint32_t, 8> batch{};
std::array<uint32_t, 12> table_root{};
std::array<uint32_t, 36> batch_root{};
const auto buffer = [](uint32_t* words, const void* data, uint32_t stride,
uint32_t count, uint32_t mode = 0u) {
const auto address = reinterpret_cast<uint64_t>(data);
words[0] = static_cast<uint32_t>(address);
words[1] = static_cast<uint32_t>(address >> 32u) | (stride << 16u);
words[2] = count;
words[3] = 0x5004u | (mode << 28u);
};
batch[5] = 1u;
batch[6] = 2u;
material[20 + 9] = 32u;
material[20 + 13] = 3u;
material[40 + 9] = 32u;
material[40 + 10] = 32u;
material[40 + 13] = 5u;
buffer(table.data() + 12u, reinterpret_cast<void*>(0x100000u), 1u, 128u, 3u);
buffer(table.data() + 20u, reinterpret_cast<void*>(0x200000u), 1u, 128u, 3u);
buffer(table_root.data() + 8u, table.data(), 16u, 16u);
buffer(batch_root.data() + 8u, material.data(), 80u, 8u);
buffer(batch_root.data() + 32u, batch.data(), 16u, 2u);
const auto table_address = reinterpret_cast<uint64_t>(table_root.data());
const auto batch_address = reinterpret_cast<uint64_t>(batch_root.data());
const std::array<uint32_t, 4> user_data{
static_cast<uint32_t>(table_address), static_cast<uint32_t>(table_address >> 32u),
static_cast<uint32_t>(batch_address), static_cast<uint32_t>(batch_address >> 32u)};
auto options = MakeCompileOptions(ShaderType::Compute);
options.wave_size = 32u;
options.user_data = user_data;
auto code = std::to_array(shader);
if (scalar_key) {
code[0xa8u / 4u] = EncodeMubuf0(0x0cu, 52u);
code[0xacu / 4u] = EncodeMubuf1(24u, 2u, 27u);
}
auto translated = ShaderRecompiler::TranslateProgram(code, options);
auto plan = ShaderRecompiler::IR::ExtractResourcePlan(translated.program);
ShaderRecompiler::IR::ResourceSnapshot snapshot;
ShaderRecompiler::IR::ResourceSpecialization specialization;
const ShaderRecompiler::IR::SrtRuntime runtime{
.user_data = user_data, .read_memory = ReadHostTestMemory,
.read_specialization_memory = ReadHostTestMemory};
const auto materialize = [&] {
return ShaderRecompiler::IR::MaterializeResources(plan, runtime, snapshot, specialization);
};
Check(materialize(), "bounded material range failed to materialize");
const auto& buffers = specialization.buffers;
const auto root = std::ranges::find_if(buffers, [](const auto& resource) {
return resource.indirect_root != ShaderRecompiler::IR::BufferResource::NoIndirectBuffer;
});
Check(root != buffers.end() && snapshot.flattened_srt[root->indirect_mapping_offset] == 2u &&
buffers.size() == plan.info.buffers.size() + 1u,
"bounded material range did not select exactly its two writable buffers");
const auto root_index = static_cast<size_t>(root - buffers.begin());
const auto first_specialization = specialization;
table[12] += 0x1000u;
Check(materialize() && specialization == first_specialization,
"buffer address changes created a shader permutation");
batch[6] = 0u;
Check(materialize() && specialization.buffers[root_index].indirect_root ==
ShaderRecompiler::IR::BufferResource::NoIndirectBuffer &&
std::ranges::all_of(snapshot.buffers[root_index].dwords,
[](uint32_t word) { return word == 0u; }) &&
specialization.buffers.size() == plan.info.buffers.size(),
"empty material batch retained writable candidates");
batch[6] = 2u;
batch[5] = 7u;
Check(!materialize(), "material batch exceeded the descriptor record count");
batch[5] = 1u;
batch_root[11] |= 1u << 30u;
Check(!materialize(), "material selection accepted a non-buffer descriptor type");
batch_root[11] &= ~(1u << 30u);
Check(materialize(), "restored material batch failed to materialize");
auto result = ShaderRecompiler::CompileProgram(std::move(translated), options, specialization, 0u);
CheckSpirvBinaryValidates(result.spirv);
#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
auto unbounded = code;
unbounded[0x3cu / 4u] = EncodeSopc(0x0bu, 14u, 13u); // s_cmp_le_u32 admits wraparound.
ExpectFatal([&] { ShaderRecompiler::TranslateProgram(unbounded, options); },
"material selection accepted an inclusive induction bound");
#endif
}
void TestNativeDescriptorProvenanceRejectsGpuSelection() { void TestNativeDescriptorProvenanceRejectsGpuSelection() {
#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS #if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
const std::array<uint32_t, 13> selected{ const std::array<uint32_t, 13> selected{
@ -14383,6 +14489,7 @@ int main() {
TestNestedSelectionPreservesDescriptorSources(); TestNestedSelectionPreservesDescriptorSources();
TestNativeScalarReadDescriptorPlanning(); TestNativeScalarReadDescriptorPlanning();
TestNativeGuardedSamplerSource(); TestNativeGuardedSamplerSource();
for (const bool scalar_key : {false, true}) TestBoundedMaterialBufferStores(scalar_key);
TestNativeDescriptorProvenanceRejectsGpuSelection(); TestNativeDescriptorProvenanceRejectsGpuSelection();
TestCfgSiblingSharedExit(); TestCfgSiblingSharedExit();
TestNewShaderRecompilerCfgNestedTailEarlyExit(); TestNewShaderRecompilerCfgNestedTailEarlyExit();