shader: prove guarded descriptor counts for interleaved image records

This commit is contained in:
nmzik 2026-10-13 00:07:06 +02:00
parent 56738aa4e1
commit a287a54f26
3 changed files with 261 additions and 54 deletions

View File

@ -1030,34 +1030,37 @@ private:
LaneQuantifier lanes = LaneQuantifier::All;
};
bool NonzeroOnEntry(Value value, const Block* block) const {
// Each unique predecessor must execute before this use. Stop at joins: an
// unrelated comparison is not a bound on FindILsb's zero-input sentinel.
bool HasEntryGuard(const Block* block, const auto& accepts, const Block* definition = nullptr) const {
// Each unique predecessor must execute before this use. Stop at joins and
// definitions so a guard from an earlier iteration cannot bound a new value.
for (size_t depth = 0; block != nullptr && depth < m_program.blocks.size(); ++depth) {
if (block->ImmPredecessors().size() != 1u) {
if (block == definition || block->ImmPredecessors().size() != 1u) {
return false;
}
const auto* previous = block->ImmPredecessors()[0];
const auto edge = ConditionalEdge(previous, block);
if (edge && edge->lanes == LaneQuantifier::All) {
const auto* test = edge->condition.TryInstruction();
if (test != nullptr && test->NumArgs() == 2u &&
((test->GetOpcode() == ValueOpcode::INotEqual32 && edge->positive) ||
(test->GetOpcode() == ValueOpcode::IEqual32 && !edge->positive))) {
for (uint32_t arg = 0; arg < 2u; ++arg) {
uint32_t immediate;
if (ImmediateU32(test->Arg(arg), immediate) && immediate == 0u &&
EquivalentValue(m_program, test->Arg(arg ^ 1u), value)) {
return true;
}
}
}
}
if (edge && edge->lanes == LaneQuantifier::All && accepts(*edge)) return true;
block = previous;
}
return false;
}
bool NonzeroOnEntry(Value value, const Block* block) const {
return HasEntryGuard(block, [&](const EdgePredicate& edge) {
const auto* test = edge.condition.TryInstruction();
if (test != nullptr && test->NumArgs() == 2u &&
((test->GetOpcode() == ValueOpcode::INotEqual32 && edge.positive) ||
(test->GetOpcode() == ValueOpcode::IEqual32 && !edge.positive))) {
for (uint32_t arg = 0; arg < 2u; ++arg) {
uint32_t immediate;
if (ImmediateU32(test->Arg(arg), immediate) && immediate == 0u &&
EquivalentValue(m_program, test->Arg(arg ^ 1u), value)) return true;
}
}
return false;
});
}
uint32_t WorkgroupAxis(Value key) const {
const auto* builtin = key.Resolve().TryInstruction();
uint32_t axis = UINT32_MAX;
@ -1656,23 +1659,33 @@ private:
return false;
}
Value UnsignedBound(Value guard, Value value) const {
Value UnsignedBound(Value guard, Value value, bool positive = true) const {
const auto* test = SimplifyGuard(guard).TryInstruction();
if (test == nullptr) return {};
if (test->GetOpcode() == ValueOpcode::LogicalAnd) {
const auto op = test->GetOpcode();
if (op == ValueOpcode::LogicalNot)
return UnsignedBound(test->Arg(0), value, !positive);
if (op == ValueOpcode::LogicalAnd && positive) {
const auto left = UnsignedBound(test->Arg(0), value);
return left.IsEmpty() ? UnsignedBound(test->Arg(1), value) : left;
}
if (test->GetOpcode() == ValueOpcode::ULessThan32 &&
if ((positive ? op == ValueOpcode::ULessThan32 : op == ValueOpcode::UGreaterThanEqual32) &&
EquivalentValue(m_program, test->Arg(0), value)) return test->Arg(1);
if (test->GetOpcode() == ValueOpcode::LogicalNot) {
const auto* inverse = test->Arg(0).Resolve().TryInstruction();
if (inverse != nullptr && inverse->GetOpcode() == ValueOpcode::ULessThanEqual32 &&
EquivalentValue(m_program, inverse->Arg(1), value)) return inverse->Arg(0);
}
if (!positive && op == ValueOpcode::ULessThanEqual32 &&
EquivalentValue(m_program, test->Arg(1), value)) return test->Arg(0);
return {};
}
Value UnsignedBoundOnEntry(Value key, const Block* block) const {
const auto* definition = key.Resolve().TryInstruction();
Value bound;
const bool found = HasEntryGuard(block, [&](const EdgePredicate& edge) {
bound = UnsignedBound(edge.condition, key, edge.positive);
return !bound.IsEmpty() && ValidateRuntimeValue(m_program, bound, RuntimeValueType::Integer);
}, definition != nullptr ? definition->Parent() : nullptr);
return found ? bound : Value {};
}
Value SharedLoadBound(Value value, Value active) {
const auto* read = ActiveValue(value, active).TryInstruction();
if (read == nullptr || read->GetOpcode() != ValueOpcode::LoadSharedU32 ||
@ -2406,12 +2419,20 @@ private:
// the GPU key using the same plan as other indirect image reads.
plan.retain_reads |= table_source.dword_count == 4u;
} else if (table_source.dword_count == 2u) {
if (table_stride != 32u) return false;
const auto guarded_count = UnsignedBoundOnEntry(key, handle.Parent());
const auto* selector = key.Resolve().TryInstruction();
const bool bitscan = selector != nullptr && selector->GetOpcode() == ValueOpcode::FindILsb32 &&
const bool bitscan = guarded_count.IsEmpty() && selector != nullptr &&
selector->GetOpcode() == ValueOpcode::FindILsb32 &&
selector->NumArgs() == 1u && !m_shader_writes &&
NonzeroOnEntry(selector->Arg(0), handle.Parent());
if (bitscan) {
if (!guarded_count.IsEmpty()) {
if (table_stride < 32u || (table_stride & 3u) != 0u) return false;
// Materialization captures the count and descriptor reads; renderer alias
// checks reject writes to those ranges while unrelated stores remain valid.
indirect.key_count = guarded_count;
} else if (table_stride != 32u) {
return false;
} else if (bitscan) {
indirect.key_count = Value(32u);
} else if (!m_shader_writes) {
if (const auto* bound = BoundedLoop(key, handle.Parent())) {

View File

@ -561,9 +561,14 @@ void TestWaterfallImageTable() {
void TestGuardedDirectImageTable() {
namespace CFG = Libs::Graphics::ShaderRecompiler::CFG;
enum class Guard { Nonzero, SccNonZero, Plain, Zero, Unrelated, Bypass, ExecZero, VccZero };
enum class Guard {
Nonzero, SccNonZero, Plain, Zero, Unrelated, Bypass, ExecZero, VccZero,
Bounded, BoundMissing, BoundUnrelated, BoundBypass, BoundAnyLane
};
const auto make_plan = [](Guard guard) {
Fixture fixture(ShaderType::Pixel);
const bool bounded = guard >= Guard::Bounded;
const bool bypass = guard == Guard::Bypass || guard == Guard::BoundBypass;
Fixture fixture(bounded ? ShaderType::Compute : ShaderType::Pixel);
fixture.program.wave_size = 64u;
auto *entry = fixture.block;
auto *middle = fixture.AddBlock();
@ -575,21 +580,41 @@ void TestGuardedDirectImageTable() {
middle->AddBranch(before_sample);
before_sample->AddBranch(sample);
sample->AddBranch(exit);
if (guard == Guard::Bypass) exit->AddBranch(sample);
const auto mask = fixture.Emit(ValueOpcode::ReadFirstLane,
{fixture.Emit(ValueOpcode::GetAttribute, {Value(0u), Value(0u)}), Value(true)});
const auto nonzero = fixture.Emit(
ValueOpcode::INotEqual32,
{Value(0u), guard == Guard::Unrelated ? fixture.UserData(2) : mask});
const auto kind = guard == Guard::ExecZero ? CFG::BranchCondition::ExecZero
if (bypass) exit->AddBranch(sample);
const auto varying = bounded
? fixture.Emit(ValueOpcode::GetBuiltin,
{Value(static_cast<uint32_t>(StageInputKind::LocalInvocationId)), Value(0u)})
: fixture.Emit(ValueOpcode::GetAttribute, {Value(0u), Value(0u)});
const auto mask = fixture.Emit(ValueOpcode::ReadFirstLane, {varying, Value(true)});
const auto srt = fixture.Address(fixture.UserData(0), fixture.UserData(1));
Value key;
Value limit;
if (bounded) {
key = fixture.Emit(ValueOpcode::FindILsb32, {mask});
MemoryInfo memory;
memory.kind = ResourceKind::ScalarAddress;
memory.offset = 4u;
limit = fixture.Emit(ValueOpcode::LoadAddressU32,
{srt, Value(0u), Value(0u), Value(true)}, fixture.AddMemory(memory, 0x10));
}
const auto kind = guard == Guard::ExecZero || guard == Guard::BoundAnyLane
? CFG::BranchCondition::ExecZero
: guard == Guard::VccZero ? CFG::BranchCondition::VccZero
: CFG::BranchCondition::SccZero;
auto condition = nonzero;
if (guard == Guard::SccNonZero) {
condition = fixture.Emit(ValueOpcode::ConditionRef, {condition},
CFG::BranchCondition::SccNonZero);
Value condition;
if (bounded) {
condition = guard == Guard::BoundMissing ? Value(false) : fixture.Emit(
ValueOpcode::UGreaterThanEqual32,
{guard == Guard::BoundUnrelated ? fixture.UserData(2) : key, limit});
} else {
condition = fixture.Emit(ValueOpcode::INotEqual32,
{Value(0u), guard == Guard::Unrelated ? fixture.UserData(2) : mask});
if (guard == Guard::SccNonZero) {
condition = fixture.Emit(ValueOpcode::ConditionRef, {condition},
CFG::BranchCondition::SccNonZero);
}
condition = fixture.Emit(ValueOpcode::LogicalNot, {condition});
}
condition = fixture.Emit(ValueOpcode::LogicalNot, {condition});
if (guard != Guard::Plain && guard != Guard::SccNonZero) {
condition = fixture.Emit(ValueOpcode::ConditionRef, {condition}, kind);
}
@ -604,12 +629,14 @@ void TestGuardedDirectImageTable() {
.true_block = fixture.program.blocks[block + 1u]};
}
fixture.program.blocks[4]->terminator = {
.kind = guard == Guard::Bypass ? CFG::TerminatorKind::Branch
: CFG::TerminatorKind::Return,
.kind = bypass ? CFG::TerminatorKind::Branch : CFG::TerminatorKind::Return,
.true_block = fixture.program.blocks[3u]};
const auto srt = fixture.Address(fixture.UserData(0), fixture.UserData(1));
std::array<Value, 2> pointer;
for (uint32_t word = 0; word < pointer.size(); ++word) {
if (bounded) {
pointer[word] = srt.Instruction()->Arg(word);
continue;
}
MemoryInfo memory;
memory.kind = ResourceKind::ScalarAddress;
memory.offset = word * 4u;
@ -618,10 +645,11 @@ void TestGuardedDirectImageTable() {
fixture.AddMemory(memory, 0x20));
}
fixture.block = sample;
const auto key = fixture.Emit(ValueOpcode::FindILsb32, {mask});
if (!bounded) key = fixture.Emit(ValueOpcode::FindILsb32, {mask});
const auto offset = fixture.Emit(ValueOpcode::IAdd32,
{fixture.Emit(ValueOpcode::ShiftLeftLogical32, {key, Value(5u)}),
Value(344u)});
{bounded ? fixture.Emit(ValueOpcode::IMul32, {key, Value(272u)})
: fixture.Emit(ValueOpcode::ShiftLeftLogical32, {key, Value(5u)}),
Value(bounded ? 3696u : 344u)});
std::array<Value, 8> words;
for (uint32_t group = 0; group < 2u; ++group) {
// Separate equivalent pointer handles mirror the two scalar x4 loads.
@ -645,13 +673,25 @@ void TestGuardedDirectImageTable() {
memory.image_dimension = Decoder::ImageDimension::Dim2D;
fixture.Emit(ValueOpcode::ImageSampleRaw, {image, sampler, fixture.ImageAddress(), Value(true)},
fixture.AddMemory(memory, 0x128));
if (bounded) {
const auto output = fixture.Image({Value(0x8000u),
Value(static_cast<uint32_t>(Libs::Graphics::Prospero::BufferFormat::k32UInt) << 20u),
Value(0u), Value((static_cast<uint32_t>(Libs::Graphics::Prospero::ImageType::kColor2D) << 28u) |
Libs::Graphics::DstSel(4, 5, 6, 7)),
Value(0u), Value(0u), Value(0u), Value(0u)}, 0x130);
const auto texel = fixture.Emit(ValueOpcode::CompositeConstructU32x4,
{Value(0u), Value(0u), Value(0u), Value(0u)});
fixture.Emit(ValueOpcode::ImageWrite, {output, fixture.ImageAddress(), texel, Value(true)},
fixture.AddMemory(memory, 0x130));
}
fixture.PlanAndTrack();
const auto source = fixture.program.info.images[0].source;
const auto &indirect = fixture.program.descriptor_sources[source].indirect_descriptor;
Check(indirect && !indirect->selector.has_value() &&
indirect->table_offset == 344u &&
indirect->key_count.Resolve().IsImmediate() &&
indirect->key_count.Resolve().U32() == 32u &&
indirect->table_offset == (bounded ? 3696u : 344u) &&
indirect->table_stride == (bounded ? 272u : 32u) &&
(bounded ? !indirect->key_count.IsEmpty()
: indirect->key_count.Resolve() == Value(32u)) &&
fixture.program.descriptor_sources[indirect->table_source].dword_count == 2u,
"guarded direct image table lost its pointer or proven selector range");
return ExtractResourcePlan(fixture.program);
@ -661,9 +701,10 @@ void TestGuardedDirectImageTable() {
make_plan(Guard::SccNonZero);
make_plan(Guard::Plain);
for (const auto guard : {Guard::Zero, Guard::Unrelated, Guard::Bypass,
Guard::ExecZero, Guard::VccZero}) {
Guard::ExecZero, Guard::VccZero, Guard::BoundMissing,
Guard::BoundUnrelated, Guard::BoundBypass, Guard::BoundAnyLane}) {
CheckFatal([&] { make_plan(guard); }, "not a valid runtime value",
"direct table accepted a selector without a dominating nonzero guard");
"direct table accepted a selector without a dominating range guard");
}
LinearTestMemory memory;
constexpr uint64_t table = 0x1800u + 344u;
@ -737,6 +778,53 @@ void TestGuardedDirectImageTable() {
Check(!MaterializeResources(plan, runtime, snapshot, specialization) &&
endpoint.watched_reads == 0u,
"batched descriptor read crossed the 48-bit endpoint");
auto bounded = make_plan(Guard::Bounded);
Check(bounded.capture_specialization_reads && bounded.info.images[1].written,
"bounded table with image writes omitted specialization read capture");
LinearTestMemory records;
constexpr uint32_t record_offset = 3696u, record_stride = 272u;
for (uint32_t key = 0; key < 2u; ++key) {
const auto word = (record_offset + key * record_stride) / 4u;
records.words[word] = 0x500u + key;
records.words[word + 1u] = static_cast<uint32_t>(
Libs::Graphics::Prospero::BufferFormat::k32_32_32_32Float) << 20u;
records.words[word + 3u] = Libs::Graphics::DstSel(4, 5, 6, 7) |
(static_cast<uint32_t>(Libs::Graphics::Prospero::ImageType::kColor2D) << 28u);
}
user_data = {static_cast<uint32_t>(records.base), 0u};
runtime.userdata = &records;
records.fail_address = records.base + record_offset;
Check(MaterializeResources(bounded, runtime, snapshot, specialization) &&
snapshot.images.size() == 2u && records.descriptor_reads == 0u &&
std::ranges::all_of(snapshot.images[0].dwords, [](uint32_t word) { return word == 0u; }),
"zero runtime count probed an unreachable descriptor row");
records.fail_address = UINT64_MAX;
records.words[1] = 2u;
records.descriptor_reads = 0u;
Check(MaterializeResources(bounded, runtime, snapshot, specialization) &&
snapshot.images.size() == 3u && records.descriptor_reads == 2u &&
snapshot.images[0].dwords[0] == 0x500u && snapshot.images[2].dwords[0] == 0x501u,
"bounded interleaved table did not read precisely its two descriptor rows");
const auto captured = [&](uint64_t address, uint64_t size) {
return std::ranges::any_of(snapshot.specialization_reads, [&](const auto &read) {
return read.first <= address && address - read.first <= read.second &&
size <= read.second - (address - read.first);
});
};
Check(captured(records.base + 4u, 4u) &&
captured(records.base + record_offset, 32u) &&
captured(records.base + record_offset + record_stride, 32u),
"bounded table omitted count or descriptor bytes from writable-alias validation");
records.fail_address = records.base + record_offset + record_stride + 28u;
Check(!MaterializeResources(bounded, runtime, snapshot, specialization),
"bounded interleaved table accepted an unreadable final descriptor word");
records.fail_address = UINT64_MAX;
records.words[1] = UINT32_MAX;
records.descriptor_reads = 0u;
Check(!MaterializeResources(bounded, runtime, snapshot, specialization) &&
records.descriptor_reads == 0u,
"unrepresentable table span was read before validating its count");
}
void TestBoundedComputeImageLoop() {

View File

@ -36410,6 +36410,102 @@ void CheckCountedRowImageSelector(VulkanHarness &vulkan) {
}
}
void CheckGuardedRecordImageTable(VulkanHarness &vulkan) {
using O = ShaderOpcode;
constexpr u32 table = 256u, stride = 272u, offset = 3696u;
for (const u32 wave : {32u, 64u})
for (const u32 count : {0u, 2u, 3u}) {
TestCase test;
test.name = wave == 32u ? "GuardedRecordImageTable32" : "GuardedRecordImageTable64";
test.initial.resize((table + offset + stride * count) / 4u, 0xdeadbeefu);
test.initial[table / 4u + 1u] = count;
for (u32 row = 0; row < count; ++row) {
auto image = MakeSampledTextureData(Prospero::BufferFormat::k32Float);
image[0] = row + 1u;
image[3] = DstSel(4, 0, 0, 1) |
(static_cast<u32>(Prospero::ImageType::kColor2D) << 28u);
std::copy_n(image.begin(), 8, test.initial.begin() + (table + offset + row * stride) / 4u);
}
test.user_data[0] = table;
auto destination = MakeSampledTextureData(Prospero::BufferFormat::k32Float);
destination[0] = 0x10000u;
destination[1] |= 3u << 30u;
destination[3] = DstSel(4, 0, 0, 1) |
(static_cast<u32>(Prospero::ImageType::kColor2D) << 28u);
std::copy_n(destination.begin(), 8, test.user_data.begin() + 32);
test.has_user_data = test.has_compute_info = true;
test.compute_info.wave_size = wave;
test.compute_info.threads_num[0] = test.compute_info.threads_num[1] =
test.compute_info.threads_num[2] = 1u;
test.compute_info.group_id[0] = true;
test.compute_info.workgroup_register = 44u;
test.dispatch_x = 4u;
auto &code = test.code;
code.insert(code.end(), {EncodeSop2(0x20, 20, InlineU32(5), 44),
EncodeSmem0(0x00, 21), EncodeSmem1(4, 125)});
AppendVMovLiteral(&code, 0, 0u);
AppendVMovLiteral(&code, 20, 0x3f000000u);
AppendVMovLiteral(&code, 21, 0x3f000000u);
const auto loop = code.size();
code.push_back(EncodeSop1(0x13, 22, 20)); // Find the next set bit in the GPU worklist.
code.push_back(EncodeSopc(0x07, 22, 193));
const auto empty_exit = code.size(); code.push_back(0u);
code.push_back(EncodeSopc(0x09, 22, 21));
const auto range_exit = code.size(); code.push_back(0u);
code.insert(code.end(), {EncodeSop2(0x26, 23, 22, 255), stride,
EncodeSop2(0x02, 23, 23, 255), offset,
EncodeSmem0(0x03, 8), EncodeSmem1(0, 23),
EncodeMimg0(0x27, 1), EncodeMimg1(4, 20, 2, 10),
EncodeVop2(0x03, 0, Vgpr(4), 0),
EncodeSop2(0x1e, 24, InlineU32(1), 22),
EncodeSop2(0x14, 20, 20, 24)});
code.push_back(EncodeSopp(0x02, loop - code.size() - 1u));
code[empty_exit] = EncodeSopp(0x04, code.size() - empty_exit - 1u);
code[range_exit] = EncodeSopp(0x05, code.size() - range_exit - 1u);
code.push_back(EncodeVop1(0x01, 30, 44));
AppendVMovLiteral(&code, 31, 0u);
code.insert(code.end(), {EncodeMimg0(0x08, 1), EncodeMimg1(0, 30, 8, 0)});
AppendEnd(&code);
test.opcodes = {O::S_LSHR_B32, O::S_LOAD_DWORD, O::V_MOV_B32, O::S_FF1_I32_B32,
O::S_CMP_LG_U32, O::S_CMP_GE_U32, O::S_CBRANCH_SCC0, O::S_CBRANCH_SCC1,
O::S_MUL_I32, O::S_ADD_I32, O::S_LOAD_DWORDX8, O::IMAGE_SAMPLE,
O::V_ADD_F32, O::S_LSHL_B32, O::S_ANDN2_B32, O::S_BRANCH,
O::IMAGE_STORE, O::S_ENDPGM};
auto compiled = CompileCase(test, vulkan.SubgroupSize());
const auto sampled_count = std::ranges::count_if(compiled.program.info.images,
[](const auto &image) {
return image.resource_class == ShaderRecompiler::IR::ImageResourceClass::Sampled;
});
Require(test.name, "bounded descriptor rows", sampled_count == std::max(1u, count),
"the runtime count lost a row or probed record padding");
std::vector<VulkanHarness::Image> images(compiled.program.info.images.size());
for (u32 resource = 0; resource < images.size(); ++resource) {
if (compiled.program.info.images[resource].resource_class !=
ShaderRecompiler::IR::ImageResourceClass::Sampled) continue;
const auto value = static_cast<float>(compiled.resources.images[resource].dwords[0]);
images[resource] = vulkan.CreateImage2D(test.name, 1, 1, vk::Format::eR32Sfloat,
vk::ImageUsageFlagBits::eSampled, {std::bit_cast<u32>(value)}, 1,
vk::ImageLayout::eShaderReadOnlyOptimal);
}
auto storage = vulkan.CreateImage2D(test.name, 4, 1, vk::Format::eR32Sfloat,
vk::ImageUsageFlagBits::eStorage, std::vector<u32>(4, 0xdeadbeefu), 1,
vk::ImageLayout::eGeneral);
auto buffer = vulkan.CreateStorageBuffer(test.name, test.initial, test.initial.size());
auto sampler = vulkan.CreateSampler(test.name, 1);
vulkan.Dispatch(test, compiled, buffer, nullptr, nullptr, &storage, nullptr, sampler, images);
const std::vector<u32> expected = count == 0 ? std::vector<u32>(4, 0u) :
std::vector<u32>{std::bit_cast<u32>(count == 3 ? 4.0f : 1.0f),
std::bit_cast<u32>(2.0f), std::bit_cast<u32>(1.0f), 0u};
CompareWords(test, "GPU worklist, guarded count and image writes", expected,
vulkan.ReadImage(test.name, &storage));
vulkan.DestroyBuffer(&buffer);
vulkan.DestroyImage(&storage);
vulkan.Device().destroySampler(sampler);
for (auto &image : images) if (image.image != nullptr) vulkan.DestroyImage(&image);
std::printf("[compute] %s count=%u ok\n", test.name, count);
}
}
void CheckIndirectImageKeySwitch(VulkanHarness &vulkan) {
constexpr const char *name = "IndirectImageKeySwitch";
constexpr uint32_t mapping_capacity = 1793u;
@ -44872,6 +44968,7 @@ int main(int argc, char **argv) {
CheckImageSamplerSpecialization();
VulkanHarness vulkan;
CheckCountedRowImageSelector(vulkan);
CheckGuardedRecordImageTable(vulkan);
CheckIndirectImageKeySwitch(vulkan);
CheckFiniteInlineSamplerPhi(vulkan);
RunCase(&vulkan, ImageCubeGradientsPreserveDerivatives());
@ -45038,6 +45135,7 @@ int main(int argc, char **argv) {
CheckRuntimeBufferRecords(vulkan);
CheckComputeThreadDimensions(vulkan);
CheckCountedRowImageSelector(vulkan);
CheckGuardedRecordImageTable(vulkan);
CheckIndirectImageKeySwitch(vulkan);
CheckDynamicSampleOffsets(vulkan, 32u);
CheckDynamicSampleOffsets(vulkan, 64u);