From ed1195d1e76ad047393c4ea4722bd00bf1cc21df Mon Sep 17 00:00:00 2001 From: nmzik Date: Tue, 13 Oct 2026 00:22:23 +0200 Subject: [PATCH] shader: prove descriptor bounds from simplified image EXEC predicates --- .../ir/passes/ConstantPropagation.cpp | 84 ++++++++-- .../recompiler/ir/passes/ResourceTracking.cpp | 28 +++- tests/ResourceTrackingTests.cpp | 152 +++++++++++------- tests/ShaderRecompilerComputeTests.cpp | 51 ++++-- 4 files changed, 231 insertions(+), 84 deletions(-) diff --git a/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp b/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp index 3de2a8df..1a4acd01 100644 --- a/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp @@ -248,6 +248,40 @@ bool FoldCompositeExtract(Inst& inst, ValueOpcode construct, size_t components) return false; } +void FoldZeroComparison(Block& block, Block::iterator instruction, FoldWorklist& worklist) { + auto& inst = *instruction; + Value value; + for (uint32_t arg = 0; arg < 2u; ++arg) + if (Arg(inst, arg) == Value(0u)) value = Arg(inst, arg ^ 1u); + const auto* source = value.TryInstruction(); + if (source == nullptr) return; + const bool equal = inst.GetOpcode() == ValueOpcode::IEqual32; + const auto boolean_select = [](const Inst* select) { + return select != nullptr && select->GetOpcode() == ValueOpcode::SelectU32 && + IsImmediate(Arg(*select, 1), Type::U32) && + IsImmediate(Arg(*select, 2), Type::U32); + }; + if (boolean_select(source)) { + const bool when_true = (Arg(*source, 1).U32() == 0u) == equal; + const bool when_false = (Arg(*source, 2).U32() == 0u) == equal; + if (when_true == when_false) Replace(inst, Value(when_true)); + else if (when_true) Replace(inst, Arg(*source, 0)); + else Replace(inst, Value(&*worklist.Prepend(block, instruction, ValueOpcode::LogicalNot, + {Arg(*source, 0)}))); + } else if (source->GetOpcode() == ValueOpcode::BitwiseOr32 && + (boolean_select(Arg(*source, 0).TryInstruction()) || + boolean_select(Arg(*source, 1).TryInstruction()))) { + // SCC encoded as a DWORD needs no select or bitwise operation when only + // its zero predicate is consumed. Keep arbitrary DWORDs as zero comparisons. + const auto lhs = worklist.Prepend(block, instruction, inst.GetOpcode(), + {Arg(*source, 0), Value(0u)}); + const auto rhs = worklist.Prepend(block, instruction, inst.GetOpcode(), + {Arg(*source, 1), Value(0u)}); + Replace(inst, Value(&*worklist.Prepend(block, instruction, + equal ? ValueOpcode::LogicalAnd : ValueOpcode::LogicalOr, {Value(&*lhs), Value(&*rhs)}))); + } +} + void FoldInstruction(Block& block, Block::iterator instruction, FoldWorklist& worklist) { auto& inst = *instruction; @@ -630,10 +664,12 @@ void FoldInstruction(Block& block, Block::iterator instruction, FoldU32(inst, [](uint32_t a, uint32_t b) { return std::max(a, b); }); return; case ValueOpcode::IEqual32: - FoldU32Compare(inst, [](uint32_t a, uint32_t b) { return a == b; }); + if (!FoldU32Compare(inst, [](uint32_t a, uint32_t b) { return a == b; })) + FoldZeroComparison(block, instruction, worklist); return; case ValueOpcode::INotEqual32: - FoldU32Compare(inst, [](uint32_t a, uint32_t b) { return a != b; }); + if (!FoldU32Compare(inst, [](uint32_t a, uint32_t b) { return a != b; })) + FoldZeroComparison(block, instruction, worklist); return; case ValueOpcode::ULessThan32: FoldU32Compare(inst, [](uint32_t a, uint32_t b) { return a < b; }); @@ -695,19 +731,43 @@ void FoldInstruction(Block& block, Block::iterator instruction, const auto lhs = Arg(inst, 0); const auto rhs = Arg(inst, 1); const auto simplify = [&](Value assumption, Value expression) { - const auto* disjunction = expression.TryInstruction(); - if (disjunction == nullptr || disjunction->GetOpcode() != ValueOpcode::LogicalOr) { - return false; + const auto* test = expression.TryInstruction(); + if (test == nullptr) return false; + const auto replace = [&](Value value) { + inst.SetArg(assumption == lhs ? 1u : 0u, value); + return true; + }; + const auto negate = [&](Value value) { + return Value(&*worklist.Prepend(block, instruction, ValueOpcode::LogicalNot, {value})); + }; + const bool inverse = test->GetOpcode() == ValueOpcode::LogicalNot; + if (inverse) { + test = Arg(*test, 0).TryInstruction(); + if (test == nullptr) return false; + if (test->GetOpcode() == ValueOpcode::LogicalAnd) + for (uint32_t arg = 0; arg < 2u; ++arg) + if (Arg(*test, arg) == assumption) + return replace(negate(Arg(*test, arg ^ 1u))); } - for (uint32_t i = 0; i < 2u; ++i) { - const auto* inverse = disjunction->Arg(i).Resolve().TryInstruction(); - if (inverse != nullptr && inverse->GetOpcode() == ValueOpcode::LogicalNot && - inverse->Arg(0).Resolve() == assumption) { - inst.SetArg(assumption == lhs ? 1u : 0u, - disjunction->Arg(i ^ 1u)); - return true; + if (test->GetOpcode() == ValueOpcode::IEqual32 || + test->GetOpcode() == ValueOpcode::INotEqual32) { + for (uint32_t arg = 0; arg < 2u; ++arg) { + if (Arg(*test, arg) != Value(0u)) continue; + const auto original = Arg(*test, arg ^ 1u); + const auto active = ResolveActiveU32(original, assumption); + if (!active.IsEmpty() && active != original) { + const auto comparison = Value(&*worklist.Prepend(block, instruction, + test->GetOpcode(), {active, Value(0u)})); + return replace(inverse ? negate(comparison) : comparison); + } } } + if (inverse || test->GetOpcode() != ValueOpcode::LogicalOr) return false; + for (uint32_t arg = 0; arg < 2u; ++arg) { + const auto* negation = Arg(*test, arg).TryInstruction(); + if (negation != nullptr && negation->GetOpcode() == ValueOpcode::LogicalNot && + Arg(*negation, 0) == assumption) return replace(Arg(*test, arg ^ 1u)); + } return false; }; if (simplify(lhs, rhs) || simplify(rhs, lhs)) return; diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp index f850de53..c27b7761 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp @@ -1656,9 +1656,10 @@ private: const auto op = test->GetOpcode(); if (op == ValueOpcode::LogicalNot) return UnsignedBound(test->Arg(0), value, !positive); - if (op == ValueOpcode::LogicalAnd && positive) { - const auto left = UnsignedBound(test->Arg(0), value); - return left.IsEmpty() ? UnsignedBound(test->Arg(1), value) : left; + if ((op == ValueOpcode::LogicalAnd && positive) || + (op == ValueOpcode::LogicalOr && !positive)) { + const auto left = UnsignedBound(test->Arg(0), value, positive); + return left.IsEmpty() ? UnsignedBound(test->Arg(1), value, positive) : left; } if ((positive ? op == ValueOpcode::ULessThan32 : op == ValueOpcode::UGreaterThanEqual32) && EquivalentValue(m_program, test->Arg(0), value)) return test->Arg(1); @@ -1684,6 +1685,24 @@ private: return guarded ? bound : Value {}; } + Value UnsignedImageBound(Value key, const Inst& handle) const { + Value bound; + for (const auto& use: handle.Uses()) { + const auto& image = *use.user; + const auto op = image.GetOpcode(); + if (use.operand != 0u || (op != ValueOpcode::ImageSampleRaw && + op != ValueOpcode::ImageGatherRaw && op != ValueOpcode::ImageRead)) return {}; + // The image's own EXEC predicate proves the range for every accessing lane; + // an EXEC-nonzero branch alone only proves that some lane is active. + const auto candidate = UnsignedBound(image.Arg(image.NumArgs() - 1u), key); + if (candidate.IsEmpty() || + !ValidateRuntimeValue(m_program, candidate, RuntimeValueType::Integer)) return {}; + if (bound.IsEmpty()) bound = candidate; + else if (!EquivalentValue(m_program, candidate, bound)) return {}; + } + return bound; + } + Value SharedLoadBound(Value value, Value active) { const auto* read = ActiveValue(value, active).TryInstruction(); if (read == nullptr || read->GetOpcode() != ValueOpcode::LoadSharedU32 || @@ -2417,7 +2436,8 @@ private: // the GPU key using the same plan as other indirect image reads. plan.retain_reads |= table_source.dword_count == 4u; } else if (table_source.dword_count == 2u) { - const auto guarded_count = UnsignedBoundOnEntry(key, handle.Parent()); + auto guarded_count = UnsignedBoundOnEntry(key, handle.Parent()); + if (guarded_count.IsEmpty()) guarded_count = UnsignedImageBound(key, handle); const auto* selector = key.Resolve().TryInstruction(); const bool bitscan = guarded_count.IsEmpty() && selector != nullptr && selector->GetOpcode() == ValueOpcode::FindILsb32 && diff --git a/tests/ResourceTrackingTests.cpp b/tests/ResourceTrackingTests.cpp index d1bced8d..80a862de 100644 --- a/tests/ResourceTrackingTests.cpp +++ b/tests/ResourceTrackingTests.cpp @@ -564,11 +564,15 @@ void TestGuardedDirectImageTable() { enum class Guard { Nonzero, SccNonZero, Plain, Zero, Unrelated, Bypass, ExecZero, VccZero, Bounded, BoundMissing, BoundUnrelated, BoundBypass, BoundAnyLane, - BoundDiamond, BoundBranchSpecific, BoundMismatched + BoundDiamond, BoundBranchSpecific, BoundMismatched, + BoundExec, BoundExecUnguarded, BoundExecMismatched, BoundExecAnd, BoundExecUnprovenActive }; const auto make_plan = [](Guard guard) { const bool bounded = guard >= Guard::Bounded; + const bool exec_bound = guard >= Guard::BoundExec; const bool bypass = guard == Guard::Bypass || guard == Guard::BoundBypass; + const uint32_t table_offset = exec_bound ? 672u : bounded ? 3696u : 344u; + const uint32_t table_stride = exec_bound ? 784u : bounded ? 272u : 32u; Fixture fixture(bounded ? ShaderType::Compute : ShaderType::Pixel); fixture.program.wave_size = 64u; auto *entry = fixture.block; @@ -592,15 +596,30 @@ void TestGuardedDirectImageTable() { limit = fixture.Emit(ValueOpcode::LoadAddressU32, {srt, Value(0u), Value(0u), Value(true)}, fixture.AddMemory(memory, 0x10)); } - const auto kind = guard == Guard::ExecZero || guard == Guard::BoundAnyLane + const auto kind = guard == Guard::ExecZero || guard == Guard::BoundAnyLane || exec_bound ? CFG::BranchCondition::ExecZero : guard == Guard::VccZero ? CFG::BranchCondition::VccZero : CFG::BranchCondition::SccZero; Value condition; + Value active(true); if (bounded) { condition = guard == Guard::BoundMissing ? Value(false) : fixture.Emit( ValueOpcode::UGreaterThanEqual32, {guard == Guard::BoundUnrelated ? fixture.UserData(2) : key, limit}); + if (exec_bound) { + const auto previous = fixture.Emit(ValueOpcode::INotEqual32, {varying, Value(2u)}); + const auto encoded = fixture.Emit(ValueOpcode::SelectU32, {condition, Value(1u), Value(0u)}); + const auto stopped = fixture.Emit(guard == Guard::BoundExecAnd + ? ValueOpcode::BitwiseAnd32 : ValueOpcode::BitwiseOr32, {encoded, varying}); + const auto retained = fixture.Emit(ValueOpcode::SelectU32, + {previous, stopped, Value(0u)}); + const auto vcc = fixture.Emit(ValueOpcode::LogicalAnd, {previous, + fixture.Emit(ValueOpcode::INotEqual32, {retained, Value(0u)})}); + const auto remaining = fixture.Emit(ValueOpcode::LogicalNot, {vcc}); + active = guard == Guard::BoundExecUnprovenActive ? remaining + : fixture.Emit(ValueOpcode::LogicalAnd, {previous, remaining}); + condition = fixture.Emit(ValueOpcode::LogicalNot, {active}); + } } else { condition = fixture.Emit(ValueOpcode::INotEqual32, {Value(0u), guard == Guard::Unrelated ? fixture.UserData(2) : mask}); @@ -626,7 +645,7 @@ void TestGuardedDirectImageTable() { fixture.program.blocks[4]->terminator = { .kind = bypass ? CFG::TerminatorKind::Branch : CFG::TerminatorKind::Return, .true_block = fixture.program.blocks[3u]}; - if (guard == Guard::BoundDiamond) { + if (guard == Guard::BoundDiamond || exec_bound) { middle->condition = fixture.Emit(ValueOpcode::IEqual32, {mask, Value(1u)}); middle->terminator = {.kind = CFG::TerminatorKind::ConditionalBranch, @@ -668,9 +687,9 @@ void TestGuardedDirectImageTable() { fixture.block = sample; if (!bounded) key = fixture.Emit(ValueOpcode::FindILsb32, {mask}); const auto offset = fixture.Emit(ValueOpcode::IAdd32, - {bounded ? fixture.Emit(ValueOpcode::IMul32, {key, Value(272u)}) + {bounded ? fixture.Emit(ValueOpcode::IMul32, {key, Value(table_stride)}) : fixture.Emit(ValueOpcode::ShiftLeftLogical32, {key, Value(5u)}), - Value(bounded ? 3696u : 344u)}); + Value(table_offset)}); std::array words; for (uint32_t group = 0; group < 2u; ++group) { // Separate equivalent pointer handles mirror the two scalar x4 loads. @@ -692,8 +711,14 @@ void TestGuardedDirectImageTable() { MemoryInfo memory; memory.kind = ResourceKind::Image; memory.image_dimension = Decoder::ImageDimension::Dim2D; - fixture.Emit(ValueOpcode::ImageSampleRaw, {image, sampler, fixture.ImageAddress(), Value(true)}, + fixture.Emit(ValueOpcode::ImageSampleRaw, {image, sampler, fixture.ImageAddress(), active}, fixture.AddMemory(memory, 0x128)); + if (guard == Guard::BoundExecUnguarded || guard == Guard::BoundExecMismatched) { + const auto other_active = guard == Guard::BoundExecUnguarded ? Value(true) + : fixture.Emit(ValueOpcode::ULessThan32, {key, fixture.UserData(2)}); + fixture.Emit(ValueOpcode::ImageSampleRaw, + {image, sampler, fixture.ImageAddress(), other_active}, fixture.AddMemory(memory, 0x12c)); + } if (bounded) { const auto output = fixture.Image({Value(0x8000u), Value(static_cast(Libs::Graphics::Prospero::BufferFormat::k32UInt) << 20u), @@ -705,12 +730,26 @@ void TestGuardedDirectImageTable() { fixture.Emit(ValueOpcode::ImageWrite, {output, fixture.ImageAddress(), texel, Value(true)}, fixture.AddMemory(memory, 0x130)); } + if (exec_bound) ConstantPropagationPass(fixture.program.blocks); + if (guard == Guard::BoundExec) { + std::vector pending{active.ResolveInstruction()}; + for (size_t index = 0; index < pending.size(); ++index) { + const auto *inst = pending[index]; + Check(inst->GetOpcode() != ValueOpcode::SelectU32 && + inst->GetOpcode() != ValueOpcode::BitwiseOr32, + "EXEC bound retained its integer Boolean encoding"); + for (size_t arg = 0; arg < inst->NumArgs(); ++arg) { + const auto *operand = inst->Arg(arg).Resolve().TryInstruction(); + if (operand != nullptr && std::ranges::find(pending, operand) == pending.end()) + pending.push_back(operand); + } + } + } fixture.PlanAndTrack(); const auto source = fixture.program.info.images[0].source; const auto &indirect = fixture.program.descriptor_sources[source].indirect_descriptor; Check(indirect && !indirect->selector.has_value() && - indirect->table_offset == (bounded ? 3696u : 344u) && - indirect->table_stride == (bounded ? 272u : 32u) && + indirect->table_offset == table_offset && indirect->table_stride == table_stride && (bounded ? !indirect->key_count.IsEmpty() : indirect->key_count.Resolve() == Value(32u)) && fixture.program.descriptor_sources[indirect->table_source].dword_count == 2u, @@ -724,7 +763,9 @@ void TestGuardedDirectImageTable() { for (const auto guard : {Guard::Zero, Guard::Unrelated, Guard::Bypass, Guard::ExecZero, Guard::VccZero, Guard::BoundMissing, Guard::BoundUnrelated, Guard::BoundBypass, Guard::BoundAnyLane, - Guard::BoundBranchSpecific, Guard::BoundMismatched}) { + Guard::BoundBranchSpecific, Guard::BoundMismatched, + Guard::BoundExecUnguarded, Guard::BoundExecMismatched, + Guard::BoundExecAnd, Guard::BoundExecUnprovenActive}) { CheckFatal([&] { make_plan(guard); }, "not a valid runtime value", "direct table accepted a selector without a dominating range guard"); } @@ -802,52 +843,55 @@ void TestGuardedDirectImageTable() { "batched descriptor read crossed the 48-bit endpoint"); make_plan(Guard::Bounded); - auto bounded = make_plan(Guard::BoundDiamond); - Check(bounded.capture_specialization_reads && bounded.info.images[1].written, - "bounded table with image writes omitted specialization read capture"); - LinearTestMemory records; - constexpr uint32_t record_offset = 3696u, record_stride = 272u; - for (uint32_t key = 0; key < 2u; ++key) { - const auto word = (record_offset + key * record_stride) / 4u; - records.words[word] = 0x500u + key; - records.words[word + 1u] = static_cast( - Libs::Graphics::Prospero::BufferFormat::k32_32_32_32Float) << 20u; - records.words[word + 3u] = Libs::Graphics::DstSel(4, 5, 6, 7) | - (static_cast(Libs::Graphics::Prospero::ImageType::kColor2D) << 28u); + for (const auto guard : {Guard::BoundDiamond, Guard::BoundExec}) { + auto bounded = make_plan(guard); + Check(bounded.capture_specialization_reads && bounded.info.images[1].written, + "bounded table with image writes omitted specialization read capture"); + LinearTestMemory records; + const uint32_t record_offset = guard == Guard::BoundExec ? 672u : 3696u; + const uint32_t record_stride = guard == Guard::BoundExec ? 784u : 272u; + for (uint32_t key = 0; key < 2u; ++key) { + const auto word = (record_offset + key * record_stride) / 4u; + records.words[word] = 0x500u + key; + records.words[word + 1u] = static_cast( + Libs::Graphics::Prospero::BufferFormat::k32_32_32_32Float) << 20u; + records.words[word + 3u] = Libs::Graphics::DstSel(4, 5, 6, 7) | + (static_cast(Libs::Graphics::Prospero::ImageType::kColor2D) << 28u); + } + user_data = {static_cast(records.base), 0u}; + runtime.userdata = &records; + records.fail_address = records.base + record_offset; + Check(MaterializeResources(bounded, runtime, snapshot, specialization) && + snapshot.images.size() == 2u && records.descriptor_reads == 0u && + std::ranges::all_of(snapshot.images[0].dwords, [](uint32_t word) { return word == 0u; }), + "zero runtime count probed an unreachable descriptor row"); + records.fail_address = UINT64_MAX; + records.words[1] = 2u; + records.descriptor_reads = 0u; + Check(MaterializeResources(bounded, runtime, snapshot, specialization) && + snapshot.images.size() == 3u && records.descriptor_reads == 2u && + snapshot.images[0].dwords[0] == 0x500u && snapshot.images[2].dwords[0] == 0x501u, + "bounded interleaved table did not read precisely its two descriptor rows"); + const auto captured = [&](uint64_t address, uint64_t size) { + return std::ranges::any_of(snapshot.specialization_reads, [&](const auto &read) { + return read.first <= address && address - read.first <= read.second && + size <= read.second - (address - read.first); + }); + }; + Check(captured(records.base + 4u, 4u) && + captured(records.base + record_offset, 32u) && + captured(records.base + record_offset + record_stride, 32u), + "bounded table omitted count or descriptor bytes from writable-alias validation"); + records.fail_address = records.base + record_offset + record_stride + 28u; + Check(!MaterializeResources(bounded, runtime, snapshot, specialization), + "bounded interleaved table accepted an unreadable final descriptor word"); + records.fail_address = UINT64_MAX; + records.words[1] = UINT32_MAX; + records.descriptor_reads = 0u; + Check(!MaterializeResources(bounded, runtime, snapshot, specialization) && + records.descriptor_reads == 0u, + "unrepresentable table span was read before validating its count"); } - user_data = {static_cast(records.base), 0u}; - runtime.userdata = &records; - records.fail_address = records.base + record_offset; - Check(MaterializeResources(bounded, runtime, snapshot, specialization) && - snapshot.images.size() == 2u && records.descriptor_reads == 0u && - std::ranges::all_of(snapshot.images[0].dwords, [](uint32_t word) { return word == 0u; }), - "zero runtime count probed an unreachable descriptor row"); - records.fail_address = UINT64_MAX; - records.words[1] = 2u; - records.descriptor_reads = 0u; - Check(MaterializeResources(bounded, runtime, snapshot, specialization) && - snapshot.images.size() == 3u && records.descriptor_reads == 2u && - snapshot.images[0].dwords[0] == 0x500u && snapshot.images[2].dwords[0] == 0x501u, - "bounded interleaved table did not read precisely its two descriptor rows"); - const auto captured = [&](uint64_t address, uint64_t size) { - return std::ranges::any_of(snapshot.specialization_reads, [&](const auto &read) { - return read.first <= address && address - read.first <= read.second && - size <= read.second - (address - read.first); - }); - }; - Check(captured(records.base + 4u, 4u) && - captured(records.base + record_offset, 32u) && - captured(records.base + record_offset + record_stride, 32u), - "bounded table omitted count or descriptor bytes from writable-alias validation"); - records.fail_address = records.base + record_offset + record_stride + 28u; - Check(!MaterializeResources(bounded, runtime, snapshot, specialization), - "bounded interleaved table accepted an unreadable final descriptor word"); - records.fail_address = UINT64_MAX; - records.words[1] = UINT32_MAX; - records.descriptor_reads = 0u; - Check(!MaterializeResources(bounded, runtime, snapshot, specialization) && - records.descriptor_reads == 0u, - "unrepresentable table span was read before validating its count"); } void TestBoundedComputeImageLoop() { diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 056e028e..2650df02 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -36412,13 +36412,17 @@ void CheckCountedRowImageSelector(VulkanHarness &vulkan) { void CheckGuardedRecordImageTable(VulkanHarness &vulkan) { using O = ShaderOpcode; - constexpr u32 table = 256u, stride = 272u, offset = 3696u; + constexpr u32 table = 256u; + for (const bool masked : {false, true}) for (const u32 wave : {32u, 64u}) for (const u32 count : {0u, 2u, 3u}) { + const u32 stride = masked ? 784u : 272u, offset = masked ? 672u : 3696u; + const u32 lanes = masked ? 2u : 1u; TestCase test; - test.name = wave == 32u ? "GuardedRecordImageTable32" : "GuardedRecordImageTable64"; + test.name = masked ? (wave == 32u ? "ExecGuardedRecordTable32" : "ExecGuardedRecordTable64") + : (wave == 32u ? "GuardedRecordImageTable32" : "GuardedRecordImageTable64"); test.initial.resize((table + offset + stride * count) / 4u, 0xdeadbeefu); - test.initial[table / 4u + 1u] = count; + test.initial[table / 4u + (masked ? 0u : 1u)] = count; for (u32 row = 0; row < count; ++row) for (const u32 member : {0u, 1u}) { auto image = MakeSampledTextureData(Prospero::BufferFormat::k32Float); @@ -36431,20 +36435,25 @@ void CheckGuardedRecordImageTable(VulkanHarness &vulkan) { test.user_data[0] = table; auto destination = MakeSampledTextureData(Prospero::BufferFormat::k32Float); destination[0] = 0x10000u; - destination[1] |= 3u << 30u; + destination[1] |= (4u * lanes - 1u) << 30u; + destination[2] |= (4u * lanes - 1u) >> 2u; destination[3] = DstSel(4, 0, 0, 1) | (static_cast(Prospero::ImageType::kColor2D) << 28u); std::copy_n(destination.begin(), 8, test.user_data.begin() + 32); test.has_user_data = test.has_compute_info = true; test.compute_info.wave_size = wave; - test.compute_info.threads_num[0] = test.compute_info.threads_num[1] = - test.compute_info.threads_num[2] = 1u; + test.compute_info.threads_num[0] = lanes; + test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1u; + test.compute_info.thread_ids_num = 1u; test.compute_info.group_id[0] = true; test.compute_info.workgroup_register = 44u; test.dispatch_x = 4u; auto &code = test.code; code.insert(code.end(), {EncodeSop2(0x20, 20, InlineU32(5), 44), - EncodeSmem0(0x00, 21), EncodeSmem1(4, 125)}); + EncodeSmem0(0x00, 21), EncodeSmem1(masked ? 0u : 4u, 125)}); + code.push_back(EncodeSop1(0x04, 30, 126)); // Preserve EXEC for the final write. + code.push_back(EncodeVop1(0x01, 13, Vgpr(0))); // Odd lane is already done. + AppendVMovLiteral(&code, 10, 0xdeadbeefu); AppendVMovLiteral(&code, 0, 0u); AppendVMovLiteral(&code, 20, 0x3f000000u); AppendVMovLiteral(&code, 21, 0x3f000000u); @@ -36452,7 +36461,13 @@ void CheckGuardedRecordImageTable(VulkanHarness &vulkan) { code.push_back(EncodeSop1(0x13, 22, 20)); // Find the next set bit in the GPU worklist. code.push_back(EncodeSopc(0x07, 22, 193)); const auto empty_exit = code.size(); code.push_back(0u); + if (masked) code.insert(code.end(), {EncodeSop2(0x0b, 28, 126, InlineU32(0)), + EncodeSop1(0x04, 126, 28)}); code.push_back(EncodeSopc(0x09, 22, 21)); + if (masked) code.insert(code.end(), { + EncodeVop2(0x1c, 10, 253, 13), // SCC encoded as 0/1, OR per-lane done. + EncodeVopc(0xc5, InlineU32(0), 10), + EncodeSop2(0x15, 126, 28, 106)}); const auto range_exit = code.size(); code.push_back(0u); code.insert(code.end(), {EncodeSop2(0x26, 23, 22, 255), stride, EncodeSop2(0x02, 23, 23, 255), offset, @@ -36472,8 +36487,10 @@ void CheckGuardedRecordImageTable(VulkanHarness &vulkan) { EncodeSop2(0x14, 20, 20, 24)}); code.push_back(EncodeSopp(0x02, loop - code.size() - 1u)); code[empty_exit] = EncodeSopp(0x04, code.size() - empty_exit - 1u); - code[range_exit] = EncodeSopp(0x05, code.size() - range_exit - 1u); - code.push_back(EncodeVop1(0x01, 30, 44)); + code[range_exit] = EncodeSopp(masked ? 0x08 : 0x05, code.size() - range_exit - 1u); + code.push_back(EncodeSop1(0x04, 126, 30)); + code.push_back(EncodeSop2(0x26, 25, 44, InlineU32(lanes))); + code.push_back(EncodeVop2(0x25, 30, 25, 13)); AppendVMovLiteral(&code, 31, 0u); code.insert(code.end(), {EncodeMimg0(0x08, 1), EncodeMimg1(0, 30, 8, 0)}); AppendEnd(&code); @@ -36482,6 +36499,9 @@ void CheckGuardedRecordImageTable(VulkanHarness &vulkan) { O::S_MUL_I32, O::S_ADD_I32, O::S_BITCMP1_B32, O::S_LOAD_DWORDX8, O::IMAGE_SAMPLE, O::V_ADD_F32, O::S_LSHL_B32, O::S_ANDN2_B32, O::S_BRANCH, O::IMAGE_STORE, O::S_ENDPGM}; + if (masked) test.opcodes.insert(test.opcodes.end(), {O::S_CSELECT_B64, + O::V_OR_B32, O::V_CMP_NE_U32, O::S_ANDN2_B64, O::S_CBRANCH_EXECZ}); + test.opcodes.insert(test.opcodes.end(), {O::S_MOV_B64, O::V_ADD_NC_U32}); auto compiled = CompileCase(test, vulkan.SubgroupSize()); const auto sampled_count = std::ranges::count_if(compiled.program.info.images, [](const auto &image) { @@ -36499,15 +36519,18 @@ void CheckGuardedRecordImageTable(VulkanHarness &vulkan) { vk::ImageUsageFlagBits::eSampled, {std::bit_cast(value)}, 1, vk::ImageLayout::eShaderReadOnlyOptimal); } - auto storage = vulkan.CreateImage2D(test.name, 4, 1, vk::Format::eR32Sfloat, - vk::ImageUsageFlagBits::eStorage, std::vector(4, 0xdeadbeefu), 1, + auto storage = vulkan.CreateImage2D(test.name, 4u * lanes, 1, vk::Format::eR32Sfloat, + vk::ImageUsageFlagBits::eStorage, std::vector(4u * lanes, 0xdeadbeefu), 1, vk::ImageLayout::eGeneral); auto buffer = vulkan.CreateStorageBuffer(test.name, test.initial, test.initial.size()); auto sampler = vulkan.CreateSampler(test.name, 1); vulkan.Dispatch(test, compiled, buffer, nullptr, nullptr, &storage, nullptr, sampler, images); - const std::vector expected = count == 0 ? std::vector(4, 0u) : - std::vector{std::bit_cast(count == 3 ? 44.0f : 11.0f), - std::bit_cast(20.0f), std::bit_cast(11.0f), 0u}; + std::vector expected(4u * lanes, 0u); + if (count != 0u) { + expected[0] = std::bit_cast(count == 3 ? 44.0f : 11.0f); + expected[lanes] = std::bit_cast(20.0f); + expected[2u * lanes] = std::bit_cast(11.0f); + } CompareWords(test, "GPU worklist, guarded count and image writes", expected, vulkan.ReadImage(test.name, &storage)); vulkan.DestroyBuffer(&buffer);