From b9a8aafee21cd7803e0082be985133d3a7a4a07d Mon Sep 17 00:00:00 2001 From: ParantezTech <12572227+par274@users.noreply.github.com> Date: Thu, 1 Oct 2026 17:18:38 +0300 Subject: [PATCH] [ShaderCompiler] Track wave-indexed image descriptor waterfalls --- .../Resources/ResourceTracker.DenseImages.cs | 147 ++++++++++++++++-- .../Resources/ScalarValueGraph.Builder.cs | 15 +- .../Resources/ShaderResourcePlan.cs | 3 +- .../Resources/DirectImageTableTests.cs | 126 +++++++++++---- 4 files changed, 243 insertions(+), 48 deletions(-) diff --git a/src/SharpEmu.ShaderCompiler/Resources/ResourceTracker.DenseImages.cs b/src/SharpEmu.ShaderCompiler/Resources/ResourceTracker.DenseImages.cs index fab01fe..2545faa 100644 --- a/src/SharpEmu.ShaderCompiler/Resources/ResourceTracker.DenseImages.cs +++ b/src/SharpEmu.ShaderCompiler/Resources/ResourceTracker.DenseImages.cs @@ -1,6 +1,8 @@ // Copyright (C) 2026 SharpEmu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +using SharpEmu.ShaderCompiler.Ir; + namespace SharpEmu.ShaderCompiler.Resources; public sealed partial class ResourceTracker @@ -195,9 +197,11 @@ public sealed partial class ResourceTracker var instructions = _graph.Program.Instructions; var firstLaneIndex = FindInstructionIndex(instructions, (uint)key.Payload); - if (firstLaneIndex < 0 || instructions[firstLaneIndex] is not { Opcode: "VReadfirstlaneB32", Sources.Count: 1, Destinations.Count: 1 } firstLane || + if (firstLaneIndex < 0 || instructions[firstLaneIndex] is not { Destinations.Count: 1 } firstLane || + firstLane.Sources.Count == 0 || firstLane.Sources[0] is not { Kind: Gen5OperandKind.VectorRegister } vectorSource || - firstLane.Destinations[0] is not { Kind: Gen5OperandKind.ScalarRegister } scalarDestination) + firstLane.Destinations[0] is not { Kind: Gen5OperandKind.ScalarRegister } || + !(firstLane is { Opcode: "VReadfirstlaneB32", Sources.Count: 1 } || ReadsActiveLane(instructions, firstLaneIndex))) { return null; } @@ -222,12 +226,8 @@ public sealed partial class ResourceTracker var descriptorLoadIndex = FindInstructionIndex(instructions, _plan.Memory[descriptorMemoryIndex].Pc); if (descriptorLoadIndex < 0 || instructions[descriptorLoadIndex] is not { - Control: Gen5ScalarMemoryControl - { - DestinationCount: >= 8, - DynamicOffsetRegister: { } dynamicOffsetRegister, - }, - } descriptorLoad || dynamicOffsetRegister != scalarDestination.Value || + Control: Gen5ScalarMemoryControl { DynamicOffsetRegister: not null }, + } descriptorLoad || descriptorLoad.Sources.Count == 0 || descriptorLoad.Sources[0] != Gen5Operand.Scalar(global.ScalarAddress)) { return null; @@ -244,8 +244,7 @@ public sealed partial class ResourceTracker if (TryGetSelfAddedConstant(instructions[addressDefinitionIndex], address, out _) && TryGetWaveIndexedStride(instructions, addressDefinitionIndex, address, heapAddress, out var maskRegister, out var bitRegister, out var maskOffset, out var indexStride) && - indexDataOffset != 0 && indexStride != 0 && instructions.Any(instruction => instruction.Opcode == "SBitset0B32" && - instruction.Destinations.Contains(maskRegister) && instruction.Sources.Contains(bitRegister))) + indexDataOffset != 0 && indexStride != 0 && ClearsMaskBit(instructions, 0, instructions.Count, maskRegister, bitRegister)) { return new(maskOffset, indexDataOffset, indexStride); } @@ -283,9 +282,7 @@ public sealed partial class ResourceTracker { if (instructions[scanIndex] is not { Opcode: "SFF1I32B32", Sources.Count: 1, Destinations.Count: 1 } scan || scan.Sources[0] != maskRegister || scan.Destinations[0] is not { Kind: Gen5OperandKind.ScalarRegister } bitRegister || - !instructions.Skip(scanIndex + 1).Take(before - scanIndex - 1) - .Any(instruction => instruction.Opcode == "SBitset0B32" && instruction.Destinations.Contains(maskRegister) && - instruction.Sources.Contains(bitRegister))) + !ClearsMaskBit(instructions, scanIndex + 1, before, maskRegister, bitRegister)) continue; for (var multiplyIndex = scanIndex + 1; multiplyIndex < before; multiplyIndex++) @@ -358,6 +355,130 @@ public sealed partial class ResourceTracker return true; } + private static bool ClearsMaskBit( + IReadOnlyList instructions, + int start, + int end, + Gen5Operand mask, + Gen5Operand bit) + { + end = Math.Min(end, instructions.Count); + for (var index = Math.Max(0, start); index < end; index++) + { + var instruction = instructions[index]; + if (instruction.Opcode == "SBitset0B32" && instruction.Destinations.Contains(mask) && instruction.Sources.Contains(bit)) + return true; + + if (instruction is not { Opcode: "SLshlB32", Sources.Count: 2, Destinations.Count: 1 } shift || + !TryGetConstant(shift.Sources[0], out var one) || one != 1 || shift.Sources[1] != bit || + shift.Destinations[0] is not { Kind: Gen5OperandKind.ScalarRegister } single) + continue; + + for (var clearIndex = index + 1; clearIndex < end; clearIndex++) + { + var clear = instructions[clearIndex]; + if (clear.Destinations.Contains(mask) && clear.Sources.Count == 2 && + ((clear.Opcode == "SXorB32" && clear.Sources.Contains(mask) && clear.Sources.Contains(single)) || + (clear.Opcode == "SAndn2B32" && clear.Sources[0] == mask && clear.Sources[1] == single))) + return true; + + if (clear.Destinations.Contains(single) || clear.Destinations.Contains(mask) || clear.Destinations.Contains(bit) || + (single.Value is 106 or 107 && clear.Opcode.StartsWith('V'))) + break; + } + } + + return false; + } + + private static bool ReadsActiveLane(IReadOnlyList instructions, int readLaneIndex) + { + var exec = Gen5Operand.Scalar(126); + var readLane = instructions[readLaneIndex]; + if (readLane is not { Opcode: "VReadlaneB32" } || readLane.Sources.Count < 2 || + readLane.Sources[1] is not { Kind: Gen5OperandKind.ScalarRegister } lane) + return false; + + var scanIndex = FindLastDefinition(instructions, readLaneIndex, lane); + if (scanIndex < 0 || instructions[scanIndex] is not { Opcode: "SFF1I32B64", Sources.Count: 1 } scan || + scan.Sources[0] is not { Kind: Gen5OperandKind.ScalarRegister } candidates) + return false; + + var copyIndex = FindLastDefinition(instructions, scanIndex, candidates); + if (copyIndex < 0 || instructions[copyIndex] is not { Opcode: "SMovB64", Sources.Count: 1 } copy || copy.Sources[0] != exec) + return false; + + for (var index = copyIndex + 1; index < readLaneIndex; index++) + { + if (WritesScalarPair(instructions[index], exec) || WritesScalarPair(instructions[index], candidates) || + Gen5IrBranchResolver.Instance.TryGetBranchTarget(instructions[index], out _)) + return false; + } + + var backIndex = -1; + for (var index = readLaneIndex + 1; index < instructions.Count; index++) + { + if (!Gen5IrBranchResolver.Instance.TryGetBranchTarget(instructions[index], out var target)) + continue; + if (target > instructions[copyIndex].Pc && target <= instructions[scanIndex].Pc) + { + backIndex = index; + break; + } + } + + if (backIndex < 0) + return false; + + var restoreIndex = -1; + for (var index = backIndex - 1; index > readLaneIndex; index--) + { + if (WritesScalarPair(instructions[index], exec)) + { + restoreIndex = index; + break; + } + } + + if (restoreIndex < 0 || instructions[restoreIndex] is not { Opcode: "SMovB64", Sources.Count: 1 } restore || + restore.Sources[0] is not { Kind: Gen5OperandKind.ScalarRegister } saved) + return false; + + var saveIndex = FindLastDefinition(instructions, restoreIndex, saved); + if (saveIndex <= readLaneIndex || instructions[saveIndex].Opcode != "SAndSaveexecB64") + return false; + + for (var index = readLaneIndex + 1; index < backIndex; index++) + { + var instruction = instructions[index]; + if (index > saveIndex && index < restoreIndex && WritesScalarPair(instruction, saved)) + return false; + if (WritesScalarPair(instruction, candidates) && + (instruction is not { Opcode: "SAndn2B64", Sources.Count: 2 } || instruction.Sources[0] != candidates)) + return false; + if (Gen5IrBranchResolver.Instance.TryGetBranchTarget(instruction, out var target) && + (target <= instruction.Pc || target > instructions[restoreIndex].Pc)) + return false; + } + + return true; + } + + private static bool WritesScalarPair(Gen5ShaderInstruction instruction, Gen5Operand register) + { + if (register.Value is 106 or 107 && instruction.Opcode.StartsWith('V')) + return true; + if (instruction.Control is Gen5Vop3Control { ScalarDestination: { } scalarDestination } && + scalarDestination + 1 >= register.Value && scalarDestination <= register.Value + 1) + return true; + if (register.Value == 126 && (instruction.Opcode.Contains("Saveexec", StringComparison.Ordinal) || + instruction.Opcode.StartsWith("VCmpx", StringComparison.Ordinal))) + return true; + var width = instruction.Opcode.Contains("64", StringComparison.Ordinal) ? 2u : 1u; + return instruction.Destinations.Any(destination => destination.Kind == Gen5OperandKind.ScalarRegister && + destination.Value + width > register.Value && destination.Value <= register.Value + 1); + } + private static int FindLastDefinition(IReadOnlyList instructions, int before, Gen5Operand destination) { for (var index = before - 1; index >= 0; index--) diff --git a/src/SharpEmu.ShaderCompiler/Resources/ScalarValueGraph.Builder.cs b/src/SharpEmu.ShaderCompiler/Resources/ScalarValueGraph.Builder.cs index e5d0c4c..ff1490c 100644 --- a/src/SharpEmu.ShaderCompiler/Resources/ScalarValueGraph.Builder.cs +++ b/src/SharpEmu.ShaderCompiler/Resources/ScalarValueGraph.Builder.cs @@ -1558,15 +1558,22 @@ public sealed partial class ScalarValueGraph var lane = instruction.Sources.Count > 1 ? Read(instruction.Sources[1], state) : _graph.Undefined(ScalarValueType.U32); if (instruction.Sources.Count < 2 || - instruction.Sources[0] is not { Kind: Gen5OperandKind.VectorRegister } source || - !lane.IsConstant || - !state.Lanes.TryGetValue((source.Value, lane.ConstantU32 & 63), out var value)) + instruction.Sources[0] is not { Kind: Gen5OperandKind.VectorRegister } source) { state.WriteScalar(destination.Value, _graph.Undefined(ScalarValueType.U32)); return; } - state.WriteScalar(destination.Value, value); + if (lane.IsConstant && state.Lanes.TryGetValue((source.Value, lane.ConstantU32 & 63), out var value)) + { + state.WriteScalar(destination.Value, value); + return; + } + + var sourceValue = state.ReadVector(source.Value); + state.WriteScalar(destination.Value, !lane.IsConstant && sourceValue.IsUndefined + ? _graph.FirstLane(sourceValue, state.Exec, instruction.Pc) + : _graph.Undefined(ScalarValueType.U32)); } // ---- memory instructions ---- diff --git a/src/SharpEmu.ShaderCompiler/Resources/ShaderResourcePlan.cs b/src/SharpEmu.ShaderCompiler/Resources/ShaderResourcePlan.cs index 8089357..f13d2d3 100644 --- a/src/SharpEmu.ShaderCompiler/Resources/ShaderResourcePlan.cs +++ b/src/SharpEmu.ShaderCompiler/Resources/ShaderResourcePlan.cs @@ -181,7 +181,8 @@ public sealed class ShaderResourcePlan var value = TableReads[slot].Value; if (value.Kind == ScalarValueKind.ScalarAddressWord && value.Operands.Length != 0 && value.Operands[0].Kind == ScalarValueKind.AddressHandle && - value.Operands[0].Operands.SequenceEqual(heap.Dwords)) + value.Operands[0].Operands.Length == heap.Dwords.Length && + value.Operands[0].Operands.Zip(heap.Dwords).All(pair => ReferenceEquals(pair.First, pair.Second) || Graph.Equivalent(pair.First, pair.Second))) { slots[slot] = 1; } diff --git a/tests/SharpEmu.ShaderCompiler.Tests/Resources/DirectImageTableTests.cs b/tests/SharpEmu.ShaderCompiler.Tests/Resources/DirectImageTableTests.cs index c7ef505..0be40ae 100644 --- a/tests/SharpEmu.ShaderCompiler.Tests/Resources/DirectImageTableTests.cs +++ b/tests/SharpEmu.ShaderCompiler.Tests/Resources/DirectImageTableTests.cs @@ -30,6 +30,41 @@ public sealed class DirectImageTableTests EndProgram(72)); } + internal static Gen5ShaderProgram CreateWaveIndexedReadLaneProgram(bool selfAddressed, bool restoreExec = true) + { + var instructions = new List + { + ScalarLoad(0, 0, 16, immediateOffset: 0x80), + Sop1(8, "SFF1I32B32", 18, Gen5Operand.Scalar(16)), + Sop2(12, "SMulI32", 19, Gen5Operand.Scalar(18), new Gen5Operand(Gen5OperandKind.LiteralConstant, 0x90)), + MoveVectorFromScalar(16, 34, 18), + Vop2(20, "VLshlrevB32", 15, Operand(4), Gen5Operand.Vector(34)), + Vop3(24, "VLshlAddU32", selfAddressed ? 15u : 16u, Gen5Operand.Vector(15), Operand(3), Gen5Operand.Vector(15)), + Sop2(32, "SLshlB32", 20, Operand(1), Gen5Operand.Scalar(18)), + Sop2(36, "SXorB32", 16, Gen5Operand.Scalar(20), Gen5Operand.Scalar(16)), + Vop2(40, "VAddI32", 15, new Gen5Operand(Gen5OperandKind.LiteralConstant, 0x40), Gen5Operand.Vector(selfAddressed ? 15u : 16u)), + GlobalMemory(44, "GlobalLoadDword", 0, 15, 22, 0), + Sop1(52, "SMovB64", 12, Gen5Operand.Scalar(126)), + Sop1(56, "SFF1I32B64", 24, Gen5Operand.Scalar(12)), + new(60, Gen5ShaderEncoding.Vop3, "VReadlaneB32", [0u, 0u], + [Gen5Operand.Vector(22), Gen5Operand.Scalar(24), Gen5Operand.Scalar(24)], [Gen5Operand.Scalar(106)], null), + new(68, Gen5ShaderEncoding.Vop3, "VCmpEqU32", [0u, 0u], + [Gen5Operand.Scalar(106), Gen5Operand.Vector(22)], [Gen5Operand.Scalar(14)], new Gen5Vop3Control(0, 0, 0, false, 0, 14)), + Sop1(76, "SAndSaveexecB64", 28, Gen5Operand.Scalar(14)), + Branch(80, "SCbranchExecz", 8), + Sop2(84, "SLshlB32", 106, Gen5Operand.Scalar(106), Operand(5)), + Sop2(88, "SAddI32", 107, Gen5Operand.Scalar(106), Operand(16)), + ScalarLoad(92, 0, 4, 4, immediateOffset: 0x100, dynamicOffsetRegister: 106), + ScalarLoad(100, 0, 8, 4, immediateOffset: 0x100, dynamicOffsetRegister: 107), + Image(108, "ImageLoad", 4, dmask: 1, vectorAddress: 1), + Sop2(116, "SAndn2B64", 12, Gen5Operand.Scalar(12), Gen5Operand.Scalar(14)), + restoreExec ? Sop1(120, "SMovB64", 126, Gen5Operand.Scalar(28)) : Nop(120), + Branch(124, "SCbranchScc1", -18), + EndProgram(128), + }; + return Program([.. instructions]); + } + public static Gen5ShaderProgram CreateProgram(uint mask = 1, bool split = true, bool bitScan = true) { var instructions = new List @@ -107,41 +142,72 @@ public sealed class DirectImageTableTests Assert.Equal(0u, selector.KeyBound); Assert.Equal(new WaveIndexedImageSelector(0x80, 0x40, 0x90), selector.WaveIndexed); - bool Read(ulong address, out uint word) - { - word = 0; - if (address == 0x1000 + 0x80) - { - word = (1u << 1) | (1u << 4); - return true; - } + var snapshot = new ResourceSnapshot(); + var specialization = new ResourceSpecialization(); + Assert.True(ResourceMaterializer.Materialize(plan, Inputs([0x1000, 0], readCleanMemory: ReadWaveIndexedMemory), ref snapshot, ref specialization)); + Assert.Equal(2, snapshot.Images.Length); + } - if (address == 0x1000 + 0x40 + 0x90 || address == 0x1000 + 0x40 + 4 * 0x90) - { - word = address == 0x1000 + 0x40 + 0x90 ? 2u : 5u; - return true; - } - - if (address < 0x1000 + 0x100 || address >= 0x1000 + 0x100 + 6 * 32) - return false; - var relative = address - 0x1000 - 0x100; - var record = relative / 32; - if (record is not (2 or 5)) - return false; - word = (relative % 32 / 4) switch - { - 0 => (record & 1) == 0 ? 0x2000u : 0x1000u, - 1 => 20u << 20, - 3 => 0xFACu | (9u << 28), - _ => 0, - }; - return true; - } + [Theory] + [InlineData(false)] + [InlineData(true)] + public void ReadLaneWaterfallSelectsWaveIndexedDescriptors(bool selfAddressed) + { + var program = CreateWaveIndexedReadLaneProgram(selfAddressed); + var plan = ShaderResourcePlan.Extract(program, ShaderStage.Compute, Hash, 0, 2); + var selector = plan.DescriptorSources[(int)plan.Info.Images[0].Source].IndirectImage!; + Assert.True(selector.Dense, "dense"); + Assert.Equal(0x100u, selector.TableOffset); + Assert.Equal(new WaveIndexedImageSelector(0x80, 0x40, 0x90), selector.WaveIndexed); + Assert.True(Assert.Single(plan.IndirectImages).KeyIsAddressOffset, "address-offset key"); var snapshot = new ResourceSnapshot(); var specialization = new ResourceSpecialization(); - Assert.True(ResourceMaterializer.Materialize(plan, Inputs([0x1000, 0], readCleanMemory: Read), ref snapshot, ref specialization)); + Assert.True(ResourceMaterializer.Materialize(plan, Inputs([0x1000, 0], readCleanMemory: ReadWaveIndexedMemory), ref snapshot, ref specialization, out var failure), $"materialize {failure}"); Assert.Equal(2, snapshot.Images.Length); + var resources = ResourceMaterializer.ApplyTo(plan, specialization); + var layout = BindingLayout.Allocate(resources.Info, BindingLayout.CollectUserDataRegisters(program, 0, 2), + false, ShaderCompileRequest.RequiresFlattenedTable(plan, resources), false); + var request = new ShaderCompileRequest(plan, resources, layout) { LocalSizeX = 64, ThreadCountX = 64 }; + Assert.True(Gen5SpirvTranslator.TryCompileProgram(request, out _, out var error), error); + } + + [Fact] + public void ReadLaneWithoutRestoredExecutionIsNotWaveIndexed() + { + Assert.Throws(() => + ShaderResourcePlan.Extract(CreateWaveIndexedReadLaneProgram(selfAddressed: true, restoreExec: false), ShaderStage.Compute, Hash, 0, 2)); + } + + private static bool ReadWaveIndexedMemory(ulong address, out uint word) + { + word = 0; + if (address == 0x1000 + 0x80) + { + word = (1u << 1) | (1u << 4); + return true; + } + + if (address == 0x1000 + 0x40 + 0x90 || address == 0x1000 + 0x40 + 4 * 0x90) + { + word = address == 0x1000 + 0x40 + 0x90 ? 2u : 5u; + return true; + } + + if (address < 0x1000 + 0x100 || address >= 0x1000 + 0x100 + 6 * 32) + return false; + var relative = address - 0x1000 - 0x100; + var record = relative / 32; + if (record is not (2 or 5)) + return false; + word = (relative % 32 / 4) switch + { + 0 => (record & 1) == 0 ? 0x2000u : 0x1000u, + 1 => 20u << 20, + 3 => 0xFACu | (9u << 28), + _ => 0, + }; + return true; } public static (ResourceSnapshot Snapshot, ShaderCompileRequest Request) PrepareMixedDimensions(uint mask, bool arrayFirst)