From 663cf207f890ecd2a192f6202454047307213636 Mon Sep 17 00:00:00 2001 From: ParantezTech <12572227+par274@users.noreply.github.com> Date: Thu, 1 Oct 2026 17:19:54 +0300 Subject: [PATCH] [SpirvTranslator] Synchronize wave64 half masks through LDS exchange --- .../Gen5SpirvTranslator.Alu.cs | 114 +++--- .../Gen5SpirvTranslator.Resources.cs | 5 +- .../Gen5SpirvTranslator.cs | 332 +++++++++++------- .../Gpu/Vulkan/Wave64HalfMaskDeviceTests.cs | 269 ++++++++++++++ 4 files changed, 524 insertions(+), 196 deletions(-) create mode 100644 tests/SharpEmu.Libs.Tests/Gpu/Vulkan/Wave64HalfMaskDeviceTests.cs diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Alu.cs b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Alu.cs index 903b1bc..763d49b 100644 --- a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Alu.cs +++ b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Alu.cs @@ -3368,7 +3368,9 @@ public static partial class Gen5SpirvTranslator var left = GetRawSource64(instruction, 0); if (instruction.Opcode.EndsWith("SaveexecB64", StringComparison.Ordinal)) { - var oldExec = BooleanToWaveMask(Load(_boolType, _exec)); + var oldExec = _emulateWave64 && _subgroupInvocationIdInput != 0 + ? LoadS64(126) + : BooleanToWaveMask(Load(_boolType, _exec)); var notLeft = _module.AddInstruction(SpirvOp.Not, _ulongType, left); var newExec = instruction.Opcode switch { @@ -3741,14 +3743,14 @@ public static partial class Gen5SpirvTranslator _module.AddInstruction( SpirvOp.Select, _uintType, - LogicalNot(SubgroupAny(Load(_boolType, _vcc))), + LogicalNot(WaveMaskAny(106, _vcc)), UInt(1), UInt(0)), Gen5OperandKind.EncodedConstant when operand.Value == 252 => _module.AddInstruction( SpirvOp.Select, _uintType, - LogicalNot(SubgroupAny(Load(_boolType, _exec))), + LogicalNot(WaveMaskAny(126, _exec)), UInt(1), UInt(0)), Gen5OperandKind.EncodedConstant when operand.Value == 253 => @@ -4825,76 +4827,48 @@ public static partial class Gen5SpirvTranslator // An emulated wave64 spans two host subgroups, so a subgroup broadcast cannot reach // the other half. The selected guest lane publishes through workgroup scratch, like - // BroadcastFirstWave64Active; the trailing barrier keeps the next use from racing. - private uint BroadcastWave64Lane(uint value, uint lane) - { - EmitConditional( - _module.AddInstruction(SpirvOp.IEqual, _boolType, GuestWaveLane(), lane), - () => Store(WaveBroadcastScratchPointer(), value)); - EmitWave64Barrier(); - var result = Load(_uintType, WaveBroadcastScratchPointer()); - EmitWave64Barrier(); - return result; - } + // BroadcastFirstWave64Active. + private uint BroadcastWave64Lane(uint value, uint lane) => + ExchangeWave64Value(_module.AddInstruction(SpirvOp.IEqual, _boolType, GuestWaveLane(), lane), value); private uint BroadcastFirstWave64Active(uint value) { var lane = GuestWaveLane(); - EmitConditional( - _module.AddInstruction( - SpirvOp.IEqual, - _boolType, - lane, - UInt(0)), - () => Store(WaveBroadcastScratchPointer(), UInt(0))); - EmitWave64Barrier(); - - var activeMask = BooleanToWaveMask(Load(_boolType, _exec)); - var lowMask = _module.AddInstruction( - SpirvOp.UConvert, - _uintType, - activeMask); - var highMask = _module.AddInstruction( - SpirvOp.UConvert, - _uintType, - ShiftRightLogical64( - activeMask, - _module.Constant64(_ulongType, 32))); - var hasLow = IsNotZero(lowMask); - var hasHigh = IsNotZero(highMask); - var firstLow = Ext(73, _uintType, lowMask); - var firstHigh = IAdd(UInt(32), Ext(73, _uintType, highMask)); - var firstLane = _module.AddInstruction( + var upperHalf = ShiftRightLogical(lane, UInt(5)); + var activeInHalf = OwnHalfBallot(Load(_boolType, _exec)); + var halfHasActive = IsNotZero(activeInHalf); + var firstInHalf = _module.AddInstruction( SpirvOp.Select, _uintType, - hasLow, - firstLow, - _module.AddInstruction( - SpirvOp.Select, - _uintType, - hasHigh, - firstHigh, - UInt(0))); - var isFirst = _module.AddInstruction( - SpirvOp.IEqual, - _boolType, - lane, - firstLane); - EmitConditional( - _module.AddInstruction( - SpirvOp.LogicalAnd, - _boolType, - isFirst, - _module.AddInstruction( - SpirvOp.LogicalOr, - _boolType, - hasLow, - hasHigh)), - () => Store(WaveBroadcastScratchPointer(), value)); + halfHasActive, + Ext(73, _uintType, activeInHalf), + UInt(0)); + var halfBase = BitwiseAnd(Load(_uintType, _subgroupInvocationIdInput), UInt(~31u)); + var halfValue = _module.AddInstruction( + SpirvOp.GroupNonUniformBroadcast, + _uintType, + UInt(3), + value, + IAdd(halfBase, firstInHalf)); + var exchange = BeginWave64Exchange(); + EmitConditional(IsHalfWaveLeader(lane), () => + { + Store(Wave64ExchangePointer(exchange, upperHalf), halfValue); + Store( + Wave64ExchangePointer(exchange, IAdd(UInt(2), upperHalf)), + _module.AddInstruction(SpirvOp.Select, _uintType, halfHasActive, UInt(1), UInt(0))); + }); EmitWave64Barrier(); - var result = Load(_uintType, WaveBroadcastScratchPointer()); - EmitWave64Barrier(); - return result; + var lowerValue = Load(_uintType, Wave64ExchangePointer(exchange, UInt(0))); + var upperValue = Load(_uintType, Wave64ExchangePointer(exchange, UInt(1))); + var lowerActive = IsNotZero(Load(_uintType, Wave64ExchangePointer(exchange, UInt(2)))); + var upperActive = IsNotZero(Load(_uintType, Wave64ExchangePointer(exchange, UInt(3)))); + return _module.AddInstruction( + SpirvOp.Select, + _uintType, + _module.AddInstruction(SpirvOp.LogicalAnd, _boolType, LogicalNot(lowerActive), upperActive), + upperValue, + lowerValue); } private void StoreCarryOut( @@ -4935,13 +4909,7 @@ public static partial class Gen5SpirvTranslator { // The selected guest lane can belong to another host subgroup. // Read it even when the guest execution mask disables that lane. - var isSelectedLane = _module.AddInstruction( - SpirvOp.IEqual, _boolType, GuestWaveLane(), selectedLane); - EmitConditional(isSelectedLane, () => Store(WaveBroadcastScratchPointer(), sourceValue)); - EmitWave64Barrier(); - var broadcast = Load(_uintType, WaveBroadcastScratchPointer()); - EmitWave64Barrier(); - StoreS(destination, broadcast); + StoreS(destination, BroadcastWave64Lane(sourceValue, selectedLane)); } else if (_subgroupInvocationIdInput != 0) { diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Resources.cs b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Resources.cs index 34df42b..43bae54 100644 --- a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Resources.cs +++ b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Resources.cs @@ -1904,10 +1904,7 @@ public static partial class Gen5SpirvTranslator uint broadcast; if (_emulateWave64) { - EmitConditional(isFirstActive, () => Store(WaveBroadcastScratchPointer(), firstValue)); - EmitWave64Barrier(); - broadcast = Load(_uintType, WaveBroadcastScratchPointer()); - EmitWave64Barrier(); + broadcast = ExchangeWave64Value(isFirstActive, firstValue); } else if (_stage == Gen5SpirvStage.Compute || _enableGraphicsSubgroupOperations) { diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.cs b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.cs index 5f6201e..b57576e 100644 --- a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.cs +++ b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.cs @@ -17,6 +17,8 @@ public static partial class Gen5SpirvTranslator // so a smaller array is safe. private const uint PrivateLdsDwordCount = 2048; private const uint RdnaWaveLaneCount = 32; + private const uint Wave64ExchangeSlotCount = 4; + private const uint Wave64ExchangeDwordCount = Wave64ExchangeSlotCount * 2; internal static SpirvImageFormat DecodeStorageImageFormat( uint dataFormat, @@ -185,12 +187,13 @@ public static partial class Gen5SpirvTranslator private uint _localInvocationIndexInput; private uint _workGroupIdInput; private uint _pushConstantUintPointer; - private uint _subgroupSizeInput; private uint _subgroupInvocationIdInput; - private uint _waveMaskScratch; - private uint _waveMaskScratchElementPointer; - private uint _waveBroadcastScratch; - private bool _waveScratchInLds; + private uint _wave64Exchange; + private uint _wave64ExchangeElementPointer; + private uint _wave64ExchangeOffset; + private uint _wave64ExchangeParity; + private Ir.Gen5Wave64HalfMaskPlan? _halfMaskPlan; + private uint? _emittingPc; private uint _glsl; private enum ImageComponentKind @@ -620,7 +623,7 @@ public static partial class Gen5SpirvTranslator private void DeclareWave64Scratch() { - if (!_emulateWave64 || !UsesSubgroupOperations()) + if (!UsesWave64Exchange()) { return; } @@ -628,36 +631,39 @@ public static partial class Gen5SpirvTranslator // Metal exposes 32 KiB of threadgroup memory on the Apple GPUs we // target. Some PS5 compute shaders legitimately request all of it, // so allocating another workgroup variable for the wave64 bridge - // makes pipeline creation fail. Reuse the final three dwords of the + // makes pipeline creation fail. Reuse the final dwords of the // existing LDS allocation in that case. The translator already // bounds guest LDS accesses to this fixed allocation; keeping the // bridge inside it preserves the host limit and still provides the // cross-subgroup rendezvous needed to model one 64-lane guest wave. if (_lds != 0) { - _waveScratchInLds = true; - _waveMaskScratchElementPointer = _ldsElementPointer; + _wave64Exchange = _lds; + _wave64ExchangeElementPointer = _ldsElementPointer; + _wave64ExchangeOffset = _ldsDwordMask + 1 < LdsDwordCount + ? _ldsDwordMask + 1 + : LdsDwordCount - Wave64ExchangeDwordCount; return; } - var maskArrayType = _module.TypeArray(_uintType, 2); - var maskArrayPointer = - _module.TypePointer(SpirvStorageClass.Workgroup, maskArrayType); - _waveMaskScratchElementPointer = - _module.TypePointer(SpirvStorageClass.Workgroup, _uintType); - _waveMaskScratch = _module.AddGlobalVariable( - maskArrayPointer, + var exchangeArrayType = _module.TypeArray(_uintType, Wave64ExchangeDwordCount); + _wave64ExchangeElementPointer = _module.TypePointer(SpirvStorageClass.Workgroup, _uintType); + _wave64Exchange = _module.AddGlobalVariable( + _module.TypePointer(SpirvStorageClass.Workgroup, exchangeArrayType), SpirvStorageClass.Workgroup); - _module.AddName(_waveMaskScratch, "wave64MaskScratch"); - _interfaces.Add(_waveMaskScratch); + _wave64ExchangeOffset = 0; + _module.AddName(_wave64Exchange, "wave64Exchange"); + _interfaces.Add(_wave64Exchange); + } - var uintPointer = - _module.TypePointer(SpirvStorageClass.Workgroup, _uintType); - _waveBroadcastScratch = _module.AddGlobalVariable( - uintPointer, - SpirvStorageClass.Workgroup); - _module.AddName(_waveBroadcastScratch, "wave64BroadcastScratch"); - _interfaces.Add(_waveBroadcastScratch); + private bool UsesWave64Exchange() => _emulateWave64 && UsesSubgroupOperations(); + + private uint ComputeLdsGuestDwordCount() + { + var declared = _request.LocalDataShareDwords; + return declared == 0 || declared > LdsDwordCount + ? LdsDwordCount + : System.Numerics.BitOperations.RoundUpToPowerOf2(declared); } private void DeclareLds() @@ -679,11 +685,14 @@ public static partial class Gen5SpirvTranslator ? SpirvStorageClass.Workgroup : SpirvStorageClass.Private; var dwordCount = _stage == Gen5SpirvStage.Compute - ? LdsDwordCount + ? ComputeLdsGuestDwordCount() : PrivateLdsDwordCount; _ldsDwordMask = dwordCount - 1; + var arrayDwordCount = UsesWave64Exchange() && dwordCount < LdsDwordCount + ? dwordCount + Wave64ExchangeDwordCount + : dwordCount; - var ldsArrayType = _module.TypeArray(_uintType, dwordCount); + var ldsArrayType = _module.TypeArray(_uintType, arrayDwordCount); var ldsPointer = _module.TypePointer(storageClass, ldsArrayType); _ldsElementPointer = _module.TypePointer(storageClass, _uintType); _lds64ElementPointer = _module.TypePointer(storageClass, _ulongType); @@ -767,24 +776,6 @@ public static partial class Gen5SpirvTranslator } _interfaces.Add(_subgroupInvocationIdInput); - if (_emulateWave64) - { - _subgroupSizeInput = _module.AddGlobalVariable( - subgroupPointer, - SpirvStorageClass.Input); - _module.AddDecoration( - _subgroupSizeInput, - SpirvDecoration.BuiltIn, - (uint)SpirvBuiltIn.SubgroupSize); - if (_stage == Gen5SpirvStage.Pixel) - { - _module.AddDecoration( - _subgroupSizeInput, - SpirvDecoration.Flat); - } - _interfaces.Add(_subgroupSizeInput); - } - // LocalInvocationIndex is a compute-stage built-in. Graphics // stages can still use native subgroup operations, but they // cannot combine two subgroup32 halves through a workgroup. @@ -1101,7 +1092,12 @@ public static partial class Gen5SpirvTranslator { Store(_pixelValidMaskActive, _module.ConstantBool(true)); } - if (_subgroupInvocationIdInput != 0) + if (_subgroupInvocationIdInput != 0 && _emulateWave64) + { + StoreS64(106, BothHalvesOfOwnBallot(_module.ConstantBool(false))); + StoreS64(126, BothHalvesOfOwnBallot(_module.ConstantBool(true))); + } + else if (_subgroupInvocationIdInput != 0) { StoreWaveMask(106, _module.ConstantBool(false)); StoreWaveMask(126, _module.ConstantBool(true)); @@ -1360,12 +1356,28 @@ public static partial class Gen5SpirvTranslator { error = string.Empty; var block = blocks[blockIndex]; + var halfMaskPlan = HalfMaskPlan(); // One guest wave can span two host subgroups. Keep its shared-memory phases ordered. - var synchronizeSharedMemory = _emulateWave64; + // The half-mask plan inserts its own barriers; other wave64 programs also + // need LDS phase ordering when reads and writes span basic blocks. + var synchronizeSharedMemory = _emulateWave64 && halfMaskPlan is null; var sharedMemoryPhase = SharedMemoryPhase.None; for (var index = block.StartIndex; index < block.EndIndex; index++) { var instruction = _request.Program.Instructions[index]; + if (halfMaskPlan is not null) + { + if (halfMaskPlan.ExactPairsBefore.TryGetValue(instruction.Pc, out var exactPairs)) + { + EmitExactWaveMasks(exactPairs); + } + + if (halfMaskPlan.SharedMemoryBarriersBefore.Contains(instruction.Pc)) + { + EmitWave64Barrier(); + } + } + if (IsBranch(instruction.Opcode) || instruction.Opcode == "SEndpgm") { continue; @@ -1386,7 +1398,9 @@ public static partial class Gen5SpirvTranslator } _execKnownFull = IsExecKnownFull(instruction.Pc); + _emittingPc = instruction.Pc; var emitted = TryEmitInstruction(instruction, out error); + _emittingPc = null; _execKnownFull = false; if (!emitted) { @@ -1650,10 +1664,10 @@ public static partial class Gen5SpirvTranslator { "SCbranchScc0" => LogicalNot(Load(_boolType, _scc)), "SCbranchScc1" => Load(_boolType, _scc), - "SCbranchVccz" => LogicalNot(SubgroupAny(Load(_boolType, _vcc))), - "SCbranchVccnz" => SubgroupAny(Load(_boolType, _vcc)), - "SCbranchExecz" => LogicalNot(SubgroupAny(Load(_boolType, _exec))), - "SCbranchExecnz" => SubgroupAny(Load(_boolType, _exec)), + "SCbranchVccz" => LogicalNot(WaveMaskAny(106, _vcc)), + "SCbranchVccnz" => WaveMaskAny(106, _vcc), + "SCbranchExecz" => LogicalNot(WaveMaskAny(126, _exec)), + "SCbranchExecnz" => WaveMaskAny(126, _exec), // The emulator does not expose a shader debug session. "SCbranchCdbgsys" or "SCbranchCdbguser" or @@ -2188,11 +2202,7 @@ public static partial class Gen5SpirvTranslator uint broadcast; if (_emulateWave64) { - EmitConditional(isFirstActive, () => - Store(WaveBroadcastScratchPointer(), firstValue)); - EmitWave64Barrier(); - broadcast = Load(_uintType, WaveBroadcastScratchPointer()); - EmitWave64Barrier(); + broadcast = ExchangeWave64Value(isFirstActive, firstValue); } else { @@ -7032,6 +7042,11 @@ public static partial class Gen5SpirvTranslator _programCounter = Variable(_uintType, _module.Constant(_uintType, 0)); _programActive = Flag(true); + if (UsesWave64Exchange()) + { + _wave64ExchangeParity = Variable(_uintType, _module.Constant(_uintType, 0)); + _module.AddName(_wave64ExchangeParity, "wave64ExchangeParity"); + } if (_maxDispatcherSteps > 0) { _iterationGuard = Variable(_uintType, _module.Constant(_uintType, 0)); @@ -7545,6 +7560,19 @@ public static partial class Gen5SpirvTranslator return BooleanToLaneMask(condition); } + if (_emulateWave64) + { + var lane = GuestWaveLane(); + var owned = OwnHalfBallot(condition); + var exchange = BeginWave64Exchange(); + EmitConditional(IsHalfWaveLeader(lane), () => + Store(Wave64ExchangePointer(exchange, ShiftRightLogical(lane, UInt(5))), owned)); + EmitWave64Barrier(); + return Pair64( + Load(_uintType, Wave64ExchangePointer(exchange, UInt(0))), + Load(_uintType, Wave64ExchangePointer(exchange, UInt(1)))); + } + var ballot = _module.AddInstruction( SpirvOp.GroupNonUniformBallot, _uvec4Type, @@ -7555,57 +7583,6 @@ public static partial class Gen5SpirvTranslator _uintType, ballot, 0); - if (_emulateWave64) - { - var high = _module.AddInstruction( - SpirvOp.CompositeExtract, - _uintType, - ballot, - 1); - var subgroupLane = - Load(_uintType, _subgroupInvocationIdInput); - var firstLane = _module.AddInstruction( - SpirvOp.IEqual, - _boolType, - subgroupLane, - UInt(0)); - var half = ShiftRightLogical(GuestWaveLane(), UInt(5)); - EmitConditional(firstLane, () => - { - Store(WaveMaskScratchPointer(half), low); - - var nativeWave64 = _module.AddInstruction( - SpirvOp.UGreaterThanEqual, - _boolType, - Load(_uintType, _subgroupSizeInput), - UInt(64)); - EmitConditional(nativeWave64, () => - { - Store(WaveMaskScratchPointer(UInt(1)), high); - }); - }); - EmitWave64Barrier(); - var lowMask = Load( - _uintType, - WaveMaskScratchPointer(UInt(0))); - var highMask = Load( - _uintType, - WaveMaskScratchPointer(UInt(1))); - var combined = BitwiseOr64( - _module.AddInstruction( - SpirvOp.UConvert, - _ulongType, - lowMask), - ShiftLeftLogical64( - _module.AddInstruction( - SpirvOp.UConvert, - _ulongType, - highMask), - _module.Constant64(_ulongType, 32))); - EmitWave64Barrier(); - return combined; - } - var widened = _module.AddInstruction(SpirvOp.UConvert, _ulongType, low); if (_waveLaneCount != 64) { @@ -7626,21 +7603,127 @@ public static partial class Gen5SpirvTranslator widened); } - private uint WaveMaskScratchPointer(uint index) => + private uint OwnHalfBallot(uint condition) + { + var ballot = _module.AddInstruction( + SpirvOp.GroupNonUniformBallot, + _uvec4Type, + UInt(3), + condition); + return _module.AddInstruction( + SpirvOp.VectorExtractDynamic, + _uintType, + ballot, + ShiftRightLogical(Load(_uintType, _subgroupInvocationIdInput), UInt(5))); + } + + private uint BooleanToHalfWaveMask(uint condition) + { + var owned = _module.AddInstruction(SpirvOp.UConvert, _ulongType, OwnHalfBallot(condition)); + return _module.AddInstruction( + SpirvOp.Select, + _ulongType, + _module.AddInstruction(SpirvOp.UGreaterThanEqual, _boolType, GuestWaveLane(), UInt(32)), + ShiftLeftLogical64(owned, _module.Constant64(_ulongType, 32)), + owned); + } + + private uint BothHalvesOfOwnBallot(uint condition) + { + var owned = _module.AddInstruction(SpirvOp.UConvert, _ulongType, OwnHalfBallot(condition)); + return BitwiseOr64(owned, ShiftLeftLogical64(owned, _module.Constant64(_ulongType, 32))); + } + + private uint IsHalfWaveLeader(uint lane) => + _module.AddInstruction(SpirvOp.IEqual, _boolType, BitwiseAnd(lane, UInt(31)), UInt(0)); + + private uint BeginWave64Exchange() + { + var parity = Load(_uintType, _wave64ExchangeParity); + Store(_wave64ExchangeParity, BitwiseXor(parity, UInt(1))); + return IAdd( + UInt(_wave64ExchangeOffset), + _module.AddInstruction(SpirvOp.IMul, _uintType, parity, UInt(Wave64ExchangeSlotCount))); + } + + private uint Wave64ExchangePointer(uint exchange, uint slot) => _module.AddInstruction( SpirvOp.AccessChain, - _waveMaskScratchElementPointer, - _waveScratchInLds ? _lds : _waveMaskScratch, - _waveScratchInLds ? IAdd(UInt(LdsDwordCount - 3), index) : index); + _wave64ExchangeElementPointer, + _wave64Exchange, + IAdd(exchange, slot)); - private uint WaveBroadcastScratchPointer() => - _waveScratchInLds - ? _module.AddInstruction( - SpirvOp.AccessChain, - _ldsElementPointer, - _lds, - UInt(LdsDwordCount - 1)) - : _waveBroadcastScratch; + private uint ExchangeWave64Value(uint writer, uint value) + { + var exchange = BeginWave64Exchange(); + EmitConditional(writer, () => Store(Wave64ExchangePointer(exchange, UInt(0)), value)); + EmitWave64Barrier(); + return Load(_uintType, Wave64ExchangePointer(exchange, UInt(0))); + } + + private void EmitExactWaveMasks(IReadOnlyList pairs) + { + var lane = GuestWaveLane(); + var upperHalf = ShiftRightLogical(lane, UInt(5)); + var lowerHalf = _module.AddInstruction(SpirvOp.IEqual, _boolType, upperHalf, UInt(0)); + const int pairsPerExchange = (int)Wave64ExchangeSlotCount / 2; + for (var start = 0; start < pairs.Count; start += pairsPerExchange) + { + var count = Math.Min(pairsPerExchange, pairs.Count - start); + var owned = new uint[count]; + for (var index = 0; index < count; index++) + { + var pair = pairs[start + index]; + owned[index] = _module.AddInstruction(SpirvOp.Select, _uintType, lowerHalf, LoadS(pair), LoadS(pair + 1)); + } + + var exchange = BeginWave64Exchange(); + EmitConditional(IsHalfWaveLeader(lane), () => + { + for (var index = 0; index < count; index++) + { + Store(Wave64ExchangePointer(exchange, IAdd(UInt((uint)index * 2), upperHalf)), owned[index]); + } + }); + EmitWave64Barrier(); + for (var index = 0; index < count; index++) + { + var pair = pairs[start + index]; + StoreS(pair, Load(_uintType, Wave64ExchangePointer(exchange, UInt((uint)index * 2)))); + StoreS(pair + 1, Load(_uintType, Wave64ExchangePointer(exchange, UInt((uint)index * 2 + 1)))); + } + } + } + + private uint WaveMaskAny(uint register, uint laneActive) => + _emulateWave64 && _subgroupInvocationIdInput != 0 + ? IsNotZero64(LoadS64(register)) + : SubgroupAny(Load(_boolType, laneActive)); + + private static readonly bool HalfWaveMasks = !string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_WAVE64_HALF_MASKS"), "0", StringComparison.Ordinal); + + private bool _halfMaskPlanBuilt; + + private Ir.Gen5Wave64HalfMaskPlan? HalfMaskPlan() + { + if (_halfMaskPlanBuilt) + { + return _halfMaskPlan; + } + + _halfMaskPlanBuilt = true; + if (HalfWaveMasks && UsesWave64Exchange() && _subgroupInvocationIdInput != 0) + { + var sharedFlatPcs = _request.Memory.Entries + .Where(static memory => memory.AddressSpace is FlatAddressSpace.Shared or FlatAddressSpace.SharedOrPrivate) + .Select(static memory => memory.Pc) + .ToHashSet(); + _halfMaskPlan = Ir.Gen5Wave64HalfMaskAnalysis.Analyze(_request.Program, sharedFlatPcs); + } + + return _halfMaskPlan; + } private void EmitWave64Barrier() { @@ -7685,6 +7768,17 @@ public static partial class Gen5SpirvTranslator return; } + if (_emulateWave64 && + _subgroupInvocationIdInput != 0 && + _emittingPc is { } pc && + HalfMaskPlan() is { } plan && + plan.HalfMaskWrites.TryGetValue(pc, out var halfRegister) && + halfRegister == register) + { + StoreS64(register, BooleanToHalfWaveMask(condition)); + return; + } + StoreS64(register, BooleanToWaveMask(condition)); } diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Vulkan/Wave64HalfMaskDeviceTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Vulkan/Wave64HalfMaskDeviceTests.cs new file mode 100644 index 0000000..5bda8e7 --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Gpu/Vulkan/Wave64HalfMaskDeviceTests.cs @@ -0,0 +1,269 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Buffers.Binary; +using SharpEmu.Libs.Gpu.Buffers; +using SharpEmu.Libs.Tests.Gpu.Images; +using SharpEmu.ShaderCompiler; +using SharpEmu.ShaderCompiler.Resources; +using SharpEmu.ShaderCompiler.Vulkan; +using Xunit; +using static SharpEmu.ShaderCompiler.Tests.Resources.ResourceTestProgram; + +namespace SharpEmu.Libs.Tests.Gpu.Vulkan; + +public sealed class Wave64HalfMaskDeviceTests(HeadlessVulkanFixture fixture) : IClassFixture +{ + private const uint Vcc = 106; + private const uint Exec = 126; + private static readonly Gen5Operand AllLanes = Gen5Operand.Source(193); + + private readonly List _instructions = []; + private uint _pc; + + private uint Add(Gen5ShaderInstruction instruction) + { + var pc = _pc; + _instructions.Add(instruction with { Pc = pc }); + _pc += 8; + return pc; + } + + private void Point(uint branchPc, uint targetPc) + { + var index = _instructions.FindIndex(instruction => instruction.Pc == branchPc); + var offset = (short)(((int)targetPc - (int)branchPc - 4) / 4); + _instructions[index] = _instructions[index] with { Words = [unchecked((uint)(ushort)offset)] }; + } + + private void Lane() + { + Add(Vop2(0, "VLshlrevB32", 2, Operand(3), Gen5Operand.Vector(1))); + Add(Vop2(0, "VAddI32", 2, Gen5Operand.Vector(0), Gen5Operand.Vector(2))); + } + + private uint CompareTo(string opcode, uint destination, Gen5Operand left, Gen5Operand right) => + Add(new Gen5ShaderInstruction(0, Gen5ShaderEncoding.Vop3, opcode, [0u, 0u], [left, right], [Gen5Operand.Scalar(destination)], + new Gen5Vop3Control(0, 0, 0, false, 0, destination))); + + private uint[] Run(uint localDataShareDwords = 0) + { + Add(Vop2(0, "VLshlrevB32", 5, Operand(2), Gen5Operand.Vector(2))); + Add(BufferAccess(0, "BufferStoreDword", 4, vectorData: 4, offsetEnabled: true, vectorAddress: 5)); + Add(EndProgram(0)); + var (plan, resources, layout) = Prepare(Program([.. _instructions])); + var request = new ShaderCompileRequest(plan, resources, layout) + { + LocalSizeX = 8, LocalSizeY = 8, ThreadCountX = 8, ThreadCountY = 8, WaveSize = 64, + LocalDataShareDwords = localDataShareDwords, + }; + Assert.True(Gen5SpirvTranslator.TryCompileProgram(request, out var shader, out var error), error); + Assert.NotNull(fixture.Vulkan); + using var harness = new ImageTestHarness(fixture.Vulkan); + using var runner = new LayoutComputeRunner(harness, request, shader.Spirv); + var output = runner.CreateBuffer(256); + var registers = new uint[256]; + registers[6] = 256; + harness.Run(() => runner.Dispatch(registers, + new Dictionary { [DescriptorBindingKind.Buffers] = [output] }, 1)); + var bytes = runner.ReadBack(output, 0, 256); + harness.AssertNoValidationMessages(); + var lanes = new uint[64]; + for (var lane = 0; lane < lanes.Length; lane++) + { + lanes[lane] = BinaryPrimitives.ReadUInt32LittleEndian(bytes.AsSpan(lane * sizeof(uint))); + } + + return lanes; + } + + private bool Ready() => GatePrerequisites.Ready(fixture.Vulkan, shaderInt64: true); + + [Fact] + public void MasksAcrossTheHalfBoundary_StayExactPerLaneAndWhenCounted() + { + if (!Ready()) return; + Lane(); + CompareTo("VCmpLtU32", 10, Gen5Operand.Vector(2), Operand(40)); + Add(Vop2(0, "VAndB32", 6, Operand(1), Gen5Operand.Vector(2))); + Add(Vopc(0, "VCmpEqU32", Operand(1), 6)); + Add(Sop2(0, "SAndB64", 12, Gen5Operand.Scalar(10), Gen5Operand.Scalar(Vcc))); + Add(Vop3(0, "VCndmaskB32", 4, Operand(0), Operand(1000), Gen5Operand.Scalar(12))); + Add(Sop1(0, "SBcnt1I32B64", 14, Gen5Operand.Scalar(12))); + Add(Vop2(0, "VAddU32", 4, Gen5Operand.Scalar(14), Gen5Operand.Vector(4))); + + var lanes = Run(); + for (uint lane = 0; lane < 64; lane++) + { + Assert.Equal((lane & 1) != 0 && lane < 40 ? 1020u : 20u, lanes[lane]); + } + } + + [Theory] + [InlineData(48u, 77u)] + [InlineData(8u, 77u)] + [InlineData(64u, 5u)] + public void AnExecBranch_IsDecidedForTheWholeWave(uint firstActiveLane, uint expected) + { + if (!Ready()) return; + Lane(); + Add(MoveScalar(0, 20, 5)); + Add(Vopc(0, "VCmpxLeU32", Operand(firstActiveLane), 2)); + var skip = Add(Branch(0, "SCbranchExecz", 0)); + Add(MoveScalar(0, 20, 77)); + var join = Add(Sop1(0, "SMovB64", Exec, AllLanes)); + Point(skip, join); + Add(MoveVectorFromScalar(0, 4, 20)); + + Assert.All(Run(), value => Assert.Equal(expected, value)); + } + + [Theory] + [InlineData(50u, 9u)] + [InlineData(10u, 9u)] + [InlineData(64u, 3u)] + public void AMaskWhoseSccIsRead_CountsBothHalves(uint threshold, uint expected) + { + if (!Ready()) return; + Lane(); + Add(MoveScalar(0, 20, 3)); + CompareTo("VCmpGeU32", 10, Gen5Operand.Vector(2), Operand(threshold)); + Add(Sop2(0, "SAndB64", 12, Gen5Operand.Scalar(10), Gen5Operand.Scalar(Exec))); + var skip = Add(Branch(0, "SCbranchScc0", 0)); + Add(MoveScalar(0, 20, 9)); + var join = Add(MoveVectorFromScalar(0, 4, 20)); + Point(skip, join); + + Assert.All(Run(), value => Assert.Equal(expected, value)); + } + + [Theory] + [InlineData(0u)] + [InlineData(64u)] + public void SharedMemoryWrittenByOneHalf_IsReadByTheOther(uint localDataShareDwords) + { + if (!Ready()) return; + Lane(); + Add(Vop2(0, "VLshlrevB32", 3, Operand(1), Gen5Operand.Vector(2))); + Add(Vop2(0, "VAddU32", 3, Gen5Operand.Vector(2), Gen5Operand.Vector(3))); + Add(Vop2(0, "VLshlrevB32", 6, Operand(2), Gen5Operand.Vector(2))); + Add(Vopc(0, "VCmpxGtU32", Operand(32), 2)); + Add(BufferAccess(0, "BufferLoadDword", 4, vectorData: 8, offsetEnabled: true, vectorAddress: 6)); + Add(Vop2(0, "VAddU32", 3, Gen5Operand.Vector(8), Gen5Operand.Vector(3))); + Add(DataShare(0, "DsWriteB32", false, [Gen5Operand.Vector(6), Gen5Operand.Vector(3)], [])); + Add(Sop1(0, "SMovB64", Exec, AllLanes)); + Add(Vop2(0, "VAndB32", 7, Operand(31), Gen5Operand.Vector(2))); + Add(Vop2(0, "VLshlrevB32", 7, Operand(2), Gen5Operand.Vector(7))); + Add(DataShare(0, "DsReadB32", false, [Gen5Operand.Vector(7)], [4])); + + var lanes = Run(localDataShareDwords); + for (uint lane = 0; lane < 64; lane++) + { + Assert.Equal((lane & 31) * 3, lanes[lane]); + } + } + + [Fact] + public void AnExecLoop_RunsEveryLaneItsOwnCount() + { + if (!Ready()) return; + Lane(); + Add(Vop2(0, "VAndB32", 3, Operand(7), Gen5Operand.Vector(2))); + Add(Vop2(0, "VAddU32", 3, Operand(1), Gen5Operand.Vector(3))); + Add(MoveVector(0, 4, 0)); + Add(Sop1(0, "SMovB64", 20, Gen5Operand.Scalar(Exec))); + var header = Add(Vopc(0, "VCmpxGtU32", Gen5Operand.Vector(3), 4)); + var exit = Add(Branch(0, "SCbranchExecz", 0)); + Add(Vop2(0, "VAddU32", 4, Operand(1), Gen5Operand.Vector(4))); + var back = Add(Branch(0, "SBranch", 0)); + var done = Add(Sop1(0, "SMovB64", Exec, Gen5Operand.Scalar(20))); + Point(exit, done); + Point(back, header); + + var lanes = Run(); + for (uint lane = 0; lane < 64; lane++) + { + Assert.Equal((lane & 7) + 1, lanes[lane]); + } + } + + [Theory] + [InlineData(40u, 140u)] + [InlineData(3u, 103u)] + [InlineData(64u, 100u)] + public void ReadFirstLane_TakesTheFirstActiveLaneOfTheWholeWave(uint firstActiveLane, uint expected) + { + if (!Ready()) return; + Lane(); + Add(Vop2(0, "VAddU32", 3, Operand(100), Gen5Operand.Vector(2))); + Add(Vopc(0, "VCmpxLeU32", Operand(firstActiveLane), 2)); + Add(ReadFirstLane(0, 20, 3)); + Add(Sop1(0, "SMovB64", Exec, AllLanes)); + Add(MoveVectorFromScalar(0, 4, 20)); + + Assert.All(Run(), value => Assert.Equal(expected, value)); + } + + [Fact] + public void ADemonsSoulsIndexRemap_CompilesOnTheDevice() + { + if (!Ready()) return; + uint[] words = + [ + 0xBEFC03FFu, 0xA65CD91Eu, 0xBF960000u, 0xBFA00003u, 0xD7460002u, 0x04010C0Cu, 0xF4201A84u, 0xFA000000u, + 0xBF8CC07Fu, 0x7DA8046Au, 0xBF88002Au, 0xF4200304u, 0xFA000004u, 0xBF8CC07Fu, 0x7E000C0Cu, 0xBF070C80u, + 0x8588807Eu, 0x7E005700u, 0x100000FFu, 0x4F800000u, 0x7E060F00u, 0xD5766A00u, 0x0202060Cu, 0x7D8A0280u, + 0x4C020080u, 0x02000101u, 0xD56A0001u, 0x00020700u, 0x4C000303u, 0x4A020303u, 0x02000101u, 0xD56A0000u, + 0x00020500u, 0xD5690001u, 0x0002000Cu, 0x4C060302u, 0x7D8C02F9u, 0x06068A02u, 0x7D86060Cu, 0x87EA6A0Au, + 0x50000080u, 0xD5286A00u, 0x002A00C1u, 0xD5010000u, 0x002200C1u, 0xD5690000u, 0x0002000Cu, 0x4C000102u, + 0xE0002000u, 0x80000000u, 0xBF8C3F70u, 0xE0102000u, 0x80010002u, 0xBF810000u, + ]; + var bytes = new byte[words.Length * sizeof(uint)]; + for (var index = 0; index < words.Length; index++) + BinaryPrimitives.WriteUInt32LittleEndian(bytes.AsSpan(index * sizeof(uint)), words[index]); + Assert.True(Gen5ShaderTranslator.TryDecodeProgram(new SharpEmu.HLE.CpuContext(new WordMemory(bytes), SharpEmu.HLE.Generation.Gen5), + 0x1000, out var program, out var decodeError), decodeError); + var (plan, resources, layout) = Prepare(program, userDataCount: 12); + var request = new ShaderCompileRequest(plan, resources, layout) + { + LocalSizeX = 64, WaveSize = 64, EnableExecGuardElision = false, + ComputeSystemRegisters = new Gen5ComputeSystemRegisters(12, null, null, null), + }; + Assert.True(Gen5SpirvTranslator.TryCompileProgram(request, out var shader, out var error), error); + Assert.NotNull(fixture.Vulkan); + using var harness = new ImageTestHarness(fixture.Vulkan); + using var runner = new LayoutComputeRunner(harness, request, shader.Spirv); + harness.AssertNoValidationMessages(); + } + + private sealed class WordMemory(byte[] bytes) : SharpEmu.HLE.ICpuMemory + { + public bool TryRead(ulong address, Span destination) + { + if (address < 0x1000 || address - 0x1000 > (ulong)(bytes.Length - destination.Length)) return false; + bytes.AsSpan((int)(address - 0x1000), destination.Length).CopyTo(destination); + return true; + } + + public bool TryWrite(ulong address, ReadOnlySpan source) => false; + } + + [Fact] + public void ASavedExec_RestoresAfterAHalfMaskedBody() + { + if (!Ready()) return; + Lane(); + Add(MoveVector(0, 4, 1)); + CompareTo("VCmpGeU32", 10, Gen5Operand.Vector(2), Operand(28)); + Add(Sop1(0, "SAndSaveexecB64", 12, Gen5Operand.Scalar(10))); + Add(MoveVector(0, 4, 2)); + Add(Sop2(0, "SOrB64", Exec, Gen5Operand.Scalar(Exec), Gen5Operand.Scalar(12))); + Add(Vop2(0, "VAddU32", 4, Operand(10), Gen5Operand.Vector(4))); + + var lanes = Run(); + for (uint lane = 0; lane < 64; lane++) + { + Assert.Equal(lane >= 28 ? 12u : 11u, lanes[lane]); + } + } +}