From 7a6e3d03086f8d0bb77eb2f5482a753bc117d546 Mon Sep 17 00:00:00 2001 From: ParantezTech <12572227+par274@users.noreply.github.com> Date: Sun, 11 Oct 2026 01:34:42 +0300 Subject: [PATCH] [ShaderCompiler] Run wave64 compute as paired lanes on 32-lane subgroups --- .../Gen5SpirvTranslator.Alu.cs | 21 +- .../Gen5SpirvTranslator.PairedLanes.cs | 464 ++++++++++++++++++ .../Gen5SpirvTranslator.Resources.cs | 18 + .../Gen5SpirvTranslator.cs | 84 +++- 4 files changed, 567 insertions(+), 20 deletions(-) create mode 100644 src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.PairedLanes.cs diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Alu.cs b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Alu.cs index a12642c..cda8ecc 100644 --- a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Alu.cs +++ b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Alu.cs @@ -51,8 +51,9 @@ public static partial class Gen5SpirvTranslator return false; } - var value = GetRawSource(instruction, 0); - value = BroadcastFirstWaveActive(value); + var value = _pairedLanes + ? PairedReadFirstLane(instruction) + : BroadcastFirstWaveActive(GetRawSource(instruction, 0)); StoreS(instruction.Destinations[0].Value, value); return true; @@ -2741,7 +2742,7 @@ public static partial class Gen5SpirvTranslator if (instruction.Opcode.EndsWith("SaveexecB32", StringComparison.Ordinal)) { - var oldExec64 = BooleanToWaveMask(Load(_boolType, _exec)); + var oldExec64 = CurrentExecMask(); var oldExec = _module.AddInstruction( SpirvOp.UConvert, _uintType, @@ -4348,6 +4349,12 @@ public static partial class Gen5SpirvTranslator private void StoreS64(uint register, uint value) { + if (_pairedLanes && _inLanePass) + { + StorePairedLaneMaskWord(register, value); + return; + } + StoreS( register, _module.AddInstruction(SpirvOp.UConvert, _uintType, value)); @@ -4950,8 +4957,14 @@ public static partial class Gen5SpirvTranslator } var destination = instruction.Destinations[0].Value; - var sourceValue = GetRawSource(instruction, 0); var selectedLane = BitwiseAnd(GetRawSource(instruction, 1), UInt(LaneSelectMask)); + if (_pairedLanes) + { + StoreS(destination, PairedReadLane(instruction, selectedLane)); + return true; + } + + var sourceValue = GetRawSource(instruction, 0); if (_stage == Gen5SpirvStage.Compute && _waveLowering is EmulatedWave64Lowering) { diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.PairedLanes.cs b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.PairedLanes.cs new file mode 100644 index 0000000..6e325ac --- /dev/null +++ b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.PairedLanes.cs @@ -0,0 +1,464 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.ShaderCompiler.Vulkan; + +public static partial class Gen5SpirvTranslator +{ + private sealed partial class CompilationContext + { + private const uint PairedHalfLaneCount = 32; + + private static readonly bool PairedWave64Enabled = !string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_WAVE64_PAIRED"), "0", StringComparison.Ordinal); + + private readonly bool _pairedLanes; + private int _laneHalf; + private bool _inLanePass; + private readonly Dictionary _upperVectorRegisterVariables = new(); + private readonly uint[] _packedHalfRegistersByHalf = new uint[2]; + private uint _lowerExec; + private uint _upperExec; + private uint _lowerVcc; + private uint _upperVcc; + private uint _lowerScratch; + private uint _upperScratch; + private uint _subgroupIdInput; + private uint _pairedCounterResult; + private Dictionary? _scalarSnapshot; + + private void SelectLaneHalf(int half) + { + if (!_pairedLanes) + { + return; + } + + _packedHalfRegistersByHalf[_laneHalf] = _packedHalfRegisters; + _laneHalf = half; + _packedHalfRegisters = _packedHalfRegistersByHalf[half]; + _exec = half == 0 ? _lowerExec : _upperExec; + _vcc = half == 0 ? _lowerVcc : _upperVcc; + if (_lowerScratch != 0) + { + _scratch = half == 0 ? _lowerScratch : _upperScratch; + } + } + + private void DeclarePairedLaneFlags(uint lowerVcc, uint lowerExec, Func flag) + { + _lowerVcc = lowerVcc; + _lowerExec = lowerExec; + if (!_pairedLanes) + { + return; + } + + _upperVcc = flag(false); + _upperExec = flag(true); + _interfaces.Add(_upperVcc); + _interfaces.Add(_upperExec); + _module.AddName(_upperVcc, "vccUpperHalf"); + _module.AddName(_upperExec, "execUpperHalf"); + } + + private void DeclarePairedScratch(uint arrayPointer, uint arrayType) + { + _lowerScratch = _scratch; + if (!_pairedLanes) + { + return; + } + + _upperScratch = _module.AddGlobalVariable( + arrayPointer, + SpirvStorageClass.Private, + _module.ConstantNull(arrayType)); + _interfaces.Add(_upperScratch); + _module.AddName(_upperScratch, "scratchUpperHalf"); + } + + private void DeclarePairedLaneInputs() + { + if (!_pairedLanes) + { + return; + } + + _subgroupIdInput = _module.AddGlobalVariable( + _module.TypePointer(SpirvStorageClass.Input, _uintType), + SpirvStorageClass.Input); + _module.AddDecoration(_subgroupIdInput, SpirvDecoration.BuiltIn, (uint)SpirvBuiltIn.SubgroupId); + _interfaces.Add(_subgroupIdInput); + } + + private uint PairedHostThreadCount() + { + var guestThreads = checked(_localSizeX * _localSizeY * _localSizeZ); + return (guestThreads + 63) / 64 * PairedHalfLaneCount; + } + + private Dictionary CurrentVectorRegisterVariables() => + _pairedLanes && _laneHalf == 1 ? _upperVectorRegisterVariables : _vectorRegisterVariables; + + private string CurrentVectorRegisterPrefix() => + _pairedLanes && _laneHalf == 1 ? "vu" : "v"; + + private uint PairedSubgroupLane() => Load(_uintType, _subgroupInvocationIdInput); + + private uint PairedGuestLane() => + _laneHalf == 0 ? PairedSubgroupLane() : IAdd(PairedSubgroupLane(), UInt(PairedHalfLaneCount)); + + private uint HalfBallot(uint condition) => + _module.AddInstruction( + SpirvOp.CompositeExtract, + _uintType, + _module.AddInstruction(SpirvOp.GroupNonUniformBallot, _uvec4Type, UInt(3), condition), + 0); + + private uint SubgroupBroadcast(uint value, uint lane) => + _module.AddInstruction(SpirvOp.GroupNonUniformBroadcast, _uintType, UInt(3), value, lane); + + private uint ExecHalfBallot(int half) + { + var current = _laneHalf; + SelectLaneHalf(half); + var ballot = HalfBallot(Load(_boolType, _exec)); + SelectLaneHalf(current); + return ballot; + } + + private uint VectorSourceOfHalf(Gen5ShaderInstruction instruction, int sourceIndex, int half) + { + var current = _laneHalf; + SelectLaneHalf(half); + var value = GetRawSource(instruction, sourceIndex); + SelectLaneHalf(current); + return value; + } + + private static bool IsPairedLaneInstruction(Gen5ShaderInstruction instruction) => + instruction.Opcode is not ("VReadlaneB32" or "VReadfirstlaneB32" or "DsAppend" or "DsConsume") && + instruction.Encoding is + Gen5ShaderEncoding.Mubuf or + Gen5ShaderEncoding.Mtbuf or + Gen5ShaderEncoding.Vop1 or + Gen5ShaderEncoding.Vop2 or + Gen5ShaderEncoding.Vopc or + Gen5ShaderEncoding.Vop3 or + Gen5ShaderEncoding.Vop3p or + Gen5ShaderEncoding.Vintrp or + Gen5ShaderEncoding.Ds or + Gen5ShaderEncoding.Flat or + Gen5ShaderEncoding.Mimg or + Gen5ShaderEncoding.Exp; + + private bool TryEmitPairedLaneInstruction(Gen5ShaderInstruction instruction, out string error) + { + _scalarSnapshot = SnapshotScalarSources(instruction); + _inLanePass = true; + try + { + for (var half = 0; half < 2; half++) + { + SelectLaneHalf(half); + if (!TryEmitInstruction(instruction, out error)) + { + return false; + } + } + + error = string.Empty; + return true; + } + finally + { + _inLanePass = false; + _scalarSnapshot = null; + SelectLaneHalf(0); + } + } + + private Dictionary? SnapshotScalarSources(Gen5ShaderInstruction instruction) + { + Dictionary? snapshot = null; + foreach (var operand in instruction.Sources) + { + if (operand.Kind != Gen5OperandKind.ScalarRegister) + { + continue; + } + + for (var register = operand.Value; register <= operand.Value + 1 && register < ScalarRegisterCount; register++) + { + if (register == 125) + { + continue; + } + + snapshot ??= new Dictionary(); + if (!snapshot.ContainsKey(register)) + { + snapshot[register] = Load(_uintType, ScalarPointer(register)); + } + } + } + + return snapshot; + } + + private void UpdateLaneMaskFlag(uint lowRegister) + { + if (!_pairedLanes || _inLanePass) + { + Store(lowRegister == 106 ? _vcc : _exec, IsLaneSetInMaskRegisters(lowRegister)); + return; + } + + var current = _laneHalf; + for (var half = 0; half < 2; half++) + { + SelectLaneHalf(half); + Store(lowRegister == 106 ? _vcc : _exec, IsLaneSetInMaskRegisters(lowRegister)); + } + + SelectLaneHalf(current); + } + + private void StorePairedLaneMaskWord(uint register, uint value) + { + var word = _laneHalf == 0 + ? Narrow(value) + : Narrow(ShiftRightLogical64(value, _module.Constant64(_ulongType, 32))); + StoreS(register + (uint)_laneHalf, word); + } + + private void EmitPairedComputeInitialState() + { + var workGroupId = Load(_uvec3Type, _workGroupIdInput); + var waveBase = _module.AddInstruction( + SpirvOp.IMul, + _uintType, + Load(_uintType, _subgroupIdInput), + UInt(2 * PairedHalfLaneCount)); + var guestThreads = checked(_localSizeX * _localSizeY * _localSizeZ); + var limits = new uint[3]; + var groups = new uint[3]; + for (uint component = 0; component < 3; component++) + { + limits[component] = ComputeThreadLimit(component); + groups[component] = _module.AddInstruction( + SpirvOp.CompositeExtract, + _uintType, + workGroupId, + (uint)_physicalAxisOfLogical[component]); + } + + var active = new uint[2]; + for (var half = 0; half < 2; half++) + { + SelectLaneHalf(half); + var index = IAdd(waveBase, PairedGuestLane()); + var inBounds = _module.AddInstruction(SpirvOp.ULessThan, _boolType, index, UInt(guestThreads)); + var row = _module.AddInstruction(SpirvOp.UDiv, _uintType, index, UInt(_localSizeX)); + var local = new[] + { + _module.AddInstruction(SpirvOp.UMod, _uintType, index, UInt(_localSizeX)), + _module.AddInstruction(SpirvOp.UMod, _uintType, row, UInt(_localSizeY)), + _module.AddInstruction(SpirvOp.UDiv, _uintType, row, UInt(_localSizeY)), + }; + var sizes = new[] { _localSizeX, _localSizeY, _localSizeZ }; + for (uint component = 0; component < 3; component++) + { + StoreV(component, local[component], guardWithExec: false); + var global = IAdd( + _module.AddInstruction(SpirvOp.IMul, _uintType, groups[component], UInt(sizes[component])), + local[component]); + inBounds = LogicalAnd( + inBounds, + _module.AddInstruction(SpirvOp.ULessThan, _boolType, global, limits[component])); + } + + active[half] = inBounds; + } + + SelectLaneHalf(0); + var lowerActive = HalfBallot(active[0]); + var upperActive = HalfBallot(active[1]); + StoreS(126, lowerActive); + StoreS(127, upperActive); + Store(_programActive, _module.AddInstruction(SpirvOp.LogicalOr, _boolType, active[0], active[1])); + + if (_request.ComputeSystemRegisters is { } registers) + { + StoreComputeSystemRegister(registers.WorkGroupXRegister, workGroupId, (uint)_physicalAxisOfLogical[0]); + StoreComputeSystemRegister(registers.WorkGroupYRegister, workGroupId, (uint)_physicalAxisOfLogical[1]); + StoreComputeSystemRegister(registers.WorkGroupZRegister, workGroupId, (uint)_physicalAxisOfLogical[2]); + if (registers.ThreadGroupSizeRegister is { } sizeRegister) + { + StoreS(sizeRegister, UInt(guestThreads)); + } + } + } + + private uint PairedReadFirstLane(Gen5ShaderInstruction instruction) + { + var lowerActive = ExecHalfBallot(0); + var upperActive = ExecHalfBallot(1); + var lowerValue = VectorSourceOfHalf(instruction, 0, 0); + var upperValue = VectorSourceOfHalf(instruction, 0, 1); + var lowerHasActive = IsNotZero(lowerActive); + var useLower = LogicalOr(lowerHasActive, LogicalNot(IsNotZero(upperActive))); + var lowerLane = _module.AddInstruction( + SpirvOp.Select, + _uintType, + lowerHasActive, + Ext(73, _uintType, lowerActive), + UInt(0)); + var lane = BitwiseAnd( + _module.AddInstruction( + SpirvOp.Select, + _uintType, + useLower, + lowerLane, + Ext(73, _uintType, upperActive)), + UInt(PairedHalfLaneCount - 1)); + return _module.AddInstruction( + SpirvOp.Select, + _uintType, + useLower, + SubgroupBroadcast(lowerValue, lane), + SubgroupBroadcast(upperValue, lane)); + } + + private uint PairedReadLane(Gen5ShaderInstruction instruction, uint selectedLane) + { + var lowerValue = VectorSourceOfHalf(instruction, 0, 0); + var upperValue = VectorSourceOfHalf(instruction, 0, 1); + var lane = BitwiseAnd(selectedLane, UInt(PairedHalfLaneCount - 1)); + return _module.AddInstruction( + SpirvOp.Select, + _uintType, + _module.AddInstruction(SpirvOp.ULessThan, _boolType, selectedLane, UInt(PairedHalfLaneCount)), + SubgroupBroadcast(lowerValue, lane), + SubgroupBroadcast(upperValue, lane)); + } + + private uint LogicalOr(uint left, uint right) => + _module.AddInstruction(SpirvOp.LogicalOr, _boolType, left, right); + + private void EmitPairedWaveCounter(uint destination, uint inBounds, Func update) + { + var lowerActive = ExecHalfBallot(0); + var upperActive = ExecHalfBallot(1); + var activeCount = IAdd( + _module.AddInstruction(SpirvOp.BitCount, _uintType, lowerActive), + _module.AddInstruction(SpirvOp.BitCount, _uintType, upperActive)); + var anyActive = IsNotZero(BitwiseOr(lowerActive, upperActive)); + var leaderLane = BitwiseAnd( + _module.AddInstruction( + SpirvOp.Select, + _uintType, + IsNotZero(lowerActive), + Ext(73, _uintType, lowerActive), + Ext(73, _uintType, upperActive)), + UInt(PairedHalfLaneCount - 1)); + if (_pairedCounterResult == 0) + { + _pairedCounterResult = DeclareFunctionUint(UInt(0)); + _module.AddName(_pairedCounterResult, "pairedCounterResult"); + } + + var valid = LogicalAnd(inBounds, anyActive); + var leader = LogicalAnd( + valid, + _module.AddInstruction(SpirvOp.IEqual, _boolType, PairedSubgroupLane(), leaderLane)); + EmitConditional(leader, () => Store(_pairedCounterResult, update(activeCount))); + var original = SubgroupBroadcast(Load(_uintType, _pairedCounterResult), leaderLane); + var result = _module.AddInstruction(SpirvOp.Select, _uintType, valid, original, UInt(0)); + var current = _laneHalf; + for (var half = 0; half < 2; half++) + { + SelectLaneHalf(half); + StoreV(destination, result); + } + + SelectLaneHalf(current); + } + + private sealed class PairedWave64Lowering : IWaveLowering + { + private readonly CompilationContext _context; + + public PairedWave64Lowering(CompilationContext context) + { + _context = context; + } + + public bool RequiresCrossSubgroupSynchronization => false; + + public uint ReserveLdsDwords(uint guestDwordCount) => guestDwordCount; + + public void DeclareResources() { } + public void DeclareFunctionResources() { } + public void BeginBlock() { } + public void BeforeInstruction(Gen5ShaderInstruction instruction) { } + public void EndBlock() { } + + public void InitializeMasks() => + _context.StoreS64(106, _context._module.Constant64(_context._ulongType, 0)); + + public uint BooleanToWaveMask(uint condition) + { + var context = RequireLanePass(nameof(BooleanToWaveMask)); + var wide = context._module.AddInstruction(SpirvOp.UConvert, context._ulongType, context.HalfBallot(condition)); + return context._laneHalf == 0 + ? wide + : context.ShiftLeftLogical64(wide, context._module.Constant64(context._ulongType, 32)); + } + + public uint SubgroupAny(uint condition) + { + var context = RequireLanePass(nameof(SubgroupAny)); + return context._module.AddInstruction(SpirvOp.GroupNonUniformAny, context._boolType, context.UInt(3), condition); + } + + public uint CurrentLaneBit() => + _context.ShiftLeftLogical64( + _context._module.Constant64(_context._ulongType, 1), + _context._module.AddInstruction(SpirvOp.UConvert, _context._ulongType, _context.PairedGuestLane())); + + public uint BroadcastFirstActive(uint value) => + throw new InvalidOperationException("paired wave64 reads the first active lane from both halves"); + + public uint BroadcastLane(uint value, uint lane) => + throw new InvalidOperationException("paired wave64 reads a lane from both halves"); + + public uint BroadcastValue(uint writer, uint value, uint lane) => + throw new InvalidOperationException("paired wave64 broadcasts a wave counter from both halves"); + + public uint WaveMaskAny(uint register, uint laneActive) => + _context.IsNotZero64(_context.LoadS64(register)); + + public uint CurrentExecMask() => _context.LoadS64(126); + + public uint NormalizeExecMask(uint mask) => mask; + + public void StoreWaveMask(uint register, uint condition) + { + var context = RequireLanePass(nameof(StoreWaveMask)); + context.StoreS(register + (uint)context._laneHalf, context.HalfBallot(condition)); + } + + private CompilationContext RequireLanePass(string operation) + { + if (!_context._inLanePass) + { + throw new InvalidOperationException($"paired wave64 {operation} needs a lane half"); + } + + return _context; + } + } + } +} diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Resources.cs b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Resources.cs index 47c3647..163b7cd 100644 --- a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Resources.cs +++ b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Resources.cs @@ -128,6 +128,11 @@ public static partial class Gen5SpirvTranslator _waveLowering = new NativeWave64Lowering(this); _guestLaneFromSubgroup = true; } + else if (_stage == Gen5SpirvStage.Compute && request.HostComputeSubgroup32 && PairedWave64Enabled) + { + _pairedLanes = true; + _waveLowering = new PairedWave64Lowering(this); + } else { _waveLowering = new EmulatedWave64Lowering(this); @@ -1897,6 +1902,19 @@ public static partial class Gen5SpirvTranslator var index = GlobalDataShareIndex(baseAddress, offset); var inBounds = LogicalAnd(IsNotZero(sizeBytes), IsBlockWordInRange(_globalDataShare, index)); var destination = instruction.Destinations[0].Value; + if (_pairedLanes) + { + EmitPairedWaveCounter(destination, inBounds, count => EmitAtomic( + instruction.Opcode == "DsAppend" ? SpirvOp.AtomicIAdd : SpirvOp.AtomicISub, + _uintType, + BlockWordPointer(_globalDataShare, index), + scope: 1, + semantics: 0x48, + value: () => count, + comparator: () => UInt(0))); + return true; + } + var active = Load(_boolType, _exec); var activeMask = BooleanToWaveMask(active); var activeLow = Narrow(activeMask); diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.cs b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.cs index bc27667..2cf2431 100644 --- a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.cs +++ b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.cs @@ -531,12 +531,19 @@ public static partial class Gen5SpirvTranslator physicalSizes[_physicalAxisOfLogical[logical]] = logicalSizes[logical]; } - _module.AddExecutionMode( - main, - SpirvExecutionMode.LocalSize, - physicalSizes[0], - physicalSizes[1], - physicalSizes[2]); + if (_pairedLanes) + { + _module.AddExecutionMode(main, SpirvExecutionMode.LocalSize, PairedHostThreadCount(), 1, 1); + } + else + { + _module.AddExecutionMode( + main, + SpirvExecutionMode.LocalSize, + physicalSizes[0], + physicalSizes[1], + physicalSizes[2]); + } } var attributeCount = _stage == Gen5SpirvStage.Vertex @@ -618,6 +625,7 @@ public static partial class Gen5SpirvTranslator _scc = Flag(false); _vcc = Flag(false); _exec = Flag(true); + DeclarePairedLaneFlags(_vcc, _exec, Flag); // Only the title-shader trace reads it; elsewhere it would be a write-only Private // global, and AMD's driver crashes linking stages that carry an unused one. if (TraceTitleShaderState) @@ -695,6 +703,7 @@ public static partial class Gen5SpirvTranslator _module.ConstantNull(arrayType)); _interfaces.Add(_scratch); _module.AddName(_scratch, "scratch"); + DeclarePairedScratch(arrayPointer, arrayType); } private uint ComputeLdsGuestDwordCount() @@ -827,6 +836,8 @@ public static partial class Gen5SpirvTranslator (uint)SpirvBuiltIn.LocalInvocationIndex); _interfaces.Add(_localInvocationIndexInput); } + + DeclarePairedLaneInputs(); } if (_stage == Gen5SpirvStage.Vertex) @@ -1199,6 +1210,10 @@ public static partial class Gen5SpirvTranslator Store(output.Variable, _module.ConstantNull(output.Type)); } } + else if (_pairedLanes) + { + EmitPairedComputeInitialState(); + } else { var localId = Load(_uvec3Type, _localInvocationIdInput); @@ -1405,7 +1420,9 @@ public static partial class Gen5SpirvTranslator } _execKnownFull = IsExecKnownFull(instruction.Pc); - var emitted = TryEmitInstruction(instruction, out error); + var emitted = _pairedLanes && IsPairedLaneInstruction(instruction) + ? TryEmitPairedLaneInstruction(instruction, out error) + : TryEmitInstruction(instruction, out error); _execKnownFull = false; if (!emitted) { @@ -2481,6 +2498,19 @@ public static partial class Gen5SpirvTranslator sizeBytes); var pointer = LdsPointer(baseAddress, offset); var destination = instruction.Destinations[0].Value; + if (_pairedLanes) + { + EmitPairedWaveCounter(destination, inBounds, count => EmitAtomic( + instruction.Opcode == "DsAppend" ? SpirvOp.AtomicIAdd : SpirvOp.AtomicISub, + _uintType, + pointer, + scope: 2, + semantics: 0x108, + value: () => count, + comparator: () => UInt(0))); + return true; + } + var active = Load(_boolType, _exec); var activeMask = BooleanToWaveMask(active); @@ -7376,7 +7406,7 @@ public static partial class Gen5SpirvTranslator private List<(uint Register, uint Lane)> FindLaneSpillSlots() { var slots = new List<(uint Register, uint Lane)>(); - if (_stage == Gen5SpirvStage.Compute && _waveLowering is EmulatedWave64Lowering) + if (_stage == Gen5SpirvStage.Compute && (_waveLowering is EmulatedWave64Lowering || _pairedLanes)) { return slots; } @@ -7510,7 +7540,7 @@ public static partial class Gen5SpirvTranslator throw new InvalidOperationException($"Vector register v{register} is outside the register file."); } - return RegisterVariable(_vectorRegisterVariables, register, "v"); + return RegisterVariable(CurrentVectorRegisterVariables(), register, CurrentVectorRegisterPrefix()); } // The V_MOVREL* opcodes address the VGPR file with a register number that @@ -7621,7 +7651,10 @@ public static partial class Gen5SpirvTranslator return _packedHalfRegisters; } - private uint LoadS(uint register) => Load(_uintType, ScalarPointer(register)); + private uint LoadS(uint register) => + _scalarSnapshot is { } snapshot && snapshot.TryGetValue(register, out var value) + ? value + : Load(_uintType, ScalarPointer(register)); private uint LoadV(uint register) => Load(_uintType, VectorPointer(register)); @@ -7632,14 +7665,14 @@ public static partial class Gen5SpirvTranslator { if (_waveLaneCount != 32 || register == 106) { - Store(_vcc, IsLaneSetInMaskRegisters(106)); + UpdateLaneMaskFlag(106); } } else if (register is 126 or 127) { if (_waveLaneCount != 32 || register == 126) { - Store(_exec, IsLaneSetInMaskRegisters(126)); + UpdateLaneMaskFlag(126); } } } @@ -7648,15 +7681,21 @@ public static partial class Gen5SpirvTranslator // so the lane's bit is tested on one dword instead of a composed 64-bit mask. private uint IsLaneSetInMaskRegisters(uint lowRegister) { + var low = Load(_uintType, ScalarPointer(lowRegister)); if (_waveLaneCount != 32) { - return IsWaveMaskActive(LoadS64(lowRegister)); + var high = Load(_uintType, ScalarPointer(lowRegister + 1)); + return IsWaveMaskActive(BitwiseOr64( + _module.AddInstruction(SpirvOp.UConvert, _ulongType, low), + ShiftLeftLogical64( + _module.AddInstruction(SpirvOp.UConvert, _ulongType, high), + _module.Constant64(_ulongType, 32)))); } var laneBit = _subgroupInvocationIdInput == 0 ? UInt(1) : ShiftLeftLogical(UInt(1), GuestWaveLane()); - return IsNotZero(BitwiseAnd(LoadS(lowRegister), laneBit)); + return IsNotZero(BitwiseAnd(low, laneBit)); } // SHARPEMU_EXEC_GUARD_ELISION=0 guards every vector register write with EXEC again. @@ -7671,7 +7710,7 @@ public static partial class Gen5SpirvTranslator private bool IsExecKnownFull(uint pc) { - if (!ExecGuardElision || !_request.EnableExecGuardElision) + if (!ExecGuardElision || !_request.EnableExecGuardElision || _pairedLanes) { return false; } @@ -7878,6 +7917,11 @@ public static partial class Gen5SpirvTranslator private uint GuestWaveLane() { + if (_pairedLanes) + { + return PairedGuestLane(); + } + // A native wave64 is one host subgroup: its lane is the subgroup lane, the same one // ballot and shuffle address, whatever order the driver gave the invocations. if (_waveLaneCount == 64 && _guestLaneFromSubgroup && _subgroupInvocationIdInput != 0) @@ -7934,7 +7978,14 @@ public static partial class Gen5SpirvTranslator private uint ShuffleLane(uint value, uint lane) => _subgroupInvocationIdInput == 0 ? value - : _module.AddInstruction(SpirvOp.GroupNonUniformShuffle, _uintType, UInt(3), value, lane); + : _module.AddInstruction( + SpirvOp.GroupNonUniformShuffle, + _uintType, + UInt(3), + value, + _pairedLanes + ? BitwiseAnd(lane, UInt(PairedHalfLaneCount - 1)) + : lane); private uint CurrentLaneBit() => _waveLowering.CurrentLaneBit(); @@ -8057,6 +8108,7 @@ public static partial class Gen5SpirvTranslator instruction.Destinations.Any(IsWaveMaskOperand)); private bool UsesSubgroupOperations() => + _pairedLanes || _enableGraphicsSubgroupOperations && (UsesSubgroupShuffle() || UsesSubgroupBroadcast() ||