[ShaderCompiler] Run wave64 compute as paired lanes on 32-lane subgroups
This commit is contained in:
ParantezTech
parent
4087ab84f8
commit
7a6e3d0308
|
|
@ -51,8 +51,9 @@ public static partial class Gen5SpirvTranslator
|
|||
return false;
|
||||
}
|
||||
|
||||
var value = GetRawSource(instruction, 0);
|
||||
value = BroadcastFirstWaveActive(value);
|
||||
var value = _pairedLanes
|
||||
? PairedReadFirstLane(instruction)
|
||||
: BroadcastFirstWaveActive(GetRawSource(instruction, 0));
|
||||
|
||||
StoreS(instruction.Destinations[0].Value, value);
|
||||
return true;
|
||||
|
|
@ -2741,7 +2742,7 @@ public static partial class Gen5SpirvTranslator
|
|||
|
||||
if (instruction.Opcode.EndsWith("SaveexecB32", StringComparison.Ordinal))
|
||||
{
|
||||
var oldExec64 = BooleanToWaveMask(Load(_boolType, _exec));
|
||||
var oldExec64 = CurrentExecMask();
|
||||
var oldExec = _module.AddInstruction(
|
||||
SpirvOp.UConvert,
|
||||
_uintType,
|
||||
|
|
@ -4348,6 +4349,12 @@ public static partial class Gen5SpirvTranslator
|
|||
|
||||
private void StoreS64(uint register, uint value)
|
||||
{
|
||||
if (_pairedLanes && _inLanePass)
|
||||
{
|
||||
StorePairedLaneMaskWord(register, value);
|
||||
return;
|
||||
}
|
||||
|
||||
StoreS(
|
||||
register,
|
||||
_module.AddInstruction(SpirvOp.UConvert, _uintType, value));
|
||||
|
|
@ -4950,8 +4957,14 @@ public static partial class Gen5SpirvTranslator
|
|||
}
|
||||
|
||||
var destination = instruction.Destinations[0].Value;
|
||||
var sourceValue = GetRawSource(instruction, 0);
|
||||
var selectedLane = BitwiseAnd(GetRawSource(instruction, 1), UInt(LaneSelectMask));
|
||||
if (_pairedLanes)
|
||||
{
|
||||
StoreS(destination, PairedReadLane(instruction, selectedLane));
|
||||
return true;
|
||||
}
|
||||
|
||||
var sourceValue = GetRawSource(instruction, 0);
|
||||
|
||||
if (_stage == Gen5SpirvStage.Compute && _waveLowering is EmulatedWave64Lowering)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -0,0 +1,464 @@
|
|||
// Copyright (C) 2026 SharpEmu Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
namespace SharpEmu.ShaderCompiler.Vulkan;
|
||||
|
||||
public static partial class Gen5SpirvTranslator
|
||||
{
|
||||
private sealed partial class CompilationContext
|
||||
{
|
||||
private const uint PairedHalfLaneCount = 32;
|
||||
|
||||
private static readonly bool PairedWave64Enabled = !string.Equals(
|
||||
Environment.GetEnvironmentVariable("SHARPEMU_WAVE64_PAIRED"), "0", StringComparison.Ordinal);
|
||||
|
||||
private readonly bool _pairedLanes;
|
||||
private int _laneHalf;
|
||||
private bool _inLanePass;
|
||||
private readonly Dictionary<uint, uint> _upperVectorRegisterVariables = new();
|
||||
private readonly uint[] _packedHalfRegistersByHalf = new uint[2];
|
||||
private uint _lowerExec;
|
||||
private uint _upperExec;
|
||||
private uint _lowerVcc;
|
||||
private uint _upperVcc;
|
||||
private uint _lowerScratch;
|
||||
private uint _upperScratch;
|
||||
private uint _subgroupIdInput;
|
||||
private uint _pairedCounterResult;
|
||||
private Dictionary<uint, uint>? _scalarSnapshot;
|
||||
|
||||
private void SelectLaneHalf(int half)
|
||||
{
|
||||
if (!_pairedLanes)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_packedHalfRegistersByHalf[_laneHalf] = _packedHalfRegisters;
|
||||
_laneHalf = half;
|
||||
_packedHalfRegisters = _packedHalfRegistersByHalf[half];
|
||||
_exec = half == 0 ? _lowerExec : _upperExec;
|
||||
_vcc = half == 0 ? _lowerVcc : _upperVcc;
|
||||
if (_lowerScratch != 0)
|
||||
{
|
||||
_scratch = half == 0 ? _lowerScratch : _upperScratch;
|
||||
}
|
||||
}
|
||||
|
||||
private void DeclarePairedLaneFlags(uint lowerVcc, uint lowerExec, Func<bool, uint> flag)
|
||||
{
|
||||
_lowerVcc = lowerVcc;
|
||||
_lowerExec = lowerExec;
|
||||
if (!_pairedLanes)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_upperVcc = flag(false);
|
||||
_upperExec = flag(true);
|
||||
_interfaces.Add(_upperVcc);
|
||||
_interfaces.Add(_upperExec);
|
||||
_module.AddName(_upperVcc, "vccUpperHalf");
|
||||
_module.AddName(_upperExec, "execUpperHalf");
|
||||
}
|
||||
|
||||
private void DeclarePairedScratch(uint arrayPointer, uint arrayType)
|
||||
{
|
||||
_lowerScratch = _scratch;
|
||||
if (!_pairedLanes)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_upperScratch = _module.AddGlobalVariable(
|
||||
arrayPointer,
|
||||
SpirvStorageClass.Private,
|
||||
_module.ConstantNull(arrayType));
|
||||
_interfaces.Add(_upperScratch);
|
||||
_module.AddName(_upperScratch, "scratchUpperHalf");
|
||||
}
|
||||
|
||||
private void DeclarePairedLaneInputs()
|
||||
{
|
||||
if (!_pairedLanes)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_subgroupIdInput = _module.AddGlobalVariable(
|
||||
_module.TypePointer(SpirvStorageClass.Input, _uintType),
|
||||
SpirvStorageClass.Input);
|
||||
_module.AddDecoration(_subgroupIdInput, SpirvDecoration.BuiltIn, (uint)SpirvBuiltIn.SubgroupId);
|
||||
_interfaces.Add(_subgroupIdInput);
|
||||
}
|
||||
|
||||
private uint PairedHostThreadCount()
|
||||
{
|
||||
var guestThreads = checked(_localSizeX * _localSizeY * _localSizeZ);
|
||||
return (guestThreads + 63) / 64 * PairedHalfLaneCount;
|
||||
}
|
||||
|
||||
private Dictionary<uint, uint> CurrentVectorRegisterVariables() =>
|
||||
_pairedLanes && _laneHalf == 1 ? _upperVectorRegisterVariables : _vectorRegisterVariables;
|
||||
|
||||
private string CurrentVectorRegisterPrefix() =>
|
||||
_pairedLanes && _laneHalf == 1 ? "vu" : "v";
|
||||
|
||||
private uint PairedSubgroupLane() => Load(_uintType, _subgroupInvocationIdInput);
|
||||
|
||||
private uint PairedGuestLane() =>
|
||||
_laneHalf == 0 ? PairedSubgroupLane() : IAdd(PairedSubgroupLane(), UInt(PairedHalfLaneCount));
|
||||
|
||||
private uint HalfBallot(uint condition) =>
|
||||
_module.AddInstruction(
|
||||
SpirvOp.CompositeExtract,
|
||||
_uintType,
|
||||
_module.AddInstruction(SpirvOp.GroupNonUniformBallot, _uvec4Type, UInt(3), condition),
|
||||
0);
|
||||
|
||||
private uint SubgroupBroadcast(uint value, uint lane) =>
|
||||
_module.AddInstruction(SpirvOp.GroupNonUniformBroadcast, _uintType, UInt(3), value, lane);
|
||||
|
||||
private uint ExecHalfBallot(int half)
|
||||
{
|
||||
var current = _laneHalf;
|
||||
SelectLaneHalf(half);
|
||||
var ballot = HalfBallot(Load(_boolType, _exec));
|
||||
SelectLaneHalf(current);
|
||||
return ballot;
|
||||
}
|
||||
|
||||
private uint VectorSourceOfHalf(Gen5ShaderInstruction instruction, int sourceIndex, int half)
|
||||
{
|
||||
var current = _laneHalf;
|
||||
SelectLaneHalf(half);
|
||||
var value = GetRawSource(instruction, sourceIndex);
|
||||
SelectLaneHalf(current);
|
||||
return value;
|
||||
}
|
||||
|
||||
private static bool IsPairedLaneInstruction(Gen5ShaderInstruction instruction) =>
|
||||
instruction.Opcode is not ("VReadlaneB32" or "VReadfirstlaneB32" or "DsAppend" or "DsConsume") &&
|
||||
instruction.Encoding is
|
||||
Gen5ShaderEncoding.Mubuf or
|
||||
Gen5ShaderEncoding.Mtbuf or
|
||||
Gen5ShaderEncoding.Vop1 or
|
||||
Gen5ShaderEncoding.Vop2 or
|
||||
Gen5ShaderEncoding.Vopc or
|
||||
Gen5ShaderEncoding.Vop3 or
|
||||
Gen5ShaderEncoding.Vop3p or
|
||||
Gen5ShaderEncoding.Vintrp or
|
||||
Gen5ShaderEncoding.Ds or
|
||||
Gen5ShaderEncoding.Flat or
|
||||
Gen5ShaderEncoding.Mimg or
|
||||
Gen5ShaderEncoding.Exp;
|
||||
|
||||
private bool TryEmitPairedLaneInstruction(Gen5ShaderInstruction instruction, out string error)
|
||||
{
|
||||
_scalarSnapshot = SnapshotScalarSources(instruction);
|
||||
_inLanePass = true;
|
||||
try
|
||||
{
|
||||
for (var half = 0; half < 2; half++)
|
||||
{
|
||||
SelectLaneHalf(half);
|
||||
if (!TryEmitInstruction(instruction, out error))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
error = string.Empty;
|
||||
return true;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_inLanePass = false;
|
||||
_scalarSnapshot = null;
|
||||
SelectLaneHalf(0);
|
||||
}
|
||||
}
|
||||
|
||||
private Dictionary<uint, uint>? SnapshotScalarSources(Gen5ShaderInstruction instruction)
|
||||
{
|
||||
Dictionary<uint, uint>? snapshot = null;
|
||||
foreach (var operand in instruction.Sources)
|
||||
{
|
||||
if (operand.Kind != Gen5OperandKind.ScalarRegister)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
for (var register = operand.Value; register <= operand.Value + 1 && register < ScalarRegisterCount; register++)
|
||||
{
|
||||
if (register == 125)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
snapshot ??= new Dictionary<uint, uint>();
|
||||
if (!snapshot.ContainsKey(register))
|
||||
{
|
||||
snapshot[register] = Load(_uintType, ScalarPointer(register));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
private void UpdateLaneMaskFlag(uint lowRegister)
|
||||
{
|
||||
if (!_pairedLanes || _inLanePass)
|
||||
{
|
||||
Store(lowRegister == 106 ? _vcc : _exec, IsLaneSetInMaskRegisters(lowRegister));
|
||||
return;
|
||||
}
|
||||
|
||||
var current = _laneHalf;
|
||||
for (var half = 0; half < 2; half++)
|
||||
{
|
||||
SelectLaneHalf(half);
|
||||
Store(lowRegister == 106 ? _vcc : _exec, IsLaneSetInMaskRegisters(lowRegister));
|
||||
}
|
||||
|
||||
SelectLaneHalf(current);
|
||||
}
|
||||
|
||||
private void StorePairedLaneMaskWord(uint register, uint value)
|
||||
{
|
||||
var word = _laneHalf == 0
|
||||
? Narrow(value)
|
||||
: Narrow(ShiftRightLogical64(value, _module.Constant64(_ulongType, 32)));
|
||||
StoreS(register + (uint)_laneHalf, word);
|
||||
}
|
||||
|
||||
private void EmitPairedComputeInitialState()
|
||||
{
|
||||
var workGroupId = Load(_uvec3Type, _workGroupIdInput);
|
||||
var waveBase = _module.AddInstruction(
|
||||
SpirvOp.IMul,
|
||||
_uintType,
|
||||
Load(_uintType, _subgroupIdInput),
|
||||
UInt(2 * PairedHalfLaneCount));
|
||||
var guestThreads = checked(_localSizeX * _localSizeY * _localSizeZ);
|
||||
var limits = new uint[3];
|
||||
var groups = new uint[3];
|
||||
for (uint component = 0; component < 3; component++)
|
||||
{
|
||||
limits[component] = ComputeThreadLimit(component);
|
||||
groups[component] = _module.AddInstruction(
|
||||
SpirvOp.CompositeExtract,
|
||||
_uintType,
|
||||
workGroupId,
|
||||
(uint)_physicalAxisOfLogical[component]);
|
||||
}
|
||||
|
||||
var active = new uint[2];
|
||||
for (var half = 0; half < 2; half++)
|
||||
{
|
||||
SelectLaneHalf(half);
|
||||
var index = IAdd(waveBase, PairedGuestLane());
|
||||
var inBounds = _module.AddInstruction(SpirvOp.ULessThan, _boolType, index, UInt(guestThreads));
|
||||
var row = _module.AddInstruction(SpirvOp.UDiv, _uintType, index, UInt(_localSizeX));
|
||||
var local = new[]
|
||||
{
|
||||
_module.AddInstruction(SpirvOp.UMod, _uintType, index, UInt(_localSizeX)),
|
||||
_module.AddInstruction(SpirvOp.UMod, _uintType, row, UInt(_localSizeY)),
|
||||
_module.AddInstruction(SpirvOp.UDiv, _uintType, row, UInt(_localSizeY)),
|
||||
};
|
||||
var sizes = new[] { _localSizeX, _localSizeY, _localSizeZ };
|
||||
for (uint component = 0; component < 3; component++)
|
||||
{
|
||||
StoreV(component, local[component], guardWithExec: false);
|
||||
var global = IAdd(
|
||||
_module.AddInstruction(SpirvOp.IMul, _uintType, groups[component], UInt(sizes[component])),
|
||||
local[component]);
|
||||
inBounds = LogicalAnd(
|
||||
inBounds,
|
||||
_module.AddInstruction(SpirvOp.ULessThan, _boolType, global, limits[component]));
|
||||
}
|
||||
|
||||
active[half] = inBounds;
|
||||
}
|
||||
|
||||
SelectLaneHalf(0);
|
||||
var lowerActive = HalfBallot(active[0]);
|
||||
var upperActive = HalfBallot(active[1]);
|
||||
StoreS(126, lowerActive);
|
||||
StoreS(127, upperActive);
|
||||
Store(_programActive, _module.AddInstruction(SpirvOp.LogicalOr, _boolType, active[0], active[1]));
|
||||
|
||||
if (_request.ComputeSystemRegisters is { } registers)
|
||||
{
|
||||
StoreComputeSystemRegister(registers.WorkGroupXRegister, workGroupId, (uint)_physicalAxisOfLogical[0]);
|
||||
StoreComputeSystemRegister(registers.WorkGroupYRegister, workGroupId, (uint)_physicalAxisOfLogical[1]);
|
||||
StoreComputeSystemRegister(registers.WorkGroupZRegister, workGroupId, (uint)_physicalAxisOfLogical[2]);
|
||||
if (registers.ThreadGroupSizeRegister is { } sizeRegister)
|
||||
{
|
||||
StoreS(sizeRegister, UInt(guestThreads));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private uint PairedReadFirstLane(Gen5ShaderInstruction instruction)
|
||||
{
|
||||
var lowerActive = ExecHalfBallot(0);
|
||||
var upperActive = ExecHalfBallot(1);
|
||||
var lowerValue = VectorSourceOfHalf(instruction, 0, 0);
|
||||
var upperValue = VectorSourceOfHalf(instruction, 0, 1);
|
||||
var lowerHasActive = IsNotZero(lowerActive);
|
||||
var useLower = LogicalOr(lowerHasActive, LogicalNot(IsNotZero(upperActive)));
|
||||
var lowerLane = _module.AddInstruction(
|
||||
SpirvOp.Select,
|
||||
_uintType,
|
||||
lowerHasActive,
|
||||
Ext(73, _uintType, lowerActive),
|
||||
UInt(0));
|
||||
var lane = BitwiseAnd(
|
||||
_module.AddInstruction(
|
||||
SpirvOp.Select,
|
||||
_uintType,
|
||||
useLower,
|
||||
lowerLane,
|
||||
Ext(73, _uintType, upperActive)),
|
||||
UInt(PairedHalfLaneCount - 1));
|
||||
return _module.AddInstruction(
|
||||
SpirvOp.Select,
|
||||
_uintType,
|
||||
useLower,
|
||||
SubgroupBroadcast(lowerValue, lane),
|
||||
SubgroupBroadcast(upperValue, lane));
|
||||
}
|
||||
|
||||
private uint PairedReadLane(Gen5ShaderInstruction instruction, uint selectedLane)
|
||||
{
|
||||
var lowerValue = VectorSourceOfHalf(instruction, 0, 0);
|
||||
var upperValue = VectorSourceOfHalf(instruction, 0, 1);
|
||||
var lane = BitwiseAnd(selectedLane, UInt(PairedHalfLaneCount - 1));
|
||||
return _module.AddInstruction(
|
||||
SpirvOp.Select,
|
||||
_uintType,
|
||||
_module.AddInstruction(SpirvOp.ULessThan, _boolType, selectedLane, UInt(PairedHalfLaneCount)),
|
||||
SubgroupBroadcast(lowerValue, lane),
|
||||
SubgroupBroadcast(upperValue, lane));
|
||||
}
|
||||
|
||||
private uint LogicalOr(uint left, uint right) =>
|
||||
_module.AddInstruction(SpirvOp.LogicalOr, _boolType, left, right);
|
||||
|
||||
private void EmitPairedWaveCounter(uint destination, uint inBounds, Func<uint, uint> update)
|
||||
{
|
||||
var lowerActive = ExecHalfBallot(0);
|
||||
var upperActive = ExecHalfBallot(1);
|
||||
var activeCount = IAdd(
|
||||
_module.AddInstruction(SpirvOp.BitCount, _uintType, lowerActive),
|
||||
_module.AddInstruction(SpirvOp.BitCount, _uintType, upperActive));
|
||||
var anyActive = IsNotZero(BitwiseOr(lowerActive, upperActive));
|
||||
var leaderLane = BitwiseAnd(
|
||||
_module.AddInstruction(
|
||||
SpirvOp.Select,
|
||||
_uintType,
|
||||
IsNotZero(lowerActive),
|
||||
Ext(73, _uintType, lowerActive),
|
||||
Ext(73, _uintType, upperActive)),
|
||||
UInt(PairedHalfLaneCount - 1));
|
||||
if (_pairedCounterResult == 0)
|
||||
{
|
||||
_pairedCounterResult = DeclareFunctionUint(UInt(0));
|
||||
_module.AddName(_pairedCounterResult, "pairedCounterResult");
|
||||
}
|
||||
|
||||
var valid = LogicalAnd(inBounds, anyActive);
|
||||
var leader = LogicalAnd(
|
||||
valid,
|
||||
_module.AddInstruction(SpirvOp.IEqual, _boolType, PairedSubgroupLane(), leaderLane));
|
||||
EmitConditional(leader, () => Store(_pairedCounterResult, update(activeCount)));
|
||||
var original = SubgroupBroadcast(Load(_uintType, _pairedCounterResult), leaderLane);
|
||||
var result = _module.AddInstruction(SpirvOp.Select, _uintType, valid, original, UInt(0));
|
||||
var current = _laneHalf;
|
||||
for (var half = 0; half < 2; half++)
|
||||
{
|
||||
SelectLaneHalf(half);
|
||||
StoreV(destination, result);
|
||||
}
|
||||
|
||||
SelectLaneHalf(current);
|
||||
}
|
||||
|
||||
private sealed class PairedWave64Lowering : IWaveLowering
|
||||
{
|
||||
private readonly CompilationContext _context;
|
||||
|
||||
public PairedWave64Lowering(CompilationContext context)
|
||||
{
|
||||
_context = context;
|
||||
}
|
||||
|
||||
public bool RequiresCrossSubgroupSynchronization => false;
|
||||
|
||||
public uint ReserveLdsDwords(uint guestDwordCount) => guestDwordCount;
|
||||
|
||||
public void DeclareResources() { }
|
||||
public void DeclareFunctionResources() { }
|
||||
public void BeginBlock() { }
|
||||
public void BeforeInstruction(Gen5ShaderInstruction instruction) { }
|
||||
public void EndBlock() { }
|
||||
|
||||
public void InitializeMasks() =>
|
||||
_context.StoreS64(106, _context._module.Constant64(_context._ulongType, 0));
|
||||
|
||||
public uint BooleanToWaveMask(uint condition)
|
||||
{
|
||||
var context = RequireLanePass(nameof(BooleanToWaveMask));
|
||||
var wide = context._module.AddInstruction(SpirvOp.UConvert, context._ulongType, context.HalfBallot(condition));
|
||||
return context._laneHalf == 0
|
||||
? wide
|
||||
: context.ShiftLeftLogical64(wide, context._module.Constant64(context._ulongType, 32));
|
||||
}
|
||||
|
||||
public uint SubgroupAny(uint condition)
|
||||
{
|
||||
var context = RequireLanePass(nameof(SubgroupAny));
|
||||
return context._module.AddInstruction(SpirvOp.GroupNonUniformAny, context._boolType, context.UInt(3), condition);
|
||||
}
|
||||
|
||||
public uint CurrentLaneBit() =>
|
||||
_context.ShiftLeftLogical64(
|
||||
_context._module.Constant64(_context._ulongType, 1),
|
||||
_context._module.AddInstruction(SpirvOp.UConvert, _context._ulongType, _context.PairedGuestLane()));
|
||||
|
||||
public uint BroadcastFirstActive(uint value) =>
|
||||
throw new InvalidOperationException("paired wave64 reads the first active lane from both halves");
|
||||
|
||||
public uint BroadcastLane(uint value, uint lane) =>
|
||||
throw new InvalidOperationException("paired wave64 reads a lane from both halves");
|
||||
|
||||
public uint BroadcastValue(uint writer, uint value, uint lane) =>
|
||||
throw new InvalidOperationException("paired wave64 broadcasts a wave counter from both halves");
|
||||
|
||||
public uint WaveMaskAny(uint register, uint laneActive) =>
|
||||
_context.IsNotZero64(_context.LoadS64(register));
|
||||
|
||||
public uint CurrentExecMask() => _context.LoadS64(126);
|
||||
|
||||
public uint NormalizeExecMask(uint mask) => mask;
|
||||
|
||||
public void StoreWaveMask(uint register, uint condition)
|
||||
{
|
||||
var context = RequireLanePass(nameof(StoreWaveMask));
|
||||
context.StoreS(register + (uint)context._laneHalf, context.HalfBallot(condition));
|
||||
}
|
||||
|
||||
private CompilationContext RequireLanePass(string operation)
|
||||
{
|
||||
if (!_context._inLanePass)
|
||||
{
|
||||
throw new InvalidOperationException($"paired wave64 {operation} needs a lane half");
|
||||
}
|
||||
|
||||
return _context;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -128,6 +128,11 @@ public static partial class Gen5SpirvTranslator
|
|||
_waveLowering = new NativeWave64Lowering(this);
|
||||
_guestLaneFromSubgroup = true;
|
||||
}
|
||||
else if (_stage == Gen5SpirvStage.Compute && request.HostComputeSubgroup32 && PairedWave64Enabled)
|
||||
{
|
||||
_pairedLanes = true;
|
||||
_waveLowering = new PairedWave64Lowering(this);
|
||||
}
|
||||
else
|
||||
{
|
||||
_waveLowering = new EmulatedWave64Lowering(this);
|
||||
|
|
@ -1897,6 +1902,19 @@ public static partial class Gen5SpirvTranslator
|
|||
var index = GlobalDataShareIndex(baseAddress, offset);
|
||||
var inBounds = LogicalAnd(IsNotZero(sizeBytes), IsBlockWordInRange(_globalDataShare, index));
|
||||
var destination = instruction.Destinations[0].Value;
|
||||
if (_pairedLanes)
|
||||
{
|
||||
EmitPairedWaveCounter(destination, inBounds, count => EmitAtomic(
|
||||
instruction.Opcode == "DsAppend" ? SpirvOp.AtomicIAdd : SpirvOp.AtomicISub,
|
||||
_uintType,
|
||||
BlockWordPointer(_globalDataShare, index),
|
||||
scope: 1,
|
||||
semantics: 0x48,
|
||||
value: () => count,
|
||||
comparator: () => UInt(0)));
|
||||
return true;
|
||||
}
|
||||
|
||||
var active = Load(_boolType, _exec);
|
||||
var activeMask = BooleanToWaveMask(active);
|
||||
var activeLow = Narrow(activeMask);
|
||||
|
|
|
|||
|
|
@ -531,12 +531,19 @@ public static partial class Gen5SpirvTranslator
|
|||
physicalSizes[_physicalAxisOfLogical[logical]] = logicalSizes[logical];
|
||||
}
|
||||
|
||||
_module.AddExecutionMode(
|
||||
main,
|
||||
SpirvExecutionMode.LocalSize,
|
||||
physicalSizes[0],
|
||||
physicalSizes[1],
|
||||
physicalSizes[2]);
|
||||
if (_pairedLanes)
|
||||
{
|
||||
_module.AddExecutionMode(main, SpirvExecutionMode.LocalSize, PairedHostThreadCount(), 1, 1);
|
||||
}
|
||||
else
|
||||
{
|
||||
_module.AddExecutionMode(
|
||||
main,
|
||||
SpirvExecutionMode.LocalSize,
|
||||
physicalSizes[0],
|
||||
physicalSizes[1],
|
||||
physicalSizes[2]);
|
||||
}
|
||||
}
|
||||
|
||||
var attributeCount = _stage == Gen5SpirvStage.Vertex
|
||||
|
|
@ -618,6 +625,7 @@ public static partial class Gen5SpirvTranslator
|
|||
_scc = Flag(false);
|
||||
_vcc = Flag(false);
|
||||
_exec = Flag(true);
|
||||
DeclarePairedLaneFlags(_vcc, _exec, Flag);
|
||||
// Only the title-shader trace reads it; elsewhere it would be a write-only Private
|
||||
// global, and AMD's driver crashes linking stages that carry an unused one.
|
||||
if (TraceTitleShaderState)
|
||||
|
|
@ -695,6 +703,7 @@ public static partial class Gen5SpirvTranslator
|
|||
_module.ConstantNull(arrayType));
|
||||
_interfaces.Add(_scratch);
|
||||
_module.AddName(_scratch, "scratch");
|
||||
DeclarePairedScratch(arrayPointer, arrayType);
|
||||
}
|
||||
|
||||
private uint ComputeLdsGuestDwordCount()
|
||||
|
|
@ -827,6 +836,8 @@ public static partial class Gen5SpirvTranslator
|
|||
(uint)SpirvBuiltIn.LocalInvocationIndex);
|
||||
_interfaces.Add(_localInvocationIndexInput);
|
||||
}
|
||||
|
||||
DeclarePairedLaneInputs();
|
||||
}
|
||||
|
||||
if (_stage == Gen5SpirvStage.Vertex)
|
||||
|
|
@ -1199,6 +1210,10 @@ public static partial class Gen5SpirvTranslator
|
|||
Store(output.Variable, _module.ConstantNull(output.Type));
|
||||
}
|
||||
}
|
||||
else if (_pairedLanes)
|
||||
{
|
||||
EmitPairedComputeInitialState();
|
||||
}
|
||||
else
|
||||
{
|
||||
var localId = Load(_uvec3Type, _localInvocationIdInput);
|
||||
|
|
@ -1405,7 +1420,9 @@ public static partial class Gen5SpirvTranslator
|
|||
}
|
||||
|
||||
_execKnownFull = IsExecKnownFull(instruction.Pc);
|
||||
var emitted = TryEmitInstruction(instruction, out error);
|
||||
var emitted = _pairedLanes && IsPairedLaneInstruction(instruction)
|
||||
? TryEmitPairedLaneInstruction(instruction, out error)
|
||||
: TryEmitInstruction(instruction, out error);
|
||||
_execKnownFull = false;
|
||||
if (!emitted)
|
||||
{
|
||||
|
|
@ -2481,6 +2498,19 @@ public static partial class Gen5SpirvTranslator
|
|||
sizeBytes);
|
||||
var pointer = LdsPointer(baseAddress, offset);
|
||||
var destination = instruction.Destinations[0].Value;
|
||||
if (_pairedLanes)
|
||||
{
|
||||
EmitPairedWaveCounter(destination, inBounds, count => EmitAtomic(
|
||||
instruction.Opcode == "DsAppend" ? SpirvOp.AtomicIAdd : SpirvOp.AtomicISub,
|
||||
_uintType,
|
||||
pointer,
|
||||
scope: 2,
|
||||
semantics: 0x108,
|
||||
value: () => count,
|
||||
comparator: () => UInt(0)));
|
||||
return true;
|
||||
}
|
||||
|
||||
var active = Load(_boolType, _exec);
|
||||
|
||||
var activeMask = BooleanToWaveMask(active);
|
||||
|
|
@ -7376,7 +7406,7 @@ public static partial class Gen5SpirvTranslator
|
|||
private List<(uint Register, uint Lane)> FindLaneSpillSlots()
|
||||
{
|
||||
var slots = new List<(uint Register, uint Lane)>();
|
||||
if (_stage == Gen5SpirvStage.Compute && _waveLowering is EmulatedWave64Lowering)
|
||||
if (_stage == Gen5SpirvStage.Compute && (_waveLowering is EmulatedWave64Lowering || _pairedLanes))
|
||||
{
|
||||
return slots;
|
||||
}
|
||||
|
|
@ -7510,7 +7540,7 @@ public static partial class Gen5SpirvTranslator
|
|||
throw new InvalidOperationException($"Vector register v{register} is outside the register file.");
|
||||
}
|
||||
|
||||
return RegisterVariable(_vectorRegisterVariables, register, "v");
|
||||
return RegisterVariable(CurrentVectorRegisterVariables(), register, CurrentVectorRegisterPrefix());
|
||||
}
|
||||
|
||||
// The V_MOVREL* opcodes address the VGPR file with a register number that
|
||||
|
|
@ -7621,7 +7651,10 @@ public static partial class Gen5SpirvTranslator
|
|||
return _packedHalfRegisters;
|
||||
}
|
||||
|
||||
private uint LoadS(uint register) => Load(_uintType, ScalarPointer(register));
|
||||
private uint LoadS(uint register) =>
|
||||
_scalarSnapshot is { } snapshot && snapshot.TryGetValue(register, out var value)
|
||||
? value
|
||||
: Load(_uintType, ScalarPointer(register));
|
||||
|
||||
private uint LoadV(uint register) => Load(_uintType, VectorPointer(register));
|
||||
|
||||
|
|
@ -7632,14 +7665,14 @@ public static partial class Gen5SpirvTranslator
|
|||
{
|
||||
if (_waveLaneCount != 32 || register == 106)
|
||||
{
|
||||
Store(_vcc, IsLaneSetInMaskRegisters(106));
|
||||
UpdateLaneMaskFlag(106);
|
||||
}
|
||||
}
|
||||
else if (register is 126 or 127)
|
||||
{
|
||||
if (_waveLaneCount != 32 || register == 126)
|
||||
{
|
||||
Store(_exec, IsLaneSetInMaskRegisters(126));
|
||||
UpdateLaneMaskFlag(126);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -7648,15 +7681,21 @@ public static partial class Gen5SpirvTranslator
|
|||
// so the lane's bit is tested on one dword instead of a composed 64-bit mask.
|
||||
private uint IsLaneSetInMaskRegisters(uint lowRegister)
|
||||
{
|
||||
var low = Load(_uintType, ScalarPointer(lowRegister));
|
||||
if (_waveLaneCount != 32)
|
||||
{
|
||||
return IsWaveMaskActive(LoadS64(lowRegister));
|
||||
var high = Load(_uintType, ScalarPointer(lowRegister + 1));
|
||||
return IsWaveMaskActive(BitwiseOr64(
|
||||
_module.AddInstruction(SpirvOp.UConvert, _ulongType, low),
|
||||
ShiftLeftLogical64(
|
||||
_module.AddInstruction(SpirvOp.UConvert, _ulongType, high),
|
||||
_module.Constant64(_ulongType, 32))));
|
||||
}
|
||||
|
||||
var laneBit = _subgroupInvocationIdInput == 0
|
||||
? UInt(1)
|
||||
: ShiftLeftLogical(UInt(1), GuestWaveLane());
|
||||
return IsNotZero(BitwiseAnd(LoadS(lowRegister), laneBit));
|
||||
return IsNotZero(BitwiseAnd(low, laneBit));
|
||||
}
|
||||
|
||||
// SHARPEMU_EXEC_GUARD_ELISION=0 guards every vector register write with EXEC again.
|
||||
|
|
@ -7671,7 +7710,7 @@ public static partial class Gen5SpirvTranslator
|
|||
|
||||
private bool IsExecKnownFull(uint pc)
|
||||
{
|
||||
if (!ExecGuardElision || !_request.EnableExecGuardElision)
|
||||
if (!ExecGuardElision || !_request.EnableExecGuardElision || _pairedLanes)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
|
@ -7878,6 +7917,11 @@ public static partial class Gen5SpirvTranslator
|
|||
|
||||
private uint GuestWaveLane()
|
||||
{
|
||||
if (_pairedLanes)
|
||||
{
|
||||
return PairedGuestLane();
|
||||
}
|
||||
|
||||
// A native wave64 is one host subgroup: its lane is the subgroup lane, the same one
|
||||
// ballot and shuffle address, whatever order the driver gave the invocations.
|
||||
if (_waveLaneCount == 64 && _guestLaneFromSubgroup && _subgroupInvocationIdInput != 0)
|
||||
|
|
@ -7934,7 +7978,14 @@ public static partial class Gen5SpirvTranslator
|
|||
private uint ShuffleLane(uint value, uint lane) =>
|
||||
_subgroupInvocationIdInput == 0
|
||||
? value
|
||||
: _module.AddInstruction(SpirvOp.GroupNonUniformShuffle, _uintType, UInt(3), value, lane);
|
||||
: _module.AddInstruction(
|
||||
SpirvOp.GroupNonUniformShuffle,
|
||||
_uintType,
|
||||
UInt(3),
|
||||
value,
|
||||
_pairedLanes
|
||||
? BitwiseAnd(lane, UInt(PairedHalfLaneCount - 1))
|
||||
: lane);
|
||||
|
||||
private uint CurrentLaneBit() =>
|
||||
_waveLowering.CurrentLaneBit();
|
||||
|
|
@ -8057,6 +8108,7 @@ public static partial class Gen5SpirvTranslator
|
|||
instruction.Destinations.Any(IsWaveMaskOperand));
|
||||
|
||||
private bool UsesSubgroupOperations() =>
|
||||
_pairedLanes ||
|
||||
_enableGraphicsSubgroupOperations &&
|
||||
(UsesSubgroupShuffle() ||
|
||||
UsesSubgroupBroadcast() ||
|
||||
|
|
|
|||
Loading…
Reference in New Issue