shifathosn
/
sharpemu
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[SpirvTranslator] Synchronize wave64 half masks through LDS exchange

This commit is contained in:
ParantezTech 2026-10-01 17:19:54 +03:00
parent 6862ad88b8
commit 663cf207f8
4 changed files with 524 additions and 196 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

114
src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Alu.cs
View File

@ -3368,7 +3368,9 @@ public static partial class Gen5SpirvTranslator
var left = GetRawSource64(instruction, 0);
if (instruction.Opcode.EndsWith("SaveexecB64", StringComparison.Ordinal))
{
var oldExec = BooleanToWaveMask(Load(_boolType, _exec));
var oldExec = _emulateWave64 && _subgroupInvocationIdInput != 0
? LoadS64(126)
: BooleanToWaveMask(Load(_boolType, _exec));
var notLeft = _module.AddInstruction(SpirvOp.Not, _ulongType, left);
var newExec = instruction.Opcode switch
{
@ -3741,14 +3743,14 @@ public static partial class Gen5SpirvTranslator
_module.AddInstruction(
SpirvOp.Select,
_uintType,
LogicalNot(SubgroupAny(Load(_boolType, _vcc))),
LogicalNot(WaveMaskAny(106, _vcc)),
UInt(1),
UInt(0)),
Gen5OperandKind.EncodedConstant when operand.Value == 252 =>
_module.AddInstruction(
SpirvOp.Select,
_uintType,
LogicalNot(SubgroupAny(Load(_boolType, _exec))),
LogicalNot(WaveMaskAny(126, _exec)),
UInt(1),
UInt(0)),
Gen5OperandKind.EncodedConstant when operand.Value == 253 =>
@ -4825,76 +4827,48 @@ public static partial class Gen5SpirvTranslator
// An emulated wave64 spans two host subgroups, so a subgroup broadcast cannot reach
// the other half. The selected guest lane publishes through workgroup scratch, like
// BroadcastFirstWave64Active; the trailing barrier keeps the next use from racing.
private uint BroadcastWave64Lane(uint value, uint lane)
{
EmitConditional(
_module.AddInstruction(SpirvOp.IEqual, _boolType, GuestWaveLane(), lane),
() => Store(WaveBroadcastScratchPointer(), value));
EmitWave64Barrier();
var result = Load(_uintType, WaveBroadcastScratchPointer());
EmitWave64Barrier();
return result;
}
// BroadcastFirstWave64Active.
private uint BroadcastWave64Lane(uint value, uint lane) =>
ExchangeWave64Value(_module.AddInstruction(SpirvOp.IEqual, _boolType, GuestWaveLane(), lane), value);
private uint BroadcastFirstWave64Active(uint value)
{
var lane = GuestWaveLane();
EmitConditional(
_module.AddInstruction(
SpirvOp.IEqual,
_boolType,
lane,
UInt(0)),
() => Store(WaveBroadcastScratchPointer(), UInt(0)));
EmitWave64Barrier();
var activeMask = BooleanToWaveMask(Load(_boolType, _exec));
var lowMask = _module.AddInstruction(
SpirvOp.UConvert,
_uintType,
activeMask);
var highMask = _module.AddInstruction(
SpirvOp.UConvert,
_uintType,
ShiftRightLogical64(
activeMask,
_module.Constant64(_ulongType, 32)));
var hasLow = IsNotZero(lowMask);
var hasHigh = IsNotZero(highMask);
var firstLow = Ext(73, _uintType, lowMask);
var firstHigh = IAdd(UInt(32), Ext(73, _uintType, highMask));
var firstLane = _module.AddInstruction(
var upperHalf = ShiftRightLogical(lane, UInt(5));
var activeInHalf = OwnHalfBallot(Load(_boolType, _exec));
var halfHasActive = IsNotZero(activeInHalf);
var firstInHalf = _module.AddInstruction(
SpirvOp.Select,
_uintType,
hasLow,
firstLow,
_module.AddInstruction(
SpirvOp.Select,
_uintType,
hasHigh,
firstHigh,
UInt(0)));
var isFirst = _module.AddInstruction(
SpirvOp.IEqual,
_boolType,
lane,
firstLane);
EmitConditional(
_module.AddInstruction(
SpirvOp.LogicalAnd,
_boolType,
isFirst,
_module.AddInstruction(
SpirvOp.LogicalOr,
_boolType,
hasLow,
hasHigh)),
() => Store(WaveBroadcastScratchPointer(), value));
halfHasActive,
Ext(73, _uintType, activeInHalf),
UInt(0));
var halfBase = BitwiseAnd(Load(_uintType, _subgroupInvocationIdInput), UInt(~31u));
var halfValue = _module.AddInstruction(
SpirvOp.GroupNonUniformBroadcast,
_uintType,
UInt(3),
value,
IAdd(halfBase, firstInHalf));
var exchange = BeginWave64Exchange();
EmitConditional(IsHalfWaveLeader(lane), () =>
{
Store(Wave64ExchangePointer(exchange, upperHalf), halfValue);
Store(
Wave64ExchangePointer(exchange, IAdd(UInt(2), upperHalf)),
_module.AddInstruction(SpirvOp.Select, _uintType, halfHasActive, UInt(1), UInt(0)));
});
EmitWave64Barrier();
var result = Load(_uintType, WaveBroadcastScratchPointer());
EmitWave64Barrier();
return result;
var lowerValue = Load(_uintType, Wave64ExchangePointer(exchange, UInt(0)));
var upperValue = Load(_uintType, Wave64ExchangePointer(exchange, UInt(1)));
var lowerActive = IsNotZero(Load(_uintType, Wave64ExchangePointer(exchange, UInt(2))));
var upperActive = IsNotZero(Load(_uintType, Wave64ExchangePointer(exchange, UInt(3))));
return _module.AddInstruction(
SpirvOp.Select,
_uintType,
_module.AddInstruction(SpirvOp.LogicalAnd, _boolType, LogicalNot(lowerActive), upperActive),
upperValue,
lowerValue);
}
private void StoreCarryOut(
@ -4935,13 +4909,7 @@ public static partial class Gen5SpirvTranslator
{
// The selected guest lane can belong to another host subgroup.
// Read it even when the guest execution mask disables that lane.
var isSelectedLane = _module.AddInstruction(
SpirvOp.IEqual, _boolType, GuestWaveLane(), selectedLane);
EmitConditional(isSelectedLane, () => Store(WaveBroadcastScratchPointer(), sourceValue));
EmitWave64Barrier();
var broadcast = Load(_uintType, WaveBroadcastScratchPointer());
EmitWave64Barrier();
StoreS(destination, broadcast);
StoreS(destination, BroadcastWave64Lane(sourceValue, selectedLane));
}
else if (_subgroupInvocationIdInput != 0)
{

5
src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Resources.cs
View File

@ -1904,10 +1904,7 @@ public static partial class Gen5SpirvTranslator
uint broadcast;
if (_emulateWave64)
{
EmitConditional(isFirstActive, () => Store(WaveBroadcastScratchPointer(), firstValue));
EmitWave64Barrier();
broadcast = Load(_uintType, WaveBroadcastScratchPointer());
EmitWave64Barrier();
broadcast = ExchangeWave64Value(isFirstActive, firstValue);
}
else if (_stage == Gen5SpirvStage.Compute || _enableGraphicsSubgroupOperations)
{

332
src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.cs
View File

@ -17,6 +17,8 @@ public static partial class Gen5SpirvTranslator
// so a smaller array is safe.
private const uint PrivateLdsDwordCount = 2048;
private const uint RdnaWaveLaneCount = 32;
private const uint Wave64ExchangeSlotCount = 4;
private const uint Wave64ExchangeDwordCount = Wave64ExchangeSlotCount * 2;
internal static SpirvImageFormat DecodeStorageImageFormat(
uint dataFormat,
@ -185,12 +187,13 @@ public static partial class Gen5SpirvTranslator
private uint _localInvocationIndexInput;
private uint _workGroupIdInput;
private uint _pushConstantUintPointer;
private uint _subgroupSizeInput;
private uint _subgroupInvocationIdInput;
private uint _waveMaskScratch;
private uint _waveMaskScratchElementPointer;
private uint _waveBroadcastScratch;
private bool _waveScratchInLds;
private uint _wave64Exchange;
private uint _wave64ExchangeElementPointer;
private uint _wave64ExchangeOffset;
private uint _wave64ExchangeParity;
private Ir.Gen5Wave64HalfMaskPlan? _halfMaskPlan;
private uint? _emittingPc;
private uint _glsl;
private enum ImageComponentKind
@ -620,7 +623,7 @@ public static partial class Gen5SpirvTranslator
private void DeclareWave64Scratch()
{
if (!_emulateWave64 || !UsesSubgroupOperations())
if (!UsesWave64Exchange())
{
return;
}
@ -628,36 +631,39 @@ public static partial class Gen5SpirvTranslator
// Metal exposes 32 KiB of threadgroup memory on the Apple GPUs we
// target. Some PS5 compute shaders legitimately request all of it,
// so allocating another workgroup variable for the wave64 bridge
// makes pipeline creation fail. Reuse the final three dwords of the
// makes pipeline creation fail. Reuse the final dwords of the
// existing LDS allocation in that case. The translator already
// bounds guest LDS accesses to this fixed allocation; keeping the
// bridge inside it preserves the host limit and still provides the
// cross-subgroup rendezvous needed to model one 64-lane guest wave.
if (_lds != 0)
{
_waveScratchInLds = true;
_waveMaskScratchElementPointer = _ldsElementPointer;
_wave64Exchange = _lds;
_wave64ExchangeElementPointer = _ldsElementPointer;
_wave64ExchangeOffset = _ldsDwordMask + 1 < LdsDwordCount
? _ldsDwordMask + 1
: LdsDwordCount - Wave64ExchangeDwordCount;
return;
}
var maskArrayType = _module.TypeArray(_uintType, 2);
var maskArrayPointer =
_module.TypePointer(SpirvStorageClass.Workgroup, maskArrayType);
_waveMaskScratchElementPointer =
_module.TypePointer(SpirvStorageClass.Workgroup, _uintType);
_waveMaskScratch = _module.AddGlobalVariable(
maskArrayPointer,
var exchangeArrayType = _module.TypeArray(_uintType, Wave64ExchangeDwordCount);
_wave64ExchangeElementPointer = _module.TypePointer(SpirvStorageClass.Workgroup, _uintType);
_wave64Exchange = _module.AddGlobalVariable(
_module.TypePointer(SpirvStorageClass.Workgroup, exchangeArrayType),
SpirvStorageClass.Workgroup);
_module.AddName(_waveMaskScratch, "wave64MaskScratch");
_interfaces.Add(_waveMaskScratch);
_wave64ExchangeOffset = 0;
_module.AddName(_wave64Exchange, "wave64Exchange");
_interfaces.Add(_wave64Exchange);
}
var uintPointer =
_module.TypePointer(SpirvStorageClass.Workgroup, _uintType);
_waveBroadcastScratch = _module.AddGlobalVariable(
uintPointer,
SpirvStorageClass.Workgroup);
_module.AddName(_waveBroadcastScratch, "wave64BroadcastScratch");
_interfaces.Add(_waveBroadcastScratch);
private bool UsesWave64Exchange() => _emulateWave64 && UsesSubgroupOperations();
private uint ComputeLdsGuestDwordCount()
{
var declared = _request.LocalDataShareDwords;
return declared == 0 || declared > LdsDwordCount
? LdsDwordCount
: System.Numerics.BitOperations.RoundUpToPowerOf2(declared);
}
private void DeclareLds()
@ -679,11 +685,14 @@ public static partial class Gen5SpirvTranslator
? SpirvStorageClass.Workgroup
: SpirvStorageClass.Private;
var dwordCount = _stage == Gen5SpirvStage.Compute
? LdsDwordCount
? ComputeLdsGuestDwordCount()
: PrivateLdsDwordCount;
_ldsDwordMask = dwordCount - 1;
var arrayDwordCount = UsesWave64Exchange() && dwordCount < LdsDwordCount
? dwordCount + Wave64ExchangeDwordCount
: dwordCount;
var ldsArrayType = _module.TypeArray(_uintType, dwordCount);
var ldsArrayType = _module.TypeArray(_uintType, arrayDwordCount);
var ldsPointer = _module.TypePointer(storageClass, ldsArrayType);
_ldsElementPointer = _module.TypePointer(storageClass, _uintType);
_lds64ElementPointer = _module.TypePointer(storageClass, _ulongType);
@ -767,24 +776,6 @@ public static partial class Gen5SpirvTranslator
}
_interfaces.Add(_subgroupInvocationIdInput);
if (_emulateWave64)
{
_subgroupSizeInput = _module.AddGlobalVariable(
subgroupPointer,
SpirvStorageClass.Input);
_module.AddDecoration(
_subgroupSizeInput,
SpirvDecoration.BuiltIn,
(uint)SpirvBuiltIn.SubgroupSize);
if (_stage == Gen5SpirvStage.Pixel)
{
_module.AddDecoration(
_subgroupSizeInput,
SpirvDecoration.Flat);
}
_interfaces.Add(_subgroupSizeInput);
}
// LocalInvocationIndex is a compute-stage built-in. Graphics
// stages can still use native subgroup operations, but they
// cannot combine two subgroup32 halves through a workgroup.
@ -1101,7 +1092,12 @@ public static partial class Gen5SpirvTranslator
{
Store(_pixelValidMaskActive, _module.ConstantBool(true));
}
if (_subgroupInvocationIdInput != 0)
if (_subgroupInvocationIdInput != 0 && _emulateWave64)
{
StoreS64(106, BothHalvesOfOwnBallot(_module.ConstantBool(false)));
StoreS64(126, BothHalvesOfOwnBallot(_module.ConstantBool(true)));
}
else if (_subgroupInvocationIdInput != 0)
{
StoreWaveMask(106, _module.ConstantBool(false));
StoreWaveMask(126, _module.ConstantBool(true));
@ -1360,12 +1356,28 @@ public static partial class Gen5SpirvTranslator
{
error = string.Empty;
var block = blocks[blockIndex];
var halfMaskPlan = HalfMaskPlan();
// One guest wave can span two host subgroups. Keep its shared-memory phases ordered.
var synchronizeSharedMemory = _emulateWave64;
// The half-mask plan inserts its own barriers; other wave64 programs also
// need LDS phase ordering when reads and writes span basic blocks.
var synchronizeSharedMemory = _emulateWave64 && halfMaskPlan is null;
var sharedMemoryPhase = SharedMemoryPhase.None;
for (var index = block.StartIndex; index < block.EndIndex; index++)
{
var instruction = _request.Program.Instructions[index];
if (halfMaskPlan is not null)
{
if (halfMaskPlan.ExactPairsBefore.TryGetValue(instruction.Pc, out var exactPairs))
{
EmitExactWaveMasks(exactPairs);
}
if (halfMaskPlan.SharedMemoryBarriersBefore.Contains(instruction.Pc))
{
EmitWave64Barrier();
}
}
if (IsBranch(instruction.Opcode) || instruction.Opcode == "SEndpgm")
{
continue;
@ -1386,7 +1398,9 @@ public static partial class Gen5SpirvTranslator
}
_execKnownFull = IsExecKnownFull(instruction.Pc);
_emittingPc = instruction.Pc;
var emitted = TryEmitInstruction(instruction, out error);
_emittingPc = null;
_execKnownFull = false;
if (!emitted)
{
@ -1650,10 +1664,10 @@ public static partial class Gen5SpirvTranslator
{
"SCbranchScc0" => LogicalNot(Load(_boolType, _scc)),
"SCbranchScc1" => Load(_boolType, _scc),
"SCbranchVccz" => LogicalNot(SubgroupAny(Load(_boolType, _vcc))),
"SCbranchVccnz" => SubgroupAny(Load(_boolType, _vcc)),
"SCbranchExecz" => LogicalNot(SubgroupAny(Load(_boolType, _exec))),
"SCbranchExecnz" => SubgroupAny(Load(_boolType, _exec)),
"SCbranchVccz" => LogicalNot(WaveMaskAny(106, _vcc)),
"SCbranchVccnz" => WaveMaskAny(106, _vcc),
"SCbranchExecz" => LogicalNot(WaveMaskAny(126, _exec)),
"SCbranchExecnz" => WaveMaskAny(126, _exec),
// The emulator does not expose a shader debug session.
"SCbranchCdbgsys" or
"SCbranchCdbguser" or
@ -2188,11 +2202,7 @@ public static partial class Gen5SpirvTranslator
uint broadcast;
if (_emulateWave64)
{
EmitConditional(isFirstActive, () =>
Store(WaveBroadcastScratchPointer(), firstValue));
EmitWave64Barrier();
broadcast = Load(_uintType, WaveBroadcastScratchPointer());
EmitWave64Barrier();
broadcast = ExchangeWave64Value(isFirstActive, firstValue);
}
else
{
@ -7032,6 +7042,11 @@ public static partial class Gen5SpirvTranslator
_programCounter = Variable(_uintType, _module.Constant(_uintType, 0));
_programActive = Flag(true);
if (UsesWave64Exchange())
{
_wave64ExchangeParity = Variable(_uintType, _module.Constant(_uintType, 0));
_module.AddName(_wave64ExchangeParity, "wave64ExchangeParity");
}
if (_maxDispatcherSteps > 0)
{
_iterationGuard = Variable(_uintType, _module.Constant(_uintType, 0));
@ -7545,6 +7560,19 @@ public static partial class Gen5SpirvTranslator
return BooleanToLaneMask(condition);
}
if (_emulateWave64)
{
var lane = GuestWaveLane();
var owned = OwnHalfBallot(condition);
var exchange = BeginWave64Exchange();
EmitConditional(IsHalfWaveLeader(lane), () =>
Store(Wave64ExchangePointer(exchange, ShiftRightLogical(lane, UInt(5))), owned));
EmitWave64Barrier();
return Pair64(
Load(_uintType, Wave64ExchangePointer(exchange, UInt(0))),
Load(_uintType, Wave64ExchangePointer(exchange, UInt(1))));
}
var ballot = _module.AddInstruction(
SpirvOp.GroupNonUniformBallot,
_uvec4Type,
@ -7555,57 +7583,6 @@ public static partial class Gen5SpirvTranslator
_uintType,
ballot,
0);
if (_emulateWave64)
{
var high = _module.AddInstruction(
SpirvOp.CompositeExtract,
_uintType,
ballot,
1);
var subgroupLane =
Load(_uintType, _subgroupInvocationIdInput);
var firstLane = _module.AddInstruction(
SpirvOp.IEqual,
_boolType,
subgroupLane,
UInt(0));
var half = ShiftRightLogical(GuestWaveLane(), UInt(5));
EmitConditional(firstLane, () =>
{
Store(WaveMaskScratchPointer(half), low);
var nativeWave64 = _module.AddInstruction(
SpirvOp.UGreaterThanEqual,
_boolType,
Load(_uintType, _subgroupSizeInput),
UInt(64));
EmitConditional(nativeWave64, () =>
{
Store(WaveMaskScratchPointer(UInt(1)), high);
});
});
EmitWave64Barrier();
var lowMask = Load(
_uintType,
WaveMaskScratchPointer(UInt(0)));
var highMask = Load(
_uintType,
WaveMaskScratchPointer(UInt(1)));
var combined = BitwiseOr64(
_module.AddInstruction(
SpirvOp.UConvert,
_ulongType,
lowMask),
ShiftLeftLogical64(
_module.AddInstruction(
SpirvOp.UConvert,
_ulongType,
highMask),
_module.Constant64(_ulongType, 32)));
EmitWave64Barrier();
return combined;
}
var widened = _module.AddInstruction(SpirvOp.UConvert, _ulongType, low);
if (_waveLaneCount != 64)
{
@ -7626,21 +7603,127 @@ public static partial class Gen5SpirvTranslator
widened);
}
private uint WaveMaskScratchPointer(uint index) =>
private uint OwnHalfBallot(uint condition)
{
var ballot = _module.AddInstruction(
SpirvOp.GroupNonUniformBallot,
_uvec4Type,
UInt(3),
condition);
return _module.AddInstruction(
SpirvOp.VectorExtractDynamic,
_uintType,
ballot,
ShiftRightLogical(Load(_uintType, _subgroupInvocationIdInput), UInt(5)));
}
private uint BooleanToHalfWaveMask(uint condition)
{
var owned = _module.AddInstruction(SpirvOp.UConvert, _ulongType, OwnHalfBallot(condition));
return _module.AddInstruction(
SpirvOp.Select,
_ulongType,
_module.AddInstruction(SpirvOp.UGreaterThanEqual, _boolType, GuestWaveLane(), UInt(32)),
ShiftLeftLogical64(owned, _module.Constant64(_ulongType, 32)),
owned);
}
private uint BothHalvesOfOwnBallot(uint condition)
{
var owned = _module.AddInstruction(SpirvOp.UConvert, _ulongType, OwnHalfBallot(condition));
return BitwiseOr64(owned, ShiftLeftLogical64(owned, _module.Constant64(_ulongType, 32)));
}
private uint IsHalfWaveLeader(uint lane) =>
_module.AddInstruction(SpirvOp.IEqual, _boolType, BitwiseAnd(lane, UInt(31)), UInt(0));
private uint BeginWave64Exchange()
{
var parity = Load(_uintType, _wave64ExchangeParity);
Store(_wave64ExchangeParity, BitwiseXor(parity, UInt(1)));
return IAdd(
UInt(_wave64ExchangeOffset),
_module.AddInstruction(SpirvOp.IMul, _uintType, parity, UInt(Wave64ExchangeSlotCount)));
}
private uint Wave64ExchangePointer(uint exchange, uint slot) =>
_module.AddInstruction(
SpirvOp.AccessChain,
_waveMaskScratchElementPointer,
_waveScratchInLds ? _lds : _waveMaskScratch,
_waveScratchInLds ? IAdd(UInt(LdsDwordCount - 3), index) : index);
_wave64ExchangeElementPointer,
_wave64Exchange,
IAdd(exchange, slot));
private uint WaveBroadcastScratchPointer() =>
_waveScratchInLds
? _module.AddInstruction(
SpirvOp.AccessChain,
_ldsElementPointer,
_lds,
UInt(LdsDwordCount - 1))
: _waveBroadcastScratch;
private uint ExchangeWave64Value(uint writer, uint value)
{
var exchange = BeginWave64Exchange();
EmitConditional(writer, () => Store(Wave64ExchangePointer(exchange, UInt(0)), value));
EmitWave64Barrier();
return Load(_uintType, Wave64ExchangePointer(exchange, UInt(0)));
}
private void EmitExactWaveMasks(IReadOnlyList<uint> pairs)
{
var lane = GuestWaveLane();
var upperHalf = ShiftRightLogical(lane, UInt(5));
var lowerHalf = _module.AddInstruction(SpirvOp.IEqual, _boolType, upperHalf, UInt(0));
const int pairsPerExchange = (int)Wave64ExchangeSlotCount / 2;
for (var start = 0; start < pairs.Count; start += pairsPerExchange)
{
var count = Math.Min(pairsPerExchange, pairs.Count - start);
var owned = new uint[count];
for (var index = 0; index < count; index++)
{
var pair = pairs[start + index];
owned[index] = _module.AddInstruction(SpirvOp.Select, _uintType, lowerHalf, LoadS(pair), LoadS(pair + 1));
}
var exchange = BeginWave64Exchange();
EmitConditional(IsHalfWaveLeader(lane), () =>
{
for (var index = 0; index < count; index++)
{
Store(Wave64ExchangePointer(exchange, IAdd(UInt((uint)index * 2), upperHalf)), owned[index]);
}
});
EmitWave64Barrier();
for (var index = 0; index < count; index++)
{
var pair = pairs[start + index];
StoreS(pair, Load(_uintType, Wave64ExchangePointer(exchange, UInt((uint)index * 2))));
StoreS(pair + 1, Load(_uintType, Wave64ExchangePointer(exchange, UInt((uint)index * 2 + 1))));
}
}
}
private uint WaveMaskAny(uint register, uint laneActive) =>
_emulateWave64 && _subgroupInvocationIdInput != 0
? IsNotZero64(LoadS64(register))
: SubgroupAny(Load(_boolType, laneActive));
private static readonly bool HalfWaveMasks = !string.Equals(
Environment.GetEnvironmentVariable("SHARPEMU_WAVE64_HALF_MASKS"), "0", StringComparison.Ordinal);
private bool _halfMaskPlanBuilt;
private Ir.Gen5Wave64HalfMaskPlan? HalfMaskPlan()
{
if (_halfMaskPlanBuilt)
{
return _halfMaskPlan;
}
_halfMaskPlanBuilt = true;
if (HalfWaveMasks && UsesWave64Exchange() && _subgroupInvocationIdInput != 0)
{
var sharedFlatPcs = _request.Memory.Entries
.Where(static memory => memory.AddressSpace is FlatAddressSpace.Shared or FlatAddressSpace.SharedOrPrivate)
.Select(static memory => memory.Pc)
.ToHashSet();
_halfMaskPlan = Ir.Gen5Wave64HalfMaskAnalysis.Analyze(_request.Program, sharedFlatPcs);
}
return _halfMaskPlan;
}
private void EmitWave64Barrier()
{
@ -7685,6 +7768,17 @@ public static partial class Gen5SpirvTranslator
return;
}
if (_emulateWave64 &&
_subgroupInvocationIdInput != 0 &&
_emittingPc is { } pc &&
HalfMaskPlan() is { } plan &&
plan.HalfMaskWrites.TryGetValue(pc, out var halfRegister) &&
halfRegister == register)
{
StoreS64(register, BooleanToHalfWaveMask(condition));
return;
}
StoreS64(register, BooleanToWaveMask(condition));
}

269
tests/SharpEmu.Libs.Tests/Gpu/Vulkan/Wave64HalfMaskDeviceTests.cs Normal file
View File

@ -0,0 +1,269 @@
// Copyright (C) 2026 SharpEmu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
using System.Buffers.Binary;
using SharpEmu.Libs.Gpu.Buffers;
using SharpEmu.Libs.Tests.Gpu.Images;
using SharpEmu.ShaderCompiler;
using SharpEmu.ShaderCompiler.Resources;
using SharpEmu.ShaderCompiler.Vulkan;
using Xunit;
using static SharpEmu.ShaderCompiler.Tests.Resources.ResourceTestProgram;
namespace SharpEmu.Libs.Tests.Gpu.Vulkan;
public sealed class Wave64HalfMaskDeviceTests(HeadlessVulkanFixture fixture) : IClassFixture<HeadlessVulkanFixture>
{
private const uint Vcc = 106;
private const uint Exec = 126;
private static readonly Gen5Operand AllLanes = Gen5Operand.Source(193);
private readonly List<Gen5ShaderInstruction> _instructions = [];
private uint _pc;
private uint Add(Gen5ShaderInstruction instruction)
{
var pc = _pc;
_instructions.Add(instruction with { Pc = pc });
_pc += 8;
return pc;
}
private void Point(uint branchPc, uint targetPc)
{
var index = _instructions.FindIndex(instruction => instruction.Pc == branchPc);
var offset = (short)(((int)targetPc - (int)branchPc - 4) / 4);
_instructions[index] = _instructions[index] with { Words = [unchecked((uint)(ushort)offset)] };
}
private void Lane()
{
Add(Vop2(0, "VLshlrevB32", 2, Operand(3), Gen5Operand.Vector(1)));
Add(Vop2(0, "VAddI32", 2, Gen5Operand.Vector(0), Gen5Operand.Vector(2)));
}
private uint CompareTo(string opcode, uint destination, Gen5Operand left, Gen5Operand right) =>
Add(new Gen5ShaderInstruction(0, Gen5ShaderEncoding.Vop3, opcode, [0u, 0u], [left, right], [Gen5Operand.Scalar(destination)],
new Gen5Vop3Control(0, 0, 0, false, 0, destination)));
private uint[] Run(uint localDataShareDwords = 0)
{
Add(Vop2(0, "VLshlrevB32", 5, Operand(2), Gen5Operand.Vector(2)));
Add(BufferAccess(0, "BufferStoreDword", 4, vectorData: 4, offsetEnabled: true, vectorAddress: 5));
Add(EndProgram(0));
var (plan, resources, layout) = Prepare(Program([.. _instructions]));
var request = new ShaderCompileRequest(plan, resources, layout)
{
LocalSizeX = 8, LocalSizeY = 8, ThreadCountX = 8, ThreadCountY = 8, WaveSize = 64,
LocalDataShareDwords = localDataShareDwords,
};
Assert.True(Gen5SpirvTranslator.TryCompileProgram(request, out var shader, out var error), error);
Assert.NotNull(fixture.Vulkan);
using var harness = new ImageTestHarness(fixture.Vulkan);
using var runner = new LayoutComputeRunner(harness, request, shader.Spirv);
var output = runner.CreateBuffer(256);
var registers = new uint[256];
registers[6] = 256;
harness.Run(() => runner.Dispatch(registers,
new Dictionary<DescriptorBindingKind, GpuBuffer[]> { [DescriptorBindingKind.Buffers] = [output] }, 1));
var bytes = runner.ReadBack(output, 0, 256);
harness.AssertNoValidationMessages();
var lanes = new uint[64];
for (var lane = 0; lane < lanes.Length; lane++)
{
lanes[lane] = BinaryPrimitives.ReadUInt32LittleEndian(bytes.AsSpan(lane * sizeof(uint)));
}
return lanes;
}
private bool Ready() => GatePrerequisites.Ready(fixture.Vulkan, shaderInt64: true);
[Fact]
public void MasksAcrossTheHalfBoundary_StayExactPerLaneAndWhenCounted()
{
if (!Ready()) return;
Lane();
CompareTo("VCmpLtU32", 10, Gen5Operand.Vector(2), Operand(40));
Add(Vop2(0, "VAndB32", 6, Operand(1), Gen5Operand.Vector(2)));
Add(Vopc(0, "VCmpEqU32", Operand(1), 6));
Add(Sop2(0, "SAndB64", 12, Gen5Operand.Scalar(10), Gen5Operand.Scalar(Vcc)));
Add(Vop3(0, "VCndmaskB32", 4, Operand(0), Operand(1000), Gen5Operand.Scalar(12)));
Add(Sop1(0, "SBcnt1I32B64", 14, Gen5Operand.Scalar(12)));
Add(Vop2(0, "VAddU32", 4, Gen5Operand.Scalar(14), Gen5Operand.Vector(4)));
var lanes = Run();
for (uint lane = 0; lane < 64; lane++)
{
Assert.Equal((lane & 1) != 0 && lane < 40 ? 1020u : 20u, lanes[lane]);
}
}
[Theory]
[InlineData(48u, 77u)]
[InlineData(8u, 77u)]
[InlineData(64u, 5u)]
public void AnExecBranch_IsDecidedForTheWholeWave(uint firstActiveLane, uint expected)
{
if (!Ready()) return;
Lane();
Add(MoveScalar(0, 20, 5));
Add(Vopc(0, "VCmpxLeU32", Operand(firstActiveLane), 2));
var skip = Add(Branch(0, "SCbranchExecz", 0));
Add(MoveScalar(0, 20, 77));
var join = Add(Sop1(0, "SMovB64", Exec, AllLanes));
Point(skip, join);
Add(MoveVectorFromScalar(0, 4, 20));
Assert.All(Run(), value => Assert.Equal(expected, value));
}
[Theory]
[InlineData(50u, 9u)]
[InlineData(10u, 9u)]
[InlineData(64u, 3u)]
public void AMaskWhoseSccIsRead_CountsBothHalves(uint threshold, uint expected)
{
if (!Ready()) return;
Lane();
Add(MoveScalar(0, 20, 3));
CompareTo("VCmpGeU32", 10, Gen5Operand.Vector(2), Operand(threshold));
Add(Sop2(0, "SAndB64", 12, Gen5Operand.Scalar(10), Gen5Operand.Scalar(Exec)));
var skip = Add(Branch(0, "SCbranchScc0", 0));
Add(MoveScalar(0, 20, 9));
var join = Add(MoveVectorFromScalar(0, 4, 20));
Point(skip, join);
Assert.All(Run(), value => Assert.Equal(expected, value));
}
[Theory]
[InlineData(0u)]
[InlineData(64u)]
public void SharedMemoryWrittenByOneHalf_IsReadByTheOther(uint localDataShareDwords)
{
if (!Ready()) return;
Lane();
Add(Vop2(0, "VLshlrevB32", 3, Operand(1), Gen5Operand.Vector(2)));
Add(Vop2(0, "VAddU32", 3, Gen5Operand.Vector(2), Gen5Operand.Vector(3)));
Add(Vop2(0, "VLshlrevB32", 6, Operand(2), Gen5Operand.Vector(2)));
Add(Vopc(0, "VCmpxGtU32", Operand(32), 2));
Add(BufferAccess(0, "BufferLoadDword", 4, vectorData: 8, offsetEnabled: true, vectorAddress: 6));
Add(Vop2(0, "VAddU32", 3, Gen5Operand.Vector(8), Gen5Operand.Vector(3)));
Add(DataShare(0, "DsWriteB32", false, [Gen5Operand.Vector(6), Gen5Operand.Vector(3)], []));
Add(Sop1(0, "SMovB64", Exec, AllLanes));
Add(Vop2(0, "VAndB32", 7, Operand(31), Gen5Operand.Vector(2)));
Add(Vop2(0, "VLshlrevB32", 7, Operand(2), Gen5Operand.Vector(7)));
Add(DataShare(0, "DsReadB32", false, [Gen5Operand.Vector(7)], [4]));
var lanes = Run(localDataShareDwords);
for (uint lane = 0; lane < 64; lane++)
{
Assert.Equal((lane & 31) * 3, lanes[lane]);
}
}
[Fact]
public void AnExecLoop_RunsEveryLaneItsOwnCount()
{
if (!Ready()) return;
Lane();
Add(Vop2(0, "VAndB32", 3, Operand(7), Gen5Operand.Vector(2)));
Add(Vop2(0, "VAddU32", 3, Operand(1), Gen5Operand.Vector(3)));
Add(MoveVector(0, 4, 0));
Add(Sop1(0, "SMovB64", 20, Gen5Operand.Scalar(Exec)));
var header = Add(Vopc(0, "VCmpxGtU32", Gen5Operand.Vector(3), 4));
var exit = Add(Branch(0, "SCbranchExecz", 0));
Add(Vop2(0, "VAddU32", 4, Operand(1), Gen5Operand.Vector(4)));
var back = Add(Branch(0, "SBranch", 0));
var done = Add(Sop1(0, "SMovB64", Exec, Gen5Operand.Scalar(20)));
Point(exit, done);
Point(back, header);
var lanes = Run();
for (uint lane = 0; lane < 64; lane++)
{
Assert.Equal((lane & 7) + 1, lanes[lane]);
}
}
[Theory]
[InlineData(40u, 140u)]
[InlineData(3u, 103u)]
[InlineData(64u, 100u)]
public void ReadFirstLane_TakesTheFirstActiveLaneOfTheWholeWave(uint firstActiveLane, uint expected)
{
if (!Ready()) return;
Lane();
Add(Vop2(0, "VAddU32", 3, Operand(100), Gen5Operand.Vector(2)));
Add(Vopc(0, "VCmpxLeU32", Operand(firstActiveLane), 2));
Add(ReadFirstLane(0, 20, 3));
Add(Sop1(0, "SMovB64", Exec, AllLanes));
Add(MoveVectorFromScalar(0, 4, 20));
Assert.All(Run(), value => Assert.Equal(expected, value));
}
[Fact]
public void ADemonsSoulsIndexRemap_CompilesOnTheDevice()
{
if (!Ready()) return;
uint[] words =
[
0xBEFC03FFu, 0xA65CD91Eu, 0xBF960000u, 0xBFA00003u, 0xD7460002u, 0x04010C0Cu, 0xF4201A84u, 0xFA000000u,
0xBF8CC07Fu, 0x7DA8046Au, 0xBF88002Au, 0xF4200304u, 0xFA000004u, 0xBF8CC07Fu, 0x7E000C0Cu, 0xBF070C80u,
0x8588807Eu, 0x7E005700u, 0x100000FFu, 0x4F800000u, 0x7E060F00u, 0xD5766A00u, 0x0202060Cu, 0x7D8A0280u,
0x4C020080u, 0x02000101u, 0xD56A0001u, 0x00020700u, 0x4C000303u, 0x4A020303u, 0x02000101u, 0xD56A0000u,
0x00020500u, 0xD5690001u, 0x0002000Cu, 0x4C060302u, 0x7D8C02F9u, 0x06068A02u, 0x7D86060Cu, 0x87EA6A0Au,
0x50000080u, 0xD5286A00u, 0x002A00C1u, 0xD5010000u, 0x002200C1u, 0xD5690000u, 0x0002000Cu, 0x4C000102u,
0xE0002000u, 0x80000000u, 0xBF8C3F70u, 0xE0102000u, 0x80010002u, 0xBF810000u,
];
var bytes = new byte[words.Length * sizeof(uint)];
for (var index = 0; index < words.Length; index++)
BinaryPrimitives.WriteUInt32LittleEndian(bytes.AsSpan(index * sizeof(uint)), words[index]);
Assert.True(Gen5ShaderTranslator.TryDecodeProgram(new SharpEmu.HLE.CpuContext(new WordMemory(bytes), SharpEmu.HLE.Generation.Gen5),
0x1000, out var program, out var decodeError), decodeError);
var (plan, resources, layout) = Prepare(program, userDataCount: 12);
var request = new ShaderCompileRequest(plan, resources, layout)
{
LocalSizeX = 64, WaveSize = 64, EnableExecGuardElision = false,
ComputeSystemRegisters = new Gen5ComputeSystemRegisters(12, null, null, null),
};
Assert.True(Gen5SpirvTranslator.TryCompileProgram(request, out var shader, out var error), error);
Assert.NotNull(fixture.Vulkan);
using var harness = new ImageTestHarness(fixture.Vulkan);
using var runner = new LayoutComputeRunner(harness, request, shader.Spirv);
harness.AssertNoValidationMessages();
}
private sealed class WordMemory(byte[] bytes) : SharpEmu.HLE.ICpuMemory
{
public bool TryRead(ulong address, Span<byte> destination)
{
if (address < 0x1000 || address - 0x1000 > (ulong)(bytes.Length - destination.Length)) return false;
bytes.AsSpan((int)(address - 0x1000), destination.Length).CopyTo(destination);
return true;
}
public bool TryWrite(ulong address, ReadOnlySpan<byte> source) => false;
}
[Fact]
public void ASavedExec_RestoresAfterAHalfMaskedBody()
{
if (!Ready()) return;
Lane();
Add(MoveVector(0, 4, 1));
CompareTo("VCmpGeU32", 10, Gen5Operand.Vector(2), Operand(28));
Add(Sop1(0, "SAndSaveexecB64", 12, Gen5Operand.Scalar(10)));
Add(MoveVector(0, 4, 2));
Add(Sop2(0, "SOrB64", Exec, Gen5Operand.Scalar(Exec), Gen5Operand.Scalar(12)));
Add(Vop2(0, "VAddU32", 4, Operand(10), Gen5Operand.Vector(4)));
var lanes = Run();
for (uint lane = 0; lane < 64; lane++)
{
Assert.Equal(lane >= 28 ? 12u : 11u, lanes[lane]);
}
}
}