8488 lines
344 KiB
C#
8488 lines
344 KiB
C#
// Copyright (C) 2026 SharpEmu Emulator Project
|
|
// SPDX-License-Identifier: GPL-2.0-or-later
|
|
|
|
using SharpEmu.ShaderCompiler;
|
|
using SharpEmu.ShaderCompiler.Resources;
|
|
|
|
namespace SharpEmu.ShaderCompiler.Vulkan;
|
|
|
|
public static partial class Gen5SpirvTranslator
|
|
{
|
|
private const uint VectorRegisterCount = 512;
|
|
private const uint LdsDwordCount = 8192;
|
|
// Graphics stages model LDS as a per-invocation Private array rather than
|
|
// real workgroup-shared memory. A full 32 KB Private array per vertex/pixel
|
|
// invocation is wasteful and risks Metal compile limits, and per-invocation
|
|
// write-then-read correctness only needs deterministic address→slot masking,
|
|
// so a smaller array is safe.
|
|
private const uint PrivateLdsDwordCount = 2048;
|
|
private const uint RdnaWaveLaneCount = 32;
|
|
private const uint Wave64ExchangeSlotCount = 4;
|
|
private const uint Wave64ExchangeDwordCount = Wave64ExchangeSlotCount * 2;
|
|
|
|
internal static SpirvImageFormat DecodeStorageImageFormat(
|
|
uint dataFormat,
|
|
uint numberType) =>
|
|
CompilationContext.DecodeStorageImageFormat(dataFormat, numberType);
|
|
|
|
// Vulkan only guarantees maxComputeWorkGroupSize >= (1024, 1024, 64) and
|
|
// maxComputeWorkGroupInvocations >= 1024, but a GNM NUM_THREAD layout can put its
|
|
// largest axis on Z (real PS5 hardware has no such asymmetry between axes). Remap
|
|
// logical GNM axes onto physical SPIR-V/dispatch axes so the largest logical size
|
|
// always lands on the physical axis with the most headroom (X), and the smallest
|
|
// lands on Z, which is the only axis with a tight guaranteed limit. This is a pure
|
|
// function of the three thread-group sizes so the host dispatch path (which permutes
|
|
// vkCmdDispatch's group counts the same way) can recompute it independently without
|
|
// any shader-compiler metadata plumbing.
|
|
//
|
|
// Returns physicalAxisOfLogical: physicalAxisOfLogical[logical 0=X/1=Y/2=Z] is the
|
|
// physical axis (0=X/1=Y/2=Z) that logical axis is remapped onto.
|
|
public static int[] ComputeWorkgroupAxisOrder(uint sizeX, uint sizeY, uint sizeZ)
|
|
{
|
|
if (sizeZ <= 64)
|
|
{
|
|
return [0, 1, 2];
|
|
}
|
|
|
|
var sizes = new[] { sizeX, sizeY, sizeZ };
|
|
var byDescendingSize = new[] { 0, 1, 2 };
|
|
Array.Sort(byDescendingSize, (a, b) => sizes[b].CompareTo(sizes[a]));
|
|
|
|
var physicalAxisOfLogical = new int[3];
|
|
for (var physical = 0; physical < 3; physical++)
|
|
{
|
|
physicalAxisOfLogical[byDescendingSize[physical]] = physical;
|
|
}
|
|
|
|
return physicalAxisOfLogical;
|
|
}
|
|
|
|
private sealed partial class CompilationContext
|
|
{
|
|
private const int ScalarRegisterCount = 128;
|
|
|
|
// M0. Used as the runtime index added to the register numbers encoded in
|
|
// the V_MOVREL* instructions, and as the LDS/GDS base elsewhere.
|
|
private const uint M0ScalarRegister = 124;
|
|
|
|
private readonly SpirvModuleBuilder _module = new();
|
|
private readonly Gen5SpirvStage _stage;
|
|
private readonly IReadOnlyList<Gen5PixelOutputBinding> _pixelOutputBindings;
|
|
private readonly bool _usesPixelValidMask;
|
|
private readonly bool _enableGraphicsSubgroupOperations;
|
|
private readonly uint _waveLaneCount;
|
|
private readonly bool _emulateWave64;
|
|
|
|
// Safety valve for the PC-dispatcher loop. Each iteration executes one
|
|
// GCN basic block; a correctly-translated shader always reaches its
|
|
// terminal block (pc out of range -> default -> exit) well within any
|
|
// real control flow. A mistranslated shader whose loop-exit condition is
|
|
// wrong would otherwise spin the dispatcher forever, hanging the single
|
|
// Metal queue and freezing every later submission (black screen, no
|
|
// recovery). Bounding the iteration count guarantees the invocation
|
|
// terminates instead: the effect may be wrong for that shader, but the
|
|
// GPU never wedges. 0 disables the guard (original unbounded behaviour).
|
|
private static readonly int _maxDispatcherSteps =
|
|
int.TryParse(
|
|
Environment.GetEnvironmentVariable("SHARPEMU_SHADER_MAX_STEPS"),
|
|
out var maxSteps) && maxSteps >= 0
|
|
? maxSteps
|
|
: 100_000;
|
|
|
|
// Diagnostic coverage probe. When enabled, every selected MRT export
|
|
// writes opaque magenta while preserving the shader's control flow,
|
|
// EXEC mask, geometry and raster state. This separates missing
|
|
// rasterization from valid fragments whose translated values are zero.
|
|
private static readonly bool _forcePixelMagenta =
|
|
string.Equals(
|
|
Environment.GetEnvironmentVariable("SHARPEMU_FORCE_PIXEL_MAGENTA"),
|
|
"1",
|
|
StringComparison.Ordinal);
|
|
|
|
// Which pixel-shader MRT export target (EXP_MRT0..7 == render-target
|
|
// slot) is routed to the single fragment output. The offscreen draw
|
|
// path renders one bound color target per pass, so a multi-render-target
|
|
// (deferred G-buffer) draw compiles one pixel variant per slot, each
|
|
// selecting that slot's export here.
|
|
// Vertex stage only: the fragment shader paired with this vertex shader
|
|
// declares interpolated inputs for locations 0..(this-1). Metal requires
|
|
// every fragment input location to be written by the vertex shader, so
|
|
// the vertex stage must export at least this many param outputs (any it
|
|
// does not naturally export are zero-filled) or pipeline creation fails
|
|
// with "Fragment input(s) `user(locnN)` ... not written by vertex shader".
|
|
private readonly int _requiredVertexOutputCount;
|
|
private readonly uint _localSizeX;
|
|
private readonly uint _localSizeY;
|
|
private readonly uint _localSizeZ;
|
|
|
|
// Vulkan only guarantees maxComputeWorkGroupSize >= (1024, 1024, 64), but a GNM
|
|
// NUM_THREAD layout can put its largest axis on Z (real hardware has no such asymmetry).
|
|
// physicalAxisOfLogical[logical 0=X/1=Y/2=Z] gives which physical SPIR-V axis (and which
|
|
// vkCmdDispatch group-count slot) that logical axis is remapped onto, so the largest size
|
|
// always lands on the physical axis with the most headroom.
|
|
private readonly int[] _physicalAxisOfLogical;
|
|
private readonly uint _pixelInputEnable;
|
|
private readonly uint _pixelInputAddress;
|
|
private readonly uint[] _pixelInputCntl;
|
|
private readonly List<uint> _interfaces = [];
|
|
private readonly Dictionary<uint, uint> _pixelInputs = [];
|
|
private readonly Dictionary<uint, SpirvPixelOutput> _pixelOutputs = [];
|
|
private readonly Dictionary<uint, uint> _vertexOutputs = [];
|
|
private readonly Dictionary<uint, SpirvVertexInput> _vertexInputsByPc = [];
|
|
private uint _voidType;
|
|
private uint _boolType;
|
|
private uint _uintType;
|
|
private uint _intType;
|
|
private uint _longType;
|
|
private uint _ulongType;
|
|
private uint _floatType;
|
|
private uint _vec2Type;
|
|
private uint _vec3Type;
|
|
private uint _vec4Type;
|
|
private uint _uvec2Type;
|
|
private uint _uvec3Type;
|
|
private uint _uvec4Type;
|
|
private uint _privateUintPointer;
|
|
private uint _privateVec2Pointer;
|
|
private readonly HashSet<uint> _flagVariables = [];
|
|
private uint _runtimeBufferBiases;
|
|
// One Private variable per register the program touches, declared on first use. A whole
|
|
// register-file array is kept only for the VGPRs of a program that indexes them at run time
|
|
// (V_MOVREL*): MoltenVK lowers a 512-entry private array to thread memory, and every
|
|
// register access of every invocation then spills through it.
|
|
private readonly Dictionary<uint, uint> _scalarRegisterVariables = new();
|
|
private readonly Dictionary<uint, uint> _vectorRegisterVariables = new();
|
|
private uint _vectorRegisters;
|
|
private (uint First, uint Last) _dynamicVectorRange;
|
|
private uint _packedHalfRegisters;
|
|
private uint _scc;
|
|
private uint _vcc;
|
|
private uint _exec;
|
|
private uint _reachedPixelExport;
|
|
private uint _pixelValidMaskActive;
|
|
private uint _programCounter;
|
|
private uint _programActive;
|
|
private uint _iterationGuard;
|
|
// One-lane waves only have lane 0 of each VGPR. Other lanes that the program spills
|
|
// SGPRs into with V_WRITELANE and reads back with V_READLANE live here, keyed by
|
|
// (VGPR, lane).
|
|
private readonly Dictionary<(uint Register, uint Lane), uint> _laneSpillSlots = new();
|
|
private uint _globalBuffers;
|
|
private uint _gfx10BufferFormatTable;
|
|
private uint _storageBlockPointer;
|
|
private uint _storageUintPointer;
|
|
private uint _lds;
|
|
private uint _ldsElementPointer;
|
|
private uint _lds64ElementPointer;
|
|
private uint _ldsDwordMask;
|
|
private uint _scratch;
|
|
private uint _scratchElementPointer;
|
|
private uint _scratchDwordCount;
|
|
private uint _positionOutput;
|
|
private uint _pointSizeOutput;
|
|
private uint _clipDistanceOutput;
|
|
private uint _cullDistanceOutput;
|
|
private uint _layerOutput;
|
|
private uint _viewportIndexOutput;
|
|
private uint _clipDistanceCount;
|
|
private uint _invalidPositionClipDistance = uint.MaxValue;
|
|
private uint _cullDistanceCount;
|
|
private uint _vertexIndexInput;
|
|
private uint _instanceIndexInput;
|
|
private uint _fragCoordInput;
|
|
private uint _localInvocationIdInput;
|
|
private uint _localInvocationIndexInput;
|
|
private uint _workGroupIdInput;
|
|
private uint _pushConstantUintPointer;
|
|
private uint _subgroupInvocationIdInput;
|
|
private uint _wave64Exchange;
|
|
private uint _wave64ExchangeElementPointer;
|
|
private uint _wave64ExchangeOffset;
|
|
private uint _wave64ExchangeParity;
|
|
private Ir.Gen5Wave64HalfMaskPlan? _halfMaskPlan;
|
|
private uint? _emittingPc;
|
|
private uint _glsl;
|
|
|
|
private enum ImageComponentKind
|
|
{
|
|
Float,
|
|
Sint,
|
|
Uint,
|
|
}
|
|
|
|
private enum VertexInputComponentKind
|
|
{
|
|
Float,
|
|
Sint,
|
|
Uint,
|
|
}
|
|
|
|
private readonly record struct SpirvImageResource(
|
|
uint Variable,
|
|
uint ImageType,
|
|
uint ObjectType,
|
|
uint ComponentType,
|
|
uint VectorType,
|
|
ImageComponentKind ComponentKind,
|
|
bool IsStorage,
|
|
bool Arrayed,
|
|
bool Cube = false,
|
|
bool Multisampled = false,
|
|
SpirvImageDim Dimension = SpirvImageDim.Dim2D,
|
|
uint ConversionFormat = 0,
|
|
uint ShaderSwizzle = 0,
|
|
int EmulatedCompareFunction = -1);
|
|
|
|
private readonly record struct SpirvVertexInput(
|
|
uint Variable,
|
|
uint Type,
|
|
uint ComponentType,
|
|
uint ComponentCount,
|
|
VertexInputComponentKind ComponentKind,
|
|
uint NumberFormat,
|
|
uint DestinationSelect);
|
|
|
|
private readonly record struct SpirvPixelOutput(
|
|
uint Variable,
|
|
uint Type,
|
|
Gen5PixelOutputKind Kind,
|
|
Gen5ColorComponentMapping ComponentMapping);
|
|
|
|
public bool TryCompile(out Gen5SpirvShader shader, out string error)
|
|
{
|
|
shader = default!;
|
|
error = string.Empty;
|
|
try
|
|
{
|
|
if (Environment.GetEnvironmentVariable(
|
|
"SHARPEMU_TRACE_TITLE_INTERFACE") == "1" &&
|
|
_request.Program.Address is 0x0000000500780000ul or
|
|
0x0000000500781200ul)
|
|
{
|
|
Console.Error.WriteLine(
|
|
$"[AGC][TITLE-INTERFACE] stage={_stage} " +
|
|
$"address=0x{_request.Program.Address:X16} " +
|
|
$"required_vertex_outputs={_requiredVertexOutputCount} " +
|
|
$"ps_ena=0x{_pixelInputEnable:X8} ps_addr=0x{_pixelInputAddress:X8}");
|
|
foreach (var instruction in _request.Program.Instructions)
|
|
{
|
|
if (instruction.Control is Gen5ExportControl export)
|
|
{
|
|
Console.Error.WriteLine(
|
|
$"[AGC][TITLE-INTERFACE] pc=0x{instruction.Pc:X4} " +
|
|
$"export_target={export.Target} mask=0x{export.EnableMask:X} " +
|
|
$"compressed={export.Compressed} src=[" +
|
|
string.Join(',', instruction.Sources) + "]");
|
|
}
|
|
else if (instruction.Control is Gen5InterpolationControl interpolation)
|
|
{
|
|
Console.Error.WriteLine(
|
|
$"[AGC][TITLE-INTERFACE] pc=0x{instruction.Pc:X4} " +
|
|
$"attribute={interpolation.Attribute} " +
|
|
$"channel={interpolation.Channel} dst=[" +
|
|
string.Join(',', instruction.Destinations) + "]");
|
|
}
|
|
|
|
}
|
|
}
|
|
|
|
var blocks = BuildBasicBlocks(_request.Program.Instructions);
|
|
// The fallback when the full structurer declines: blocks in program order behind a
|
|
// next-block guard, with natural loops as structured loops.
|
|
var structuredForward = StructuredForwardBlocks && blocks.Count != 0 && TryBuildLoopRegions(blocks, out _loopLatchByHeader);
|
|
DeclareModule();
|
|
if (blocks.Count == 0)
|
|
{
|
|
error = "shader contains no executable blocks";
|
|
return false;
|
|
}
|
|
|
|
|
|
var functionType = _module.TypeFunction(_voidType);
|
|
var main = _module.BeginFunction(_voidType, functionType);
|
|
_module.AddName(main, "main");
|
|
_module.AddLabel();
|
|
if (_stage == Gen5SpirvStage.Pixel &&
|
|
Environment.GetEnvironmentVariable(
|
|
"SHARPEMU_FORCE_TITLE_EARLY_COLOR") == "1" &&
|
|
_request.Program.Address == 0x0000000500781200ul)
|
|
{
|
|
var earlyOutput = _pixelOutputs
|
|
.OrderBy(static pair => pair.Key)
|
|
.Select(static pair => pair.Value)
|
|
.First();
|
|
var earlyColor = earlyOutput.Kind switch
|
|
{
|
|
Gen5PixelOutputKind.Float =>
|
|
_module.AddInstruction(
|
|
SpirvOp.CompositeConstruct,
|
|
earlyOutput.Type,
|
|
Float(1f),
|
|
Float(0f),
|
|
Float(1f),
|
|
Float(1f)),
|
|
Gen5PixelOutputKind.Sint =>
|
|
_module.ConstantNull(earlyOutput.Type),
|
|
_ => _module.ConstantNull(earlyOutput.Type),
|
|
};
|
|
Store(earlyOutput.Variable, earlyColor);
|
|
_module.AddStatement(SpirvOp.Return);
|
|
_module.AddLabel();
|
|
}
|
|
EmitInitialState();
|
|
|
|
if (StructuredControlFlow && blocks.Count > 0 && TryStructureRange(blocks, 0, blocks.Count, blocks.Count, emit: false, out _))
|
|
{
|
|
// As the dispatcher does before its first block: an out-of-bounds invocation runs nothing.
|
|
var runLabel = _module.AllocateId();
|
|
var doneLabel = _module.AllocateId();
|
|
var isActive = Load(_boolType, _programActive);
|
|
_module.AddStatement(SpirvOp.SelectionMerge, doneLabel, 0);
|
|
_module.AddStatement(SpirvOp.BranchConditional, isActive, runLabel, doneLabel);
|
|
_module.AddLabel(runLabel);
|
|
if (!TryStructureRange(blocks, 0, blocks.Count, blocks.Count, emit: true, out error))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
_module.AddStatement(SpirvOp.Branch, doneLabel);
|
|
_module.AddLabel(doneLabel);
|
|
}
|
|
else
|
|
{
|
|
var loopHeader = _module.AllocateId();
|
|
var switchHeader = _module.AllocateId();
|
|
var switchMerge = _module.AllocateId();
|
|
var loopContinue = _module.AllocateId();
|
|
var loopMerge = _module.AllocateId();
|
|
var defaultLabel = _module.AllocateId();
|
|
var caseLabels = new uint[blocks.Count];
|
|
for (var index = 0; index < caseLabels.Length; index++)
|
|
{
|
|
caseLabels[index] = _module.AllocateId();
|
|
}
|
|
|
|
if (structuredForward)
|
|
{
|
|
// Each invocation visits its blocks in program order, except for the back
|
|
// edges of natural loops. Emitting the blocks in order, each guarded by "this
|
|
// is the next block", and every loop as a structured SPIR-V loop runs exactly
|
|
// what the dispatcher loop would. The driver then sees ifs and loops and can
|
|
// keep only the live guest registers in hardware registers, where the
|
|
// dispatcher keeps nearly all of them live and pays a switch per block.
|
|
if (!TryEmitStructuredRange(blocks, 0, blocks.Count - 1, -1, out error))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
_module.AddStatement(SpirvOp.Branch, loopMerge);
|
|
}
|
|
else
|
|
{
|
|
_module.AddStatement(SpirvOp.Branch, loopHeader);
|
|
_module.AddLabel(loopHeader);
|
|
// Check before the first block can write from an inactive invocation.
|
|
var programIsActive = Load(_boolType, _programActive);
|
|
_module.AddStatement(SpirvOp.LoopMerge, loopMerge, loopContinue, 0);
|
|
_module.AddStatement(SpirvOp.BranchConditional, programIsActive, switchHeader, loopMerge);
|
|
|
|
_module.AddLabel(switchHeader);
|
|
var selector = Load(_uintType, _programCounter);
|
|
_module.AddStatement(SpirvOp.SelectionMerge, switchMerge, 0);
|
|
var switchOperands = new uint[2 + (blocks.Count * 2)];
|
|
switchOperands[0] = selector;
|
|
switchOperands[1] = defaultLabel;
|
|
for (var index = 0; index < blocks.Count; index++)
|
|
{
|
|
switchOperands[2 + (index * 2)] = (uint)index;
|
|
switchOperands[3 + (index * 2)] = caseLabels[index];
|
|
}
|
|
|
|
_module.AddStatement(SpirvOp.Switch, switchOperands);
|
|
for (var index = 0; index < blocks.Count; index++)
|
|
{
|
|
_module.AddLabel(caseLabels[index]);
|
|
if (!TryEmitBlock(blocks, index, out error))
|
|
{
|
|
error = $"block=0x{blocks[index].StartPc:X}: {error}";
|
|
return false;
|
|
}
|
|
|
|
_module.AddStatement(SpirvOp.Branch, switchMerge);
|
|
}
|
|
|
|
_module.AddLabel(defaultLabel);
|
|
Store(_programActive, _module.ConstantBool(false));
|
|
_module.AddStatement(SpirvOp.Branch, switchMerge);
|
|
|
|
_module.AddLabel(switchMerge);
|
|
_module.AddStatement(SpirvOp.Branch, loopContinue);
|
|
_module.AddLabel(loopContinue);
|
|
var active = Load(_boolType, _programActive);
|
|
if (_maxDispatcherSteps > 0)
|
|
{
|
|
var steps = IAdd(Load(_uintType, _iterationGuard), UInt(1));
|
|
Store(_iterationGuard, steps);
|
|
var withinLimit = _module.AddInstruction(
|
|
SpirvOp.ULessThan,
|
|
_boolType,
|
|
steps,
|
|
UInt((uint)_maxDispatcherSteps));
|
|
active = _module.AddInstruction(
|
|
SpirvOp.LogicalAnd,
|
|
_boolType,
|
|
active,
|
|
withinLimit);
|
|
}
|
|
|
|
_module.AddStatement(
|
|
SpirvOp.BranchConditional,
|
|
active,
|
|
loopHeader,
|
|
loopMerge);
|
|
}
|
|
|
|
_module.AddLabel(loopMerge);
|
|
}
|
|
if (_stage == Gen5SpirvStage.Pixel &&
|
|
Environment.GetEnvironmentVariable(
|
|
"SHARPEMU_TRACE_TITLE_SHADER_STATE") == "1" &&
|
|
_request.Program.Address == 0x0000000500781200ul)
|
|
{
|
|
var stateOutput = _pixelOutputs
|
|
.OrderBy(static pair => pair.Key)
|
|
.Select(static pair => pair.Value)
|
|
.FirstOrDefault(static output =>
|
|
output.Kind == Gen5PixelOutputKind.Float);
|
|
if (stateOutput.Variable != 0)
|
|
{
|
|
uint EncodeBool(uint condition) =>
|
|
_module.AddInstruction(
|
|
SpirvOp.Select,
|
|
_floatType,
|
|
condition,
|
|
Float(1f),
|
|
Float(0f));
|
|
|
|
Store(
|
|
stateOutput.Variable,
|
|
_module.AddInstruction(
|
|
SpirvOp.CompositeConstruct,
|
|
stateOutput.Type,
|
|
EncodeBool(Load(_boolType, _exec)),
|
|
EncodeBool(IsWaveMaskActive(LoadS64(52))),
|
|
EncodeBool(Load(_boolType, _reachedPixelExport)),
|
|
Float(1f)));
|
|
}
|
|
|
|
StoreS64(
|
|
126,
|
|
_module.Constant64(_ulongType, 1));
|
|
}
|
|
if (_stage == Gen5SpirvStage.Pixel)
|
|
{
|
|
// EXP.VM publishes EXEC as the pixel-valid mask. EXEC may
|
|
// subsequently be restored for control-flow convergence,
|
|
// so the final EXEC value is not the fragment-validity
|
|
// decision. Programs without VM retain the old EXEC
|
|
// fallback so malformed shaders still fail conservatively.
|
|
var returnLabel = _module.AllocateId();
|
|
var killLabel = _module.AllocateId();
|
|
// Materialize the condition before SelectionMerge: SPIR-V
|
|
// requires the merge instruction to be immediately followed
|
|
// by its structured branch terminator.
|
|
var laneActive = Load(
|
|
_boolType,
|
|
_usesPixelValidMask ? _pixelValidMaskActive : _exec);
|
|
_module.AddStatement(
|
|
SpirvOp.SelectionMerge,
|
|
returnLabel,
|
|
0);
|
|
_module.AddStatement(
|
|
SpirvOp.BranchConditional,
|
|
laneActive,
|
|
returnLabel,
|
|
killLabel);
|
|
_module.AddLabel(killLabel);
|
|
_module.AddStatement(SpirvOp.Kill);
|
|
_module.AddLabel(returnLabel);
|
|
}
|
|
|
|
_module.AddStatement(SpirvOp.Return);
|
|
_module.EndFunction();
|
|
|
|
var model = _stage switch
|
|
{
|
|
Gen5SpirvStage.Vertex => SpirvExecutionModel.Vertex,
|
|
Gen5SpirvStage.Pixel => SpirvExecutionModel.Fragment,
|
|
_ => SpirvExecutionModel.GLCompute,
|
|
};
|
|
_module.AddEntryPoint(model, main, "main", _interfaces);
|
|
if (_stage == Gen5SpirvStage.Pixel)
|
|
{
|
|
_module.AddExecutionMode(main, SpirvExecutionMode.OriginUpperLeft);
|
|
}
|
|
else if (_stage == Gen5SpirvStage.Compute)
|
|
{
|
|
var logicalSizes = new[] { _localSizeX, _localSizeY, _localSizeZ };
|
|
var physicalSizes = new uint[3];
|
|
for (var logical = 0; logical < 3; logical++)
|
|
{
|
|
physicalSizes[_physicalAxisOfLogical[logical]] = logicalSizes[logical];
|
|
}
|
|
|
|
_module.AddExecutionMode(
|
|
main,
|
|
SpirvExecutionMode.LocalSize,
|
|
physicalSizes[0],
|
|
physicalSizes[1],
|
|
physicalSizes[2]);
|
|
}
|
|
|
|
var attributeCount = _stage == Gen5SpirvStage.Vertex
|
|
? (uint)_vertexOutputs.Count
|
|
: (uint)_pixelInputs.Count;
|
|
shader = new Gen5SpirvShader(_module.Build(), attributeCount);
|
|
return true;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
error = exception.Message;
|
|
return false;
|
|
}
|
|
}
|
|
|
|
private void DeclareModule()
|
|
{
|
|
_module.AddCapability(SpirvCapability.Shader);
|
|
_module.AddCapability(SpirvCapability.Int64);
|
|
_module.AddCapability(SpirvCapability.ImageQuery);
|
|
if (UsesSubgroupOperations())
|
|
{
|
|
_module.AddCapability(SpirvCapability.GroupNonUniform);
|
|
_module.AddCapability(SpirvCapability.GroupNonUniformBallot);
|
|
|
|
if (UsesSubgroupShuffle())
|
|
{
|
|
_module.AddCapability(SpirvCapability.GroupNonUniformShuffle);
|
|
}
|
|
|
|
if (UsesWaveControl())
|
|
{
|
|
_module.AddCapability(SpirvCapability.GroupNonUniformVote);
|
|
}
|
|
|
|
}
|
|
|
|
_glsl = _module.ImportExtInst("GLSL.std.450");
|
|
_voidType = _module.TypeVoid();
|
|
_boolType = _module.TypeBool();
|
|
_uintType = _module.TypeInt(32, signed: false);
|
|
_intType = _module.TypeInt(32, signed: true);
|
|
_longType = _module.TypeInt(64, signed: true);
|
|
_ulongType = _module.TypeInt(64, signed: false);
|
|
_floatType = _module.TypeFloat(32);
|
|
_vec2Type = _module.TypeVector(_floatType, 2);
|
|
_vec3Type = _module.TypeVector(_floatType, 3);
|
|
_vec4Type = _module.TypeVector(_floatType, 4);
|
|
_uvec2Type = _module.TypeVector(_uintType, 2);
|
|
_uvec3Type = _module.TypeVector(_uintType, 3);
|
|
_uvec4Type = _module.TypeVector(_uintType, 4);
|
|
_privateUintPointer =
|
|
_module.TypePointer(SpirvStorageClass.Private, _uintType);
|
|
_privateVec2Pointer =
|
|
_module.TypePointer(SpirvStorageClass.Private, _vec2Type);
|
|
|
|
if (_request.Program.Instructions.Any(static instruction => instruction.Opcode.StartsWith("VMovrel", StringComparison.Ordinal)))
|
|
{
|
|
// A dynamically indexed private array cannot live in registers: Metal spills the
|
|
// whole 2 KiB file to thread memory and every VGPR access becomes a memory access
|
|
// (Octopath's instanced vertex shader: 112 ms per draw). Keep one variable per
|
|
// register and resolve V_MOVREL* with a select over the registers the program uses.
|
|
_dynamicVectorRange = DynamicVectorRange(_request.Program);
|
|
}
|
|
|
|
// Keep scalar state in uint variables, matching upstream's flag load/store conversion.
|
|
uint Flag(bool initialValue)
|
|
{
|
|
var variable = _module.AddGlobalVariable(_privateUintPointer, SpirvStorageClass.Private, UInt(initialValue ? 1u : 0u));
|
|
_flagVariables.Add(variable);
|
|
return variable;
|
|
}
|
|
|
|
_scc = Flag(false);
|
|
_vcc = Flag(false);
|
|
_exec = Flag(true);
|
|
_reachedPixelExport = Flag(false);
|
|
if (_usesPixelValidMask)
|
|
{
|
|
_pixelValidMaskActive = Flag(true);
|
|
}
|
|
_programCounter = _module.AddGlobalVariable(
|
|
_privateUintPointer,
|
|
SpirvStorageClass.Private,
|
|
_module.Constant(_uintType, 0));
|
|
_programActive = Flag(true);
|
|
if (UsesWave64Exchange())
|
|
{
|
|
_wave64ExchangeParity = _module.AddGlobalVariable(_privateUintPointer, SpirvStorageClass.Private, UInt(0));
|
|
_interfaces.Add(_wave64ExchangeParity);
|
|
_module.AddName(_wave64ExchangeParity, "wave64ExchangeParity");
|
|
}
|
|
if (_maxDispatcherSteps > 0)
|
|
{
|
|
_iterationGuard = _module.AddGlobalVariable(
|
|
_privateUintPointer,
|
|
SpirvStorageClass.Private,
|
|
_module.Constant(_uintType, 0));
|
|
_interfaces.Add(_iterationGuard);
|
|
_module.AddName(_iterationGuard, "pcGuard");
|
|
}
|
|
|
|
foreach (var slot in FindLaneSpillSlots())
|
|
{
|
|
var variable = _module.AddGlobalVariable(_privateUintPointer, SpirvStorageClass.Private, _module.Constant(_uintType, 0));
|
|
_interfaces.Add(variable);
|
|
_module.AddName(variable, $"v{slot.Register}_lane{slot.Lane}");
|
|
_laneSpillSlots.Add(slot, variable);
|
|
}
|
|
|
|
if (_vectorRegisters != 0)
|
|
{
|
|
_interfaces.Add(_vectorRegisters);
|
|
_module.AddName(_vectorRegisters, "vgpr");
|
|
}
|
|
|
|
_interfaces.Add(_scc);
|
|
_interfaces.Add(_vcc);
|
|
_interfaces.Add(_exec);
|
|
_interfaces.Add(_reachedPixelExport);
|
|
if (_pixelValidMaskActive != 0)
|
|
{
|
|
_interfaces.Add(_pixelValidMaskActive);
|
|
_module.AddName(_pixelValidMaskActive, "pixelValidMaskActive");
|
|
}
|
|
_interfaces.Add(_programCounter);
|
|
_interfaces.Add(_programActive);
|
|
|
|
{
|
|
DeclareLayoutBindings();
|
|
DeclareScratch();
|
|
DeclareLds();
|
|
DeclareWave64Scratch();
|
|
DeclareStageInterface();
|
|
return;
|
|
}
|
|
}
|
|
|
|
private void DeclareScratch()
|
|
{
|
|
if (!_request.Program.Instructions.Any(static instruction =>
|
|
instruction.Opcode.StartsWith("Scratch", StringComparison.Ordinal)) &&
|
|
!_request.Memory.Entries.Any(static memory =>
|
|
memory.AddressSpace is FlatAddressSpace.Private or FlatAddressSpace.SharedOrPrivate))
|
|
{
|
|
return;
|
|
}
|
|
|
|
_scratchDwordCount = Math.Max(_request.ScratchDwords, 1u);
|
|
var arrayType = _module.TypeArray(_uintType, _scratchDwordCount);
|
|
var arrayPointer = _module.TypePointer(SpirvStorageClass.Private, arrayType);
|
|
_scratchElementPointer = _module.TypePointer(SpirvStorageClass.Private, _uintType);
|
|
_scratch = _module.AddGlobalVariable(
|
|
arrayPointer,
|
|
SpirvStorageClass.Private,
|
|
_module.ConstantNull(arrayType));
|
|
_interfaces.Add(_scratch);
|
|
_module.AddName(_scratch, "scratch");
|
|
}
|
|
|
|
private void DeclareWave64Scratch()
|
|
{
|
|
if (!UsesWave64Exchange())
|
|
{
|
|
return;
|
|
}
|
|
|
|
// Metal exposes 32 KiB of threadgroup memory on the Apple GPUs we
|
|
// target. Some PS5 compute shaders legitimately request all of it,
|
|
// so allocating another workgroup variable for the wave64 bridge
|
|
// makes pipeline creation fail. Reuse the final dwords of the
|
|
// existing LDS allocation in that case. The translator already
|
|
// bounds guest LDS accesses to this fixed allocation; keeping the
|
|
// bridge inside it preserves the host limit and still provides the
|
|
// cross-subgroup rendezvous needed to model one 64-lane guest wave.
|
|
if (_lds != 0)
|
|
{
|
|
_wave64Exchange = _lds;
|
|
_wave64ExchangeElementPointer = _ldsElementPointer;
|
|
_wave64ExchangeOffset = _ldsDwordMask + 1 < LdsDwordCount
|
|
? _ldsDwordMask + 1
|
|
: LdsDwordCount - Wave64ExchangeDwordCount;
|
|
return;
|
|
}
|
|
|
|
var exchangeArrayType = _module.TypeArray(_uintType, Wave64ExchangeDwordCount);
|
|
_wave64ExchangeElementPointer = _module.TypePointer(SpirvStorageClass.Workgroup, _uintType);
|
|
_wave64Exchange = _module.AddGlobalVariable(
|
|
_module.TypePointer(SpirvStorageClass.Workgroup, exchangeArrayType),
|
|
SpirvStorageClass.Workgroup);
|
|
_wave64ExchangeOffset = 0;
|
|
_module.AddName(_wave64Exchange, "wave64Exchange");
|
|
_interfaces.Add(_wave64Exchange);
|
|
}
|
|
|
|
private bool UsesWave64Exchange() => _emulateWave64 && UsesSubgroupOperations();
|
|
|
|
private uint ComputeLdsGuestDwordCount()
|
|
{
|
|
var declared = _request.LocalDataShareDwords;
|
|
return declared == 0 || declared > LdsDwordCount
|
|
? LdsDwordCount
|
|
: System.Numerics.BitOperations.RoundUpToPowerOf2(declared);
|
|
}
|
|
|
|
private void DeclareLds()
|
|
{
|
|
if (!UsesLds())
|
|
{
|
|
return;
|
|
}
|
|
|
|
// Compute shaders get genuine workgroup-shared LDS. Graphics stages
|
|
// (NGG export/vertex, pixel) cannot use the Workgroup storage class
|
|
// in SPIR-V, but they still emit ds_write/ds_read — typically as
|
|
// per-invocation scratch/spill or as NGG staging whose cross-lane
|
|
// reads don't feed this stage's exports. Model those as a
|
|
// per-invocation Private array so the shader is valid SPIR-V and its
|
|
// draw stops being dropped. Index masking in LdsPointer keeps the
|
|
// arbitrary computed addresses inside the array.
|
|
var storageClass = _stage == Gen5SpirvStage.Compute
|
|
? SpirvStorageClass.Workgroup
|
|
: SpirvStorageClass.Private;
|
|
var dwordCount = _stage == Gen5SpirvStage.Compute
|
|
? ComputeLdsGuestDwordCount()
|
|
: PrivateLdsDwordCount;
|
|
_ldsDwordMask = dwordCount - 1;
|
|
var arrayDwordCount = UsesWave64Exchange() && dwordCount < LdsDwordCount
|
|
? dwordCount + Wave64ExchangeDwordCount
|
|
: dwordCount;
|
|
|
|
var ldsArrayType = _module.TypeArray(_uintType, arrayDwordCount);
|
|
var ldsPointer = _module.TypePointer(storageClass, ldsArrayType);
|
|
_ldsElementPointer = _module.TypePointer(storageClass, _uintType);
|
|
_lds64ElementPointer = _module.TypePointer(storageClass, _ulongType);
|
|
_lds = storageClass == SpirvStorageClass.Workgroup
|
|
? _module.AddGlobalVariable(ldsPointer, storageClass)
|
|
: _module.AddGlobalVariable(
|
|
ldsPointer,
|
|
storageClass,
|
|
_module.ConstantNull(ldsArrayType));
|
|
_module.AddName(_lds, "lds");
|
|
_interfaces.Add(_lds);
|
|
}
|
|
|
|
internal static SpirvImageFormat DecodeStorageImageFormat(
|
|
uint dataFormat,
|
|
uint numberType) =>
|
|
(dataFormat, numberType) switch
|
|
{
|
|
(1, 0 or 9) => SpirvImageFormat.R8,
|
|
(1, 1) => SpirvImageFormat.R8Snorm,
|
|
(1, 4) => SpirvImageFormat.R8ui,
|
|
(1, 5) => SpirvImageFormat.R8i,
|
|
(2, 0) => SpirvImageFormat.R16,
|
|
(2, 1) => SpirvImageFormat.R16Snorm,
|
|
(2, 4) => SpirvImageFormat.R16ui,
|
|
(2, 5) => SpirvImageFormat.R16i,
|
|
(2, 7) => SpirvImageFormat.R16f,
|
|
(3, 0 or 9) => SpirvImageFormat.Rg8,
|
|
(3, 1) => SpirvImageFormat.Rg8Snorm,
|
|
(3, 4) => SpirvImageFormat.Rg8ui,
|
|
(3, 5) => SpirvImageFormat.Rg8i,
|
|
(4, 4) => SpirvImageFormat.R32ui,
|
|
(4, 5) => SpirvImageFormat.R32i,
|
|
(4, 7) => SpirvImageFormat.R32f,
|
|
(5, 0) => SpirvImageFormat.Rg16,
|
|
(5, 1) => SpirvImageFormat.Rg16Snorm,
|
|
(5, 4) => SpirvImageFormat.Rg16ui,
|
|
(5, 5) => SpirvImageFormat.Rg16i,
|
|
(5, 7) => SpirvImageFormat.Rg16f,
|
|
(6 or 7, 7) => SpirvImageFormat.R11fG11fB10f,
|
|
(8 or 9, 0) => SpirvImageFormat.Rgb10A2,
|
|
(8 or 9, 4) => SpirvImageFormat.Rgb10A2ui,
|
|
(10, 0 or 9) => SpirvImageFormat.Rgba8,
|
|
(10, 1) => SpirvImageFormat.Rgba8Snorm,
|
|
(10, 4) => SpirvImageFormat.Rgba8ui,
|
|
(10, 5) => SpirvImageFormat.Rgba8i,
|
|
(11, 4) => SpirvImageFormat.Rg32ui,
|
|
(11, 5) => SpirvImageFormat.Rg32i,
|
|
(11, 7) => SpirvImageFormat.Rg32f,
|
|
(12, 0) => SpirvImageFormat.Rgba16,
|
|
(12, 1) => SpirvImageFormat.Rgba16Snorm,
|
|
(12, 4) => SpirvImageFormat.Rgba16ui,
|
|
(12, 5) => SpirvImageFormat.Rgba16i,
|
|
(12, 7) => SpirvImageFormat.Rgba16f,
|
|
(13 or 14, 4) => SpirvImageFormat.Rgba32ui,
|
|
(13 or 14, 5) => SpirvImageFormat.Rgba32i,
|
|
(13 or 14, 7) => SpirvImageFormat.Rgba32f,
|
|
_ => SpirvImageFormat.Unknown,
|
|
};
|
|
|
|
private void DeclareStageInterface()
|
|
{
|
|
if (UsesSubgroupOperations())
|
|
{
|
|
var subgroupPointer =
|
|
_module.TypePointer(SpirvStorageClass.Input, _uintType);
|
|
_subgroupInvocationIdInput = _module.AddGlobalVariable(
|
|
subgroupPointer,
|
|
SpirvStorageClass.Input);
|
|
_module.AddDecoration(
|
|
_subgroupInvocationIdInput,
|
|
SpirvDecoration.BuiltIn,
|
|
(uint)SpirvBuiltIn.SubgroupLocalInvocationId);
|
|
if (_stage == Gen5SpirvStage.Pixel)
|
|
{
|
|
// Vulkan requires integer fragment inputs, including subgroup
|
|
// built-ins, to use flat interpolation.
|
|
_module.AddDecoration(
|
|
_subgroupInvocationIdInput,
|
|
SpirvDecoration.Flat);
|
|
}
|
|
_interfaces.Add(_subgroupInvocationIdInput);
|
|
|
|
// LocalInvocationIndex is a compute-stage built-in. Graphics
|
|
// stages can still use native subgroup operations, but they
|
|
// cannot combine two subgroup32 halves through a workgroup.
|
|
if (_stage == Gen5SpirvStage.Compute && _waveLaneCount == 64)
|
|
{
|
|
_localInvocationIndexInput = _module.AddGlobalVariable(
|
|
subgroupPointer,
|
|
SpirvStorageClass.Input);
|
|
_module.AddDecoration(
|
|
_localInvocationIndexInput,
|
|
SpirvDecoration.BuiltIn,
|
|
(uint)SpirvBuiltIn.LocalInvocationIndex);
|
|
_interfaces.Add(_localInvocationIndexInput);
|
|
}
|
|
}
|
|
|
|
if (_stage == Gen5SpirvStage.Vertex)
|
|
{
|
|
DeclareVertexInputs();
|
|
|
|
var inputPointer =
|
|
_module.TypePointer(SpirvStorageClass.Input, _uintType);
|
|
_vertexIndexInput = _module.AddGlobalVariable(
|
|
inputPointer,
|
|
SpirvStorageClass.Input);
|
|
_module.AddDecoration(
|
|
_vertexIndexInput,
|
|
SpirvDecoration.BuiltIn,
|
|
(uint)SpirvBuiltIn.VertexIndex);
|
|
_interfaces.Add(_vertexIndexInput);
|
|
|
|
_instanceIndexInput = _module.AddGlobalVariable(
|
|
inputPointer,
|
|
SpirvStorageClass.Input);
|
|
_module.AddDecoration(
|
|
_instanceIndexInput,
|
|
SpirvDecoration.BuiltIn,
|
|
(uint)SpirvBuiltIn.InstanceIndex);
|
|
_interfaces.Add(_instanceIndexInput);
|
|
|
|
var outputPointer =
|
|
_module.TypePointer(SpirvStorageClass.Output, _vec4Type);
|
|
_positionOutput = _module.AddGlobalVariable(
|
|
outputPointer,
|
|
SpirvStorageClass.Output);
|
|
_module.AddDecoration(
|
|
_positionOutput,
|
|
SpirvDecoration.BuiltIn,
|
|
(uint)SpirvBuiltIn.Position);
|
|
_interfaces.Add(_positionOutput);
|
|
|
|
DeclareAuxPositionOutputs();
|
|
|
|
var parameters = _request.Program.Instructions
|
|
.Select(instruction => instruction.Control)
|
|
.OfType<Gen5ExportControl>()
|
|
.Where(export => export.Target is >= 32 and < 64)
|
|
.Select(export => export.Target - 32)
|
|
// Cover every location the paired fragment shader reads, even
|
|
// ones this vertex program never exports, so Metal's exact
|
|
// vertex-out/fragment-in interface match succeeds. Extras are
|
|
// zero-filled in EmitInitialState.
|
|
.Concat(Enumerable
|
|
.Range(0, Math.Max(_requiredVertexOutputCount, 0))
|
|
.Select(location => (uint)location))
|
|
.Distinct()
|
|
.Order()
|
|
.ToArray();
|
|
foreach (var parameter in parameters)
|
|
{
|
|
var variable = _module.AddGlobalVariable(
|
|
outputPointer,
|
|
SpirvStorageClass.Output);
|
|
_module.AddDecoration(variable, SpirvDecoration.Location, parameter);
|
|
_vertexOutputs.Add(parameter, variable);
|
|
_interfaces.Add(variable);
|
|
}
|
|
}
|
|
else if (_stage == Gen5SpirvStage.Pixel)
|
|
{
|
|
var inputVec4Pointer =
|
|
_module.TypePointer(SpirvStorageClass.Input, _vec4Type);
|
|
var attributes = _request.Program.Instructions
|
|
.Select(instruction => instruction.Control)
|
|
.OfType<Gen5InterpolationControl>()
|
|
.Select(control => control.Attribute)
|
|
.Distinct()
|
|
.Order()
|
|
.ToArray();
|
|
DeclareInterpolationParameters();
|
|
// Several PS input slots may read one VS parameter (the guest compiler emits a
|
|
// slot per use). Relocating a duplicate left it on a location the vertex program
|
|
// never writes, so it read zero: Astro Bot's save-slot card normals decoded to NaN
|
|
// and a machine's albedo came out black. Slots of one kind share one host input;
|
|
// when a V_INTERP_MOV slot is among them, the others read the per-vertex values
|
|
// (interpolated with the barycentrics, or vertex 0 for a flat slot).
|
|
var shared = new Dictionary<uint, uint>();
|
|
bool IsFlatSlot(uint attribute) =>
|
|
((attribute < (uint)_pixelInputCntl.Length ? _pixelInputCntl[attribute] : 0u) & 0x400u) != 0 ||
|
|
_flatParameterAttributes.Contains(attribute);
|
|
foreach (var group in attributes.GroupBy(attribute =>
|
|
(attribute < (uint)_pixelInputCntl.Length ? _pixelInputCntl[attribute] : attribute) & 0x1Fu))
|
|
{
|
|
uint? perVertexOwner = null;
|
|
foreach (var attribute in group)
|
|
{
|
|
if (_perVertexAttributes.Contains(attribute))
|
|
{
|
|
perVertexOwner = attribute;
|
|
break;
|
|
}
|
|
}
|
|
|
|
var owners = new Dictionary<bool, uint>();
|
|
foreach (var attribute in group)
|
|
{
|
|
if (perVertexOwner is { } perVertex)
|
|
{
|
|
if (attribute != perVertex)
|
|
{
|
|
shared.Add(attribute, perVertex);
|
|
if (!_perVertexAttributes.Contains(attribute))
|
|
{
|
|
_perVertexSourcedAttributes.Add(attribute, IsFlatSlot(attribute));
|
|
}
|
|
}
|
|
|
|
continue;
|
|
}
|
|
|
|
var flat = IsFlatSlot(attribute);
|
|
if (owners.TryGetValue(flat, out var owner))
|
|
{
|
|
shared.Add(attribute, owner);
|
|
}
|
|
else
|
|
{
|
|
owners.Add(flat, attribute);
|
|
}
|
|
}
|
|
}
|
|
|
|
if (_perVertexSourcedAttributes.ContainsValue(false))
|
|
{
|
|
DeclarePerspectiveBarycentric();
|
|
}
|
|
|
|
attributes = attributes.Where(attribute => !shared.ContainsKey(attribute)).ToArray();
|
|
var locations = Gen5PixelInputMapping.ResolveLocations(
|
|
_pixelInputCntl,
|
|
attributes);
|
|
for (var index = 0; index < attributes.Length; index++)
|
|
{
|
|
var attribute = attributes[index];
|
|
// V_INTERP_MOV reads one vertex of the primitive, so its attribute is a
|
|
// per-vertex array rather than an interpolated value.
|
|
var variable = _module.AddGlobalVariable(
|
|
_perVertexAttributes.Contains(attribute)
|
|
? _module.TypePointer(SpirvStorageClass.Input, _module.TypeArray(_vec4Type, 3))
|
|
: inputVec4Pointer,
|
|
SpirvStorageClass.Input);
|
|
// VINTRP ATTR selects the PS input slot. SPI_PS_INPUT_CNTL
|
|
// maps that slot to a VS parameter export location.
|
|
var cntl = attribute < (uint)_pixelInputCntl.Length
|
|
? _pixelInputCntl[attribute]
|
|
: attribute;
|
|
_module.AddDecoration(
|
|
variable,
|
|
SpirvDecoration.Location,
|
|
locations[index]);
|
|
if (_perVertexAttributes.Contains(attribute))
|
|
{
|
|
_module.AddDecoration(variable, SpirvDecoration.PerVertexKhr);
|
|
}
|
|
else if ((cntl & 0x400u) != 0 || _flatParameterAttributes.Contains(attribute))
|
|
{
|
|
_module.AddDecoration(variable, SpirvDecoration.Flat);
|
|
}
|
|
|
|
_pixelInputs.Add(attribute, variable);
|
|
_interfaces.Add(variable);
|
|
}
|
|
|
|
foreach (var (attribute, owner) in shared)
|
|
{
|
|
_pixelInputs.Add(attribute, _pixelInputs[owner]);
|
|
}
|
|
|
|
_fragCoordInput = _module.AddGlobalVariable(
|
|
inputVec4Pointer,
|
|
SpirvStorageClass.Input);
|
|
_module.AddDecoration(
|
|
_fragCoordInput,
|
|
SpirvDecoration.BuiltIn,
|
|
(uint)SpirvBuiltIn.FragCoord);
|
|
_interfaces.Add(_fragCoordInput);
|
|
DeclarePixelSystemInputs();
|
|
|
|
var declaredPixelOutputs =
|
|
Environment.GetEnvironmentVariable(
|
|
"SHARPEMU_FORCE_TITLE_SINGLE_MRT") == "1" &&
|
|
_request.Program.Address == 0x0000000500781200ul
|
|
? _pixelOutputBindings.Take(1)
|
|
: _pixelOutputBindings;
|
|
foreach (var binding in declaredPixelOutputs)
|
|
{
|
|
var outputType = GetPixelOutputType(binding.Kind);
|
|
var outputPointer =
|
|
_module.TypePointer(SpirvStorageClass.Output, outputType);
|
|
var variable = _module.AddGlobalVariable(
|
|
outputPointer,
|
|
SpirvStorageClass.Output);
|
|
_module.AddName(variable, $"mrt{binding.GuestSlot}");
|
|
_module.AddDecoration(
|
|
variable,
|
|
SpirvDecoration.Location,
|
|
binding.HostLocation);
|
|
_pixelOutputs.Add(
|
|
binding.ExportTarget,
|
|
new SpirvPixelOutput(
|
|
variable,
|
|
outputType,
|
|
binding.Kind,
|
|
binding.ComponentMapping));
|
|
_interfaces.Add(variable);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
var inputPointer =
|
|
_module.TypePointer(SpirvStorageClass.Input, _uvec3Type);
|
|
_localInvocationIdInput = _module.AddGlobalVariable(
|
|
inputPointer,
|
|
SpirvStorageClass.Input);
|
|
_module.AddDecoration(
|
|
_localInvocationIdInput,
|
|
SpirvDecoration.BuiltIn,
|
|
(uint)SpirvBuiltIn.LocalInvocationId);
|
|
_workGroupIdInput = _module.AddGlobalVariable(
|
|
inputPointer,
|
|
SpirvStorageClass.Input);
|
|
_module.AddDecoration(
|
|
_workGroupIdInput,
|
|
SpirvDecoration.BuiltIn,
|
|
(uint)SpirvBuiltIn.WorkgroupId);
|
|
_interfaces.Add(_localInvocationIdInput);
|
|
_interfaces.Add(_workGroupIdInput);
|
|
}
|
|
}
|
|
|
|
private void DeclareVertexInputs()
|
|
{
|
|
foreach (var input in _request.VertexInputs)
|
|
{
|
|
var componentKind = input.NumberFormat switch
|
|
{
|
|
4 => VertexInputComponentKind.Uint,
|
|
5 => VertexInputComponentKind.Sint,
|
|
_ => VertexInputComponentKind.Float,
|
|
};
|
|
var componentType = componentKind switch
|
|
{
|
|
VertexInputComponentKind.Uint => _uintType,
|
|
VertexInputComponentKind.Sint => _intType,
|
|
_ => _floatType,
|
|
};
|
|
var type = input.ComponentCount switch
|
|
{
|
|
1u => componentType,
|
|
>= 2u and <= 4u =>
|
|
_module.TypeVector(componentType, input.ComponentCount),
|
|
_ => 0u,
|
|
};
|
|
if (type == 0)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
var pointer = _module.TypePointer(SpirvStorageClass.Input, type);
|
|
var variable = _module.AddGlobalVariable(
|
|
pointer,
|
|
SpirvStorageClass.Input);
|
|
_module.AddName(variable, $"attr{input.Location}");
|
|
_module.AddDecoration(
|
|
variable,
|
|
SpirvDecoration.Location,
|
|
input.Location);
|
|
var vertexInput = new SpirvVertexInput(
|
|
variable,
|
|
type,
|
|
componentType,
|
|
input.ComponentCount,
|
|
componentKind,
|
|
input.NumberFormat,
|
|
input.DestinationSelect);
|
|
_vertexInputsByPc.TryAdd(input.Pc, vertexInput);
|
|
foreach (var aliasPc in input.AliasPcs ?? [])
|
|
{
|
|
_vertexInputsByPc.TryAdd(aliasPc, vertexInput);
|
|
}
|
|
|
|
_interfaces.Add(variable);
|
|
}
|
|
}
|
|
|
|
private void EmitInitialState()
|
|
{
|
|
{
|
|
EmitLayoutInitialState();
|
|
}
|
|
|
|
Store(_scc, _module.ConstantBool(false));
|
|
Store(_reachedPixelExport, _module.ConstantBool(false));
|
|
if (_pixelValidMaskActive != 0)
|
|
{
|
|
Store(_pixelValidMaskActive, _module.ConstantBool(true));
|
|
}
|
|
if (_subgroupInvocationIdInput != 0 && _emulateWave64)
|
|
{
|
|
StoreS64(106, BothHalvesOfOwnBallot(_module.ConstantBool(false)));
|
|
StoreS64(126, BothHalvesOfOwnBallot(_module.ConstantBool(true)));
|
|
}
|
|
else if (_subgroupInvocationIdInput != 0)
|
|
{
|
|
StoreWaveMask(106, _module.ConstantBool(false));
|
|
StoreWaveMask(126, _module.ConstantBool(true));
|
|
}
|
|
else
|
|
{
|
|
// Graphics stages emulate one logical wave lane. Keep the
|
|
// guest-visible VCC/EXEC scalar pairs synchronized with the
|
|
// internal booleans: shaders commonly save EXEC from s126:s127
|
|
// and restore it after divergent work. Initializing only _exec
|
|
// left those registers at zero, so the first restore disabled
|
|
// every fragment before its color export.
|
|
StoreS64(106, _module.Constant64(_ulongType, 0));
|
|
StoreS64(126, _module.Constant64(_ulongType, 1));
|
|
}
|
|
|
|
// Primitive (NGG) vertex programs rebuild EXEC from the merged wave info in s3:
|
|
// vertex count in bits 0-7, primitive count in bits 8-15. Left at zero, EXEC
|
|
// becomes all 64 lanes, and a waterfall loop then waits forever for lanes that
|
|
// do not exist to clear their bits (Astro Bot intro_next froze the GPU this way).
|
|
if (_stage == Gen5SpirvStage.Vertex && _request.UserDataBase > 3)
|
|
{
|
|
uint laneCount;
|
|
if (_subgroupInvocationIdInput == 0)
|
|
{
|
|
laneCount = UInt(1);
|
|
}
|
|
else
|
|
{
|
|
var lanes = BooleanToWaveMask(_module.ConstantBool(true));
|
|
laneCount = IAdd(
|
|
_module.AddInstruction(SpirvOp.BitCount, _uintType, Narrow(lanes)),
|
|
_module.AddInstruction(SpirvOp.BitCount, _uintType, Narrow(ShiftRightLogical64(lanes, _module.Constant64(_ulongType, 32)))));
|
|
}
|
|
|
|
StoreS(3, BitwiseOr(laneCount, ShiftLeftLogical(laneCount, UInt(8))));
|
|
}
|
|
|
|
Store(_programCounter, UInt(0));
|
|
Store(_programActive, _module.ConstantBool(true));
|
|
|
|
if (_stage == Gen5SpirvStage.Vertex)
|
|
{
|
|
StoreV(5, Load(_uintType, _vertexIndexInput), guardWithExec: false);
|
|
StoreV(8, Load(_uintType, _instanceIndexInput), guardWithExec: false);
|
|
|
|
// Give every declared param output a defined starting value.
|
|
// Outputs the program actually exports overwrite this; the
|
|
// extras that only exist to satisfy the fragment interface stay
|
|
// zero. The explicit store also keeps SPIRV-Cross from pruning
|
|
// an unexported output (which would re-break the interface).
|
|
foreach (var output in _vertexOutputs.Values)
|
|
{
|
|
Store(output, _module.ConstantNull(_vec4Type));
|
|
}
|
|
if (_pointSizeOutput != 0)
|
|
{
|
|
Store(_pointSizeOutput, Float(0f));
|
|
}
|
|
if (_layerOutput != 0)
|
|
{
|
|
Store(_layerOutput, UInt(0));
|
|
}
|
|
if (_viewportIndexOutput != 0)
|
|
{
|
|
Store(_viewportIndexOutput, UInt(0));
|
|
}
|
|
InitializeDistanceOutput(_clipDistanceOutput, _clipDistanceCount);
|
|
InitializeDistanceOutput(_cullDistanceOutput, _cullDistanceCount);
|
|
}
|
|
else if (_stage == Gen5SpirvStage.Pixel)
|
|
{
|
|
var fragCoord = Load(_vec4Type, _fragCoordInput);
|
|
EmitPixelInputState(fragCoord);
|
|
foreach (var output in _pixelOutputs.Values)
|
|
{
|
|
Store(output.Variable, _module.ConstantNull(output.Type));
|
|
}
|
|
}
|
|
else
|
|
{
|
|
var localId = Load(_uvec3Type, _localInvocationIdInput);
|
|
var workGroupId = Load(_uvec3Type, _workGroupIdInput);
|
|
var invocationInBounds = _module.ConstantBool(true);
|
|
for (uint component = 0; component < 3; component++)
|
|
{
|
|
// gl_LocalInvocationId/gl_WorkGroupId are indexed by the physical SPIR-V
|
|
// axis, which can differ from the logical GNM axis when the workgroup was
|
|
// remapped to respect the device's tighter Z-axis limit (see
|
|
// ComputeWorkgroupAxisOrder).
|
|
var physicalComponent = (uint)_physicalAxisOfLogical[component];
|
|
var localComponent = _module.AddInstruction(
|
|
SpirvOp.CompositeExtract,
|
|
_uintType,
|
|
localId,
|
|
physicalComponent);
|
|
StoreV(component, localComponent, guardWithExec: false);
|
|
|
|
var groupComponent = _module.AddInstruction(
|
|
SpirvOp.CompositeExtract,
|
|
_uintType,
|
|
workGroupId,
|
|
physicalComponent);
|
|
var localSize = component switch
|
|
{
|
|
0 => _localSizeX,
|
|
1 => _localSizeY,
|
|
_ => _localSizeZ,
|
|
};
|
|
var globalComponent = IAdd(
|
|
_module.AddInstruction(
|
|
SpirvOp.IMul,
|
|
_uintType,
|
|
groupComponent,
|
|
UInt(localSize)),
|
|
localComponent);
|
|
// Keep inactive invocations masked when the final workgroup is partial.
|
|
var limit = ComputeThreadLimit(component);
|
|
var componentInBounds = _module.AddInstruction(
|
|
SpirvOp.ULessThan,
|
|
_boolType,
|
|
globalComponent,
|
|
limit);
|
|
invocationInBounds = _module.AddInstruction(
|
|
SpirvOp.LogicalAnd,
|
|
_boolType,
|
|
invocationInBounds,
|
|
componentInBounds);
|
|
}
|
|
|
|
Store(_programActive, invocationInBounds);
|
|
|
|
if (_request.ComputeSystemRegisters is { } registers)
|
|
{
|
|
StoreComputeSystemRegister(
|
|
registers.WorkGroupXRegister,
|
|
workGroupId,
|
|
(uint)_physicalAxisOfLogical[0]);
|
|
StoreComputeSystemRegister(
|
|
registers.WorkGroupYRegister,
|
|
workGroupId,
|
|
(uint)_physicalAxisOfLogical[1]);
|
|
StoreComputeSystemRegister(
|
|
registers.WorkGroupZRegister,
|
|
workGroupId,
|
|
(uint)_physicalAxisOfLogical[2]);
|
|
if (registers.ThreadGroupSizeRegister is { } sizeRegister)
|
|
{
|
|
StoreS(
|
|
sizeRegister,
|
|
UInt(checked(_localSizeX * _localSizeY * _localSizeZ)));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
private void EmitPixelInputState(uint fragCoord)
|
|
{
|
|
uint vgpr = 0;
|
|
|
|
// Keep input registers in SPI_PS_INPUT_ADDR order, including slots
|
|
// whose values are read through the interpolated input interface.
|
|
AdvancePixelInput(0, 2, ref vgpr); // PERSP_SAMPLE
|
|
AdvancePixelInput(1, 2, ref vgpr); // PERSP_CENTER
|
|
AdvancePixelInput(2, 2, ref vgpr); // PERSP_CENTROID
|
|
AdvancePixelInput(3, 3, ref vgpr); // PERSP_PULL_MODEL
|
|
AdvancePixelInput(4, 2, ref vgpr); // LINEAR_SAMPLE
|
|
AdvancePixelInput(5, 2, ref vgpr); // LINEAR_CENTER
|
|
AdvancePixelInput(6, 2, ref vgpr); // LINEAR_CENTROID
|
|
AdvancePixelInput(7, 1, ref vgpr); // LINE_STIPPLE
|
|
|
|
EmitPixelPositionInput(8, 0, fragCoord, ref vgpr); // POS_X_FLOAT
|
|
EmitPixelPositionInput(9, 1, fragCoord, ref vgpr); // POS_Y_FLOAT
|
|
EmitPixelPositionInput(10, 2, fragCoord, ref vgpr); // POS_Z_FLOAT
|
|
EmitPixelPositionInput(11, 3, fragCoord, ref vgpr); // POS_W_FLOAT
|
|
|
|
// FRONT_FACE, ANCILLARY, SAMPLE_COVERAGE and POS_FIXED_PT follow
|
|
// position inputs.
|
|
EmitPixelSystemInput(12, _frontFacingInput == 0 ? 0 : LoadFrontFaceInput(), ref vgpr);
|
|
EmitPixelSystemInput(13, _ancillaryLayerInput == 0 ? 0 : LoadAncillaryInput(), ref vgpr);
|
|
EmitPixelSystemInput(14, _sampleMaskInput == 0 ? 0 : LoadSampleCoverageInput(), ref vgpr);
|
|
EmitPixelSystemInput(15, (_pixelInputAddress & _pixelInputEnable & 0x8000u) == 0 ? 0 : LoadFixedPointPositionInput(fragCoord), ref vgpr);
|
|
}
|
|
|
|
private void AdvancePixelInput(int bit, uint dwordCount, ref uint vgpr)
|
|
{
|
|
if ((_pixelInputAddress & (1u << bit)) != 0)
|
|
{
|
|
// Shaders that interpolate by hand (P1/P2 on per-vertex attributes) read
|
|
// the I/J barycentrics from these registers.
|
|
if (_barycentricInputs.TryGetValue(bit, out var barycentricInput))
|
|
{
|
|
var coordinates = LoadBarycentricCoordinates(bit, barycentricInput);
|
|
for (uint component = 0; component < 2; component++)
|
|
{
|
|
var coordinate = _module.AddInstruction(
|
|
SpirvOp.CompositeExtract, _floatType, coordinates, component + 1);
|
|
StoreV(vgpr + component, Bitcast(_uintType, coordinate), guardWithExec: false);
|
|
}
|
|
}
|
|
|
|
vgpr += dwordCount;
|
|
}
|
|
}
|
|
|
|
private void EmitPixelPositionInput(
|
|
int bit,
|
|
uint component,
|
|
uint fragCoord,
|
|
ref uint vgpr)
|
|
{
|
|
var mask = 1u << bit;
|
|
if ((_pixelInputAddress & mask) == 0)
|
|
{
|
|
return;
|
|
}
|
|
|
|
if ((_pixelInputEnable & mask) != 0)
|
|
{
|
|
var value = _module.AddInstruction(
|
|
SpirvOp.CompositeExtract,
|
|
_floatType,
|
|
fragCoord,
|
|
component);
|
|
if (component == 3)
|
|
{
|
|
value = _module.AddInstruction(SpirvOp.FDiv, _floatType, Float(1f), value);
|
|
}
|
|
|
|
StoreV(vgpr, Bitcast(_uintType, value), guardWithExec: false);
|
|
}
|
|
|
|
vgpr++;
|
|
}
|
|
|
|
private void StoreComputeSystemRegister(
|
|
uint? register,
|
|
uint workGroupId,
|
|
uint component)
|
|
{
|
|
if (register is null)
|
|
{
|
|
return;
|
|
}
|
|
|
|
var value = _module.AddInstruction(
|
|
SpirvOp.CompositeExtract,
|
|
_uintType,
|
|
workGroupId,
|
|
component);
|
|
StoreS(register.Value, value);
|
|
}
|
|
|
|
private enum SharedMemoryPhase { None, Read, Write }
|
|
|
|
private bool TryEmitBlock(
|
|
IReadOnlyList<ShaderBlock> blocks,
|
|
int blockIndex,
|
|
out string error)
|
|
{
|
|
if (!TryEmitBlockBody(blocks, blockIndex, out error))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
return TryEmitBlockTerminator(blocks, blockIndex, out error);
|
|
}
|
|
|
|
// Every instruction of a block except its terminating branch or s_endpgm.
|
|
private bool TryEmitBlockBody(
|
|
IReadOnlyList<ShaderBlock> blocks,
|
|
int blockIndex,
|
|
out string error)
|
|
{
|
|
error = string.Empty;
|
|
var block = blocks[blockIndex];
|
|
var halfMaskPlan = HalfMaskPlan();
|
|
// One guest wave can span two host subgroups. Keep its shared-memory phases ordered.
|
|
// The half-mask plan inserts its own barriers; other wave64 programs also
|
|
// need LDS phase ordering when reads and writes span basic blocks.
|
|
var synchronizeSharedMemory = _emulateWave64 && halfMaskPlan is null;
|
|
var sharedMemoryPhase = SharedMemoryPhase.None;
|
|
for (var index = block.StartIndex; index < block.EndIndex; index++)
|
|
{
|
|
var instruction = _request.Program.Instructions[index];
|
|
if (halfMaskPlan is not null)
|
|
{
|
|
if (halfMaskPlan.ExactPairsBefore.TryGetValue(instruction.Pc, out var exactPairs))
|
|
{
|
|
EmitExactWaveMasks(exactPairs);
|
|
}
|
|
|
|
if (halfMaskPlan.SharedMemoryBarriersBefore.Contains(instruction.Pc))
|
|
{
|
|
EmitWave64Barrier();
|
|
}
|
|
}
|
|
|
|
if (IsBranch(instruction.Opcode) || instruction.Opcode == "SEndpgm")
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (synchronizeSharedMemory)
|
|
{
|
|
var nextPhase = instruction.Control is Gen5DataShareControl { Gds: false }
|
|
? instruction.Opcode.StartsWith("DsRead", StringComparison.Ordinal) ? SharedMemoryPhase.Read
|
|
: instruction.Opcode.StartsWith("DsWrite", StringComparison.Ordinal) ? SharedMemoryPhase.Write : SharedMemoryPhase.None
|
|
: SharedMemoryPhase.None;
|
|
if (instruction.Opcode == "SBarrier") sharedMemoryPhase = SharedMemoryPhase.None;
|
|
if (nextPhase != SharedMemoryPhase.None)
|
|
{
|
|
if (sharedMemoryPhase != SharedMemoryPhase.None && sharedMemoryPhase != nextPhase) EmitWave64Barrier();
|
|
sharedMemoryPhase = nextPhase;
|
|
}
|
|
}
|
|
|
|
_execKnownFull = IsExecKnownFull(instruction.Pc);
|
|
_emittingPc = instruction.Pc;
|
|
var emitted = TryEmitInstruction(instruction, out error);
|
|
_emittingPc = null;
|
|
_execKnownFull = false;
|
|
if (!emitted)
|
|
{
|
|
error = $"pc=0x{instruction.Pc:X} {instruction.Opcode}: {error}";
|
|
return false;
|
|
}
|
|
|
|
CapturePixelVgprs(instruction);
|
|
CapturePixelVgprPoints(instruction);
|
|
MarkPixelPath(instruction);
|
|
CapturePixelExec(instruction);
|
|
}
|
|
|
|
if (synchronizeSharedMemory && sharedMemoryPhase != SharedMemoryPhase.None) EmitWave64Barrier();
|
|
return true;
|
|
}
|
|
|
|
// Stores the next dispatcher block (or ends the program) for a block's terminator.
|
|
private bool TryEmitBlockTerminator(
|
|
IReadOnlyList<ShaderBlock> blocks,
|
|
int blockIndex,
|
|
out string error)
|
|
{
|
|
error = string.Empty;
|
|
var block = blocks[blockIndex];
|
|
var terminator = _request.Program.Instructions[block.EndIndex - 1];
|
|
if (terminator.Opcode == "SEndpgm")
|
|
{
|
|
Store(_programActive, _module.ConstantBool(false));
|
|
return true;
|
|
}
|
|
|
|
var fallthrough = blockIndex + 1 < blocks.Count
|
|
? (uint)(blockIndex + 1)
|
|
: uint.MaxValue;
|
|
if (terminator.Opcode == "SBranch")
|
|
{
|
|
if (!TryGetBranchTargetPc(terminator, out var targetPc))
|
|
{
|
|
error = "invalid scalar branch target";
|
|
return false;
|
|
}
|
|
|
|
if (IsExitBranchTarget(_request.Program.Instructions, targetPc))
|
|
{
|
|
Store(_programActive, _module.ConstantBool(false));
|
|
return true;
|
|
}
|
|
|
|
if (!TryFindBlock(blocks, targetPc, out var targetBlock))
|
|
{
|
|
error = $"invalid scalar branch target pc=0x{terminator.Pc:X} target=0x{targetPc:X} blocks={FormatBlockStarts(blocks)}";
|
|
return false;
|
|
}
|
|
|
|
Store(_programCounter, UInt((uint)targetBlock));
|
|
return true;
|
|
}
|
|
|
|
if (terminator.Opcode.StartsWith("SCbranch", StringComparison.Ordinal))
|
|
{
|
|
var hasTarget = TryGetBranchTargetPc(terminator, out var targetPc);
|
|
var targetBlock = -1;
|
|
var hasTargetBlock = hasTarget && TryFindBlock(blocks, targetPc, out targetBlock);
|
|
var targetExits = hasTarget && IsExitBranchTarget(_request.Program.Instructions, targetPc);
|
|
var hasCondition = TryGetBranchCondition(terminator.Opcode, out var condition);
|
|
if (!hasTarget || (!hasTargetBlock && !targetExits) || !hasCondition)
|
|
{
|
|
error =
|
|
$"invalid conditional scalar branch opcode={terminator.Opcode} " +
|
|
$"pc=0x{terminator.Pc:X} " +
|
|
$"target={(hasTarget ? $"0x{targetPc:X}" : "invalid")} " +
|
|
$"target_block={(hasTargetBlock ? targetBlock.ToString() : targetExits ? "exit" : "missing")} " +
|
|
$"fallthrough={(fallthrough == uint.MaxValue ? "end" : fallthrough.ToString())} " +
|
|
$"condition={hasCondition} " +
|
|
$"blocks={FormatBlockStarts(blocks)}";
|
|
return false;
|
|
}
|
|
|
|
var takenBlock = targetExits ? uint.MaxValue : (uint)targetBlock;
|
|
var selected = _module.AddInstruction(
|
|
SpirvOp.Select,
|
|
_uintType,
|
|
condition,
|
|
UInt(takenBlock),
|
|
UInt(fallthrough));
|
|
Store(_programCounter, selected);
|
|
return true;
|
|
}
|
|
|
|
if (fallthrough == uint.MaxValue)
|
|
{
|
|
Store(_programActive, _module.ConstantBool(false));
|
|
}
|
|
else
|
|
{
|
|
Store(_programCounter, UInt(fallthrough));
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
// Forward-only, properly nested control flow is emitted as structured selections instead of
|
|
// the PC-dispatch loop: a loop around a switch keeps every register live across iterations,
|
|
// blocks cannot be optimized together, and the Metal compiler spills. SHARPEMU_STRUCTURED_CF=0
|
|
// keeps the dispatcher for every program.
|
|
private static readonly bool StructuredControlFlow =
|
|
Environment.GetEnvironmentVariable("SHARPEMU_STRUCTURED_CF") != "0";
|
|
|
|
private static bool IsStructuredCondition(string opcode) => opcode is
|
|
"SCbranchScc0" or "SCbranchScc1" or "SCbranchVccz" or "SCbranchVccnz" or
|
|
"SCbranchExecz" or "SCbranchExecnz" or
|
|
"SCbranchCdbgsys" or "SCbranchCdbguser" or "SCbranchCdbgsysOrUser" or "SCbranchCdbgsysAndUser";
|
|
|
|
// The block a branch continues at, or blocks.Count for a branch past the last instruction.
|
|
private bool TryResolveBranchBlock(IReadOnlyList<ShaderBlock> blocks, Gen5ShaderInstruction branch, out int target)
|
|
{
|
|
target = -1;
|
|
if (!TryGetBranchTargetPc(branch, out var targetPc))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
if (IsExitBranchTarget(_request.Program.Instructions, targetPc))
|
|
{
|
|
target = blocks.Count;
|
|
return true;
|
|
}
|
|
|
|
return TryFindBlock(blocks, targetPc, out target);
|
|
}
|
|
|
|
// Emits (or, without emit, only checks) blocks [begin, end). A branch from the range's last
|
|
// block to leave falls through to the range end. Inside a loop body, latch is the block
|
|
// whose backward branch the caller emits as the loop's continue, and a branch to end would
|
|
// leave the loop, which a structured body cannot express.
|
|
private bool TryStructureRange(
|
|
IReadOnlyList<ShaderBlock> blocks,
|
|
int begin,
|
|
int end,
|
|
int leave,
|
|
bool emit,
|
|
out string error,
|
|
int latch = -1)
|
|
{
|
|
error = string.Empty;
|
|
var instructions = _request.Program.Instructions;
|
|
var index = begin;
|
|
while (index < end)
|
|
{
|
|
// A loop body starts at its own header, which must not open the loop again.
|
|
if (!(latch >= 0 && index == begin) && index != latch &&
|
|
TryFindLoopLatch(blocks, index, end, out var loopLatch))
|
|
{
|
|
if (!TryStructureLoop(blocks, index, loopLatch, emit, out error))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
index = loopLatch + 1;
|
|
continue;
|
|
}
|
|
|
|
if (emit && !TryEmitBlockBody(blocks, index, out error))
|
|
{
|
|
error = $"block=0x{blocks[index].StartPc:X}: {error}";
|
|
return false;
|
|
}
|
|
|
|
if (index == latch)
|
|
{
|
|
return true;
|
|
}
|
|
|
|
var terminator = instructions[blocks[index].EndIndex - 1];
|
|
var next = index + 1;
|
|
if (terminator.Opcode == "SEndpgm")
|
|
{
|
|
if (next != blocks.Count)
|
|
{
|
|
error = "s_endpgm before the last block";
|
|
return false;
|
|
}
|
|
|
|
if (emit)
|
|
{
|
|
Store(_programActive, _module.ConstantBool(false));
|
|
}
|
|
|
|
index = next;
|
|
continue;
|
|
}
|
|
|
|
if (terminator.Opcode == "SBranch")
|
|
{
|
|
if (!TryResolveBranchBlock(blocks, terminator, out var branchTarget) ||
|
|
!(branchTarget == next || (next == end && branchTarget == leave)))
|
|
{
|
|
error = $"unstructured s_branch at 0x{terminator.Pc:X}";
|
|
return false;
|
|
}
|
|
|
|
index = next;
|
|
continue;
|
|
}
|
|
|
|
if (!terminator.Opcode.StartsWith("SCbranch", StringComparison.Ordinal))
|
|
{
|
|
index = next;
|
|
continue;
|
|
}
|
|
|
|
if (!IsStructuredCondition(terminator.Opcode) ||
|
|
!TryResolveBranchBlock(blocks, terminator, out var target) || target <= index)
|
|
{
|
|
error = $"unstructured {terminator.Opcode} at 0x{terminator.Pc:X}";
|
|
return false;
|
|
}
|
|
|
|
// if (taken) break; -- only out of the innermost loop, to the block after its latch.
|
|
if (_structuredLoopExit >= 0 && target == _structuredLoopExit)
|
|
{
|
|
if (emit)
|
|
{
|
|
TryGetBranchCondition(terminator.Opcode, out var leaveLoop);
|
|
var breakLabel = _module.AllocateId();
|
|
var stayLabel = _module.AllocateId();
|
|
_module.AddStatement(SpirvOp.SelectionMerge, stayLabel, 0);
|
|
_module.AddStatement(SpirvOp.BranchConditional, leaveLoop, breakLabel, stayLabel);
|
|
_module.AddLabel(breakLabel);
|
|
_module.AddStatement(SpirvOp.Branch, _structuredLoopMerge);
|
|
_module.AddLabel(stayLabel);
|
|
}
|
|
|
|
index = next;
|
|
continue;
|
|
}
|
|
|
|
if (target > end || (latch >= 0 && target == end))
|
|
{
|
|
error = $"unstructured {terminator.Opcode} at 0x{terminator.Pc:X}";
|
|
return false;
|
|
}
|
|
|
|
if (target == next)
|
|
{
|
|
index = next;
|
|
continue;
|
|
}
|
|
|
|
// if/else: the skipped range ends with an s_branch over the taken range.
|
|
var thenLast = instructions[blocks[target - 1].EndIndex - 1];
|
|
if (target < blocks.Count && target - 1 > index && thenLast.Opcode == "SBranch" &&
|
|
TryResolveBranchBlock(blocks, thenLast, out var joinTarget) && joinTarget > target && joinTarget <= end &&
|
|
!(latch >= 0 && joinTarget == end))
|
|
{
|
|
if (!emit)
|
|
{
|
|
if (!TryStructureRange(blocks, next, target, joinTarget, emit: false, out error) ||
|
|
!TryStructureRange(blocks, target, joinTarget, joinTarget, emit: false, out error))
|
|
{
|
|
return false;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
TryGetBranchCondition(terminator.Opcode, out var taken);
|
|
var thenLabel = _module.AllocateId();
|
|
var elseLabel = _module.AllocateId();
|
|
var joinLabel = _module.AllocateId();
|
|
_module.AddStatement(SpirvOp.SelectionMerge, joinLabel, 0);
|
|
_module.AddStatement(SpirvOp.BranchConditional, taken, elseLabel, thenLabel);
|
|
_module.AddLabel(thenLabel);
|
|
if (!TryStructureRange(blocks, next, target, joinTarget, emit: true, out error))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
_module.AddStatement(SpirvOp.Branch, joinLabel);
|
|
_module.AddLabel(elseLabel);
|
|
if (!TryStructureRange(blocks, target, joinTarget, joinTarget, emit: true, out error))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
_module.AddStatement(SpirvOp.Branch, joinLabel);
|
|
_module.AddLabel(joinLabel);
|
|
}
|
|
|
|
index = joinTarget;
|
|
continue;
|
|
}
|
|
|
|
// if (!taken) { skipped range }
|
|
if (!emit)
|
|
{
|
|
if (!TryStructureRange(blocks, next, target, target, emit: false, out error))
|
|
{
|
|
return false;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
TryGetBranchCondition(terminator.Opcode, out var taken);
|
|
var bodyLabel = _module.AllocateId();
|
|
var mergeLabel = _module.AllocateId();
|
|
_module.AddStatement(SpirvOp.SelectionMerge, mergeLabel, 0);
|
|
_module.AddStatement(SpirvOp.BranchConditional, taken, mergeLabel, bodyLabel);
|
|
_module.AddLabel(bodyLabel);
|
|
if (!TryStructureRange(blocks, next, target, target, emit: true, out error))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
_module.AddStatement(SpirvOp.Branch, mergeLabel);
|
|
_module.AddLabel(mergeLabel);
|
|
}
|
|
|
|
index = target;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
// The innermost structured loop's exit block and merge label, for its breaks; -1 outside loops.
|
|
private int _structuredLoopExit = -1;
|
|
private uint _structuredLoopMerge;
|
|
|
|
// The last block in [header, end) that branches back to header, unconditionally or on a
|
|
// wave-uniform condition.
|
|
private bool TryFindLoopLatch(IReadOnlyList<ShaderBlock> blocks, int header, int end, out int latch)
|
|
{
|
|
latch = -1;
|
|
var instructions = _request.Program.Instructions;
|
|
for (var index = end - 1; index >= header; index--)
|
|
{
|
|
var terminator = instructions[blocks[index].EndIndex - 1];
|
|
if ((IsStructuredCondition(terminator.Opcode) || terminator.Opcode == "SBranch") &&
|
|
TryResolveBranchBlock(blocks, terminator, out var target) && target == header)
|
|
{
|
|
latch = index;
|
|
return true;
|
|
}
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
// do { blocks [header, latch] } while (latch condition), bounded by the same step guard as
|
|
// the dispatcher so a guest loop that never ends cannot wedge the GPU queue.
|
|
private bool TryStructureLoop(IReadOnlyList<ShaderBlock> blocks, int header, int latch, bool emit, out string error)
|
|
{
|
|
var outerExit = _structuredLoopExit;
|
|
var outerMerge = _structuredLoopMerge;
|
|
try
|
|
{
|
|
return TryStructureLoopCore(blocks, header, latch, emit, out error);
|
|
}
|
|
finally
|
|
{
|
|
_structuredLoopExit = outerExit;
|
|
_structuredLoopMerge = outerMerge;
|
|
}
|
|
}
|
|
|
|
private bool TryStructureLoopCore(IReadOnlyList<ShaderBlock> blocks, int header, int latch, bool emit, out string error)
|
|
{
|
|
_structuredLoopExit = latch + 1;
|
|
if (!emit)
|
|
{
|
|
_structuredLoopMerge = 0;
|
|
return TryStructureRange(blocks, header, latch + 1, latch + 1, emit: false, out error, latch);
|
|
}
|
|
|
|
var headerLabel = _module.AllocateId();
|
|
var bodyLabel = _module.AllocateId();
|
|
var continueLabel = _module.AllocateId();
|
|
var mergeLabel = _module.AllocateId();
|
|
_structuredLoopMerge = mergeLabel;
|
|
_module.AddStatement(SpirvOp.Branch, headerLabel);
|
|
_module.AddLabel(headerLabel);
|
|
_module.AddStatement(SpirvOp.LoopMerge, mergeLabel, continueLabel, 0);
|
|
_module.AddStatement(SpirvOp.Branch, bodyLabel);
|
|
_module.AddLabel(bodyLabel);
|
|
if (!TryStructureRange(blocks, header, latch + 1, latch + 1, emit: true, out error, latch))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
_module.AddStatement(SpirvOp.Branch, continueLabel);
|
|
_module.AddLabel(continueLabel);
|
|
var terminator = _request.Program.Instructions[blocks[latch].EndIndex - 1];
|
|
var again = _module.ConstantBool(true);
|
|
if (terminator.Opcode != "SBranch")
|
|
{
|
|
TryGetBranchCondition(terminator.Opcode, out again);
|
|
}
|
|
|
|
if (_maxDispatcherSteps > 0)
|
|
{
|
|
var steps = IAdd(Load(_uintType, _iterationGuard), UInt(1));
|
|
Store(_iterationGuard, steps);
|
|
var withinLimit = _module.AddInstruction(SpirvOp.ULessThan, _boolType, steps, UInt((uint)_maxDispatcherSteps));
|
|
again = _module.AddInstruction(SpirvOp.LogicalAnd, _boolType, again, withinLimit);
|
|
}
|
|
|
|
_module.AddStatement(SpirvOp.BranchConditional, again, headerLabel, mergeLabel);
|
|
_module.AddLabel(mergeLabel);
|
|
return true;
|
|
}
|
|
|
|
// SHARPEMU_STRUCTURED_FORWARD_BLOCKS=0 always emits the block dispatcher loop.
|
|
private static readonly bool StructuredForwardBlocks = !string.Equals(
|
|
Environment.GetEnvironmentVariable("SHARPEMU_STRUCTURED_FORWARD_BLOCKS"), "0", StringComparison.Ordinal);
|
|
|
|
// The natural loops of the program: each block targeted by a backward branch heads a
|
|
// loop that ends at the last block branching back to it. Loops must nest; a program
|
|
// whose loops overlap keeps the block dispatcher.
|
|
private Dictionary<int, int> _loopLatchByHeader = [];
|
|
|
|
private bool TryBuildLoopRegions(IReadOnlyList<ShaderBlock> blocks, out Dictionary<int, int> latchByHeader)
|
|
{
|
|
latchByHeader = [];
|
|
for (var index = 0; index < blocks.Count; index++)
|
|
{
|
|
var terminator = _request.Program.Instructions[blocks[index].EndIndex - 1];
|
|
if (terminator.Opcode != "SBranch" && !terminator.Opcode.StartsWith("SCbranch", StringComparison.Ordinal))
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (!TryGetBranchTargetPc(terminator, out var targetPc))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
if (IsExitBranchTarget(_request.Program.Instructions, targetPc))
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (!TryFindBlock(blocks, targetPc, out var target))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
if (target <= index)
|
|
{
|
|
latchByHeader[target] = latchByHeader.TryGetValue(target, out var latch) ? Math.Max(latch, index) : index;
|
|
}
|
|
}
|
|
|
|
foreach (var (outerHeader, outerLatch) in latchByHeader)
|
|
{
|
|
foreach (var (innerHeader, innerLatch) in latchByHeader)
|
|
{
|
|
if (outerHeader < innerHeader && innerHeader <= outerLatch && innerLatch > outerLatch)
|
|
{
|
|
return false;
|
|
}
|
|
}
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
// Emits blocks first..last in order; a nested loop header opens a structured loop.
|
|
private bool TryEmitStructuredRange(IReadOnlyList<ShaderBlock> blocks, int first, int last, int enclosingHeader, out string error)
|
|
{
|
|
error = string.Empty;
|
|
for (var index = first; index <= last;)
|
|
{
|
|
if (index != enclosingHeader && _loopLatchByHeader.TryGetValue(index, out var latch))
|
|
{
|
|
if (!TryEmitStructuredLoop(blocks, index, latch, out error))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
index = latch + 1;
|
|
continue;
|
|
}
|
|
|
|
var isNext = _module.AddInstruction(
|
|
SpirvOp.LogicalAnd,
|
|
_boolType,
|
|
Load(_boolType, _programActive),
|
|
_module.AddInstruction(SpirvOp.IEqual, _boolType, Load(_uintType, _programCounter), UInt((uint)index)));
|
|
var blockBody = _module.AllocateId();
|
|
var blockMerge = _module.AllocateId();
|
|
_module.AddStatement(SpirvOp.SelectionMerge, blockMerge, 0);
|
|
_module.AddStatement(SpirvOp.BranchConditional, isNext, blockBody, blockMerge);
|
|
_module.AddLabel(blockBody);
|
|
if (!TryEmitBlock(blocks, index, out error))
|
|
{
|
|
error = $"block=0x{blocks[index].StartPc:X}: {error}";
|
|
return false;
|
|
}
|
|
|
|
_module.AddStatement(SpirvOp.Branch, blockMerge);
|
|
_module.AddLabel(blockMerge);
|
|
index++;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
// One natural loop: the body runs its blocks in order and repeats while the next block
|
|
// is the header. An exit leaves the program counter past the loop (or at an outer
|
|
// header), so the guards after the loop pick up where the invocation continues.
|
|
private bool TryEmitStructuredLoop(IReadOnlyList<ShaderBlock> blocks, int header, int latch, out string error)
|
|
{
|
|
// Once a single-block loop is entered, its back edge already proves that
|
|
// this block is active and next. Keep its entry guard outside the loop;
|
|
// a conditional around every iteration obstructs driver optimization.
|
|
var singleBlock = header == latch;
|
|
var entryMerge = singleBlock ? _module.AllocateId() : 0u;
|
|
var skippedEntry = singleBlock ? _module.AllocateId() : 0u;
|
|
var loopHeader = _module.AllocateId();
|
|
var loopBody = _module.AllocateId();
|
|
var loopContinue = _module.AllocateId();
|
|
var loopMerge = _module.AllocateId();
|
|
if (singleBlock)
|
|
{
|
|
var enters = LogicalAnd(Load(_boolType, _programActive),
|
|
_module.AddInstruction(SpirvOp.IEqual, _boolType, Load(_uintType, _programCounter), UInt((uint)header)));
|
|
_module.AddStatement(SpirvOp.SelectionMerge, entryMerge, 0);
|
|
_module.AddStatement(SpirvOp.BranchConditional, enters, loopHeader, skippedEntry);
|
|
}
|
|
else
|
|
{
|
|
_module.AddStatement(SpirvOp.Branch, loopHeader);
|
|
}
|
|
_module.AddLabel(loopHeader);
|
|
_module.AddStatement(SpirvOp.LoopMerge, loopMerge, loopContinue, 0);
|
|
_module.AddStatement(SpirvOp.Branch, loopBody);
|
|
_module.AddLabel(loopBody);
|
|
if (!(singleBlock
|
|
? TryEmitBlock(blocks, header, out error)
|
|
: TryEmitStructuredRange(blocks, header, latch, header, out error)))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
_module.AddStatement(SpirvOp.Branch, loopContinue);
|
|
_module.AddLabel(loopContinue);
|
|
var again = _module.AddInstruction(
|
|
SpirvOp.LogicalAnd,
|
|
_boolType,
|
|
Load(_boolType, _programActive),
|
|
_module.AddInstruction(SpirvOp.IEqual, _boolType, Load(_uintType, _programCounter), UInt((uint)header)));
|
|
if (_maxDispatcherSteps > 0)
|
|
{
|
|
// The dispatcher's safety valve, counted per loop iteration: a mistranslated
|
|
// exit ends the invocation instead of wedging the GPU queue.
|
|
var steps = IAdd(Load(_uintType, _iterationGuard), UInt(1));
|
|
Store(_iterationGuard, steps);
|
|
var withinLimit = _module.AddInstruction(SpirvOp.ULessThan, _boolType, steps, UInt((uint)_maxDispatcherSteps));
|
|
again = _module.AddInstruction(SpirvOp.LogicalAnd, _boolType, again, withinLimit);
|
|
}
|
|
|
|
_module.AddStatement(SpirvOp.BranchConditional, again, loopHeader, loopMerge);
|
|
_module.AddLabel(loopMerge);
|
|
if (singleBlock)
|
|
{
|
|
_module.AddStatement(SpirvOp.Branch, entryMerge);
|
|
_module.AddLabel(skippedEntry);
|
|
// The previous guarded do-while still counted one iteration when
|
|
// the block was skipped. Preserve that diagnostic safety budget.
|
|
if (_maxDispatcherSteps > 0)
|
|
{
|
|
Store(_iterationGuard, IAdd(Load(_uintType, _iterationGuard), UInt(1)));
|
|
}
|
|
_module.AddStatement(SpirvOp.Branch, entryMerge);
|
|
_module.AddLabel(entryMerge);
|
|
}
|
|
return true;
|
|
}
|
|
|
|
private static string FormatBlockStarts(IReadOnlyList<ShaderBlock> blocks)
|
|
{
|
|
const int maxBlocks = 32;
|
|
var count = Math.Min(blocks.Count, maxBlocks);
|
|
var starts = new string[count];
|
|
for (var index = 0; index < count; index++)
|
|
{
|
|
starts[index] = $"0x{blocks[index].StartPc:X}";
|
|
}
|
|
|
|
return blocks.Count <= maxBlocks
|
|
? string.Join(",", starts)
|
|
: string.Join(",", starts) + $",...({blocks.Count})";
|
|
}
|
|
|
|
private static bool IsExitBranchTarget(
|
|
IReadOnlyList<Gen5ShaderInstruction> instructions,
|
|
uint targetPc)
|
|
{
|
|
if (instructions.Count == 0)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
var last = instructions[^1];
|
|
var lastEndPc = last.Pc + (uint)(last.Words.Count * sizeof(uint));
|
|
return targetPc >= lastEndPc;
|
|
}
|
|
|
|
private bool TryGetBranchCondition(string opcode, out uint condition)
|
|
{
|
|
condition = opcode switch
|
|
{
|
|
"SCbranchScc0" => LogicalNot(Load(_boolType, _scc)),
|
|
"SCbranchScc1" => Load(_boolType, _scc),
|
|
"SCbranchVccz" => LogicalNot(WaveMaskAny(106, _vcc)),
|
|
"SCbranchVccnz" => WaveMaskAny(106, _vcc),
|
|
"SCbranchExecz" => LogicalNot(WaveMaskAny(126, _exec)),
|
|
"SCbranchExecnz" => WaveMaskAny(126, _exec),
|
|
// The emulator does not expose a shader debug session.
|
|
"SCbranchCdbgsys" or
|
|
"SCbranchCdbguser" or
|
|
"SCbranchCdbgsysOrUser" or
|
|
"SCbranchCdbgsysAndUser" => _module.ConstantBool(false),
|
|
_ => 0,
|
|
};
|
|
return condition != 0;
|
|
}
|
|
|
|
private bool TryEmitInstruction(
|
|
Gen5ShaderInstruction instruction,
|
|
out string error)
|
|
{
|
|
error = string.Empty;
|
|
// No shader trap handler is installed, so S_TRAP has no effect.
|
|
if (instruction.Opcode == "STrap")
|
|
{
|
|
return true;
|
|
}
|
|
if (instruction.Opcode is
|
|
"SNop" or
|
|
"SSetregB32" or
|
|
"SWaitcnt" or
|
|
"SInstPrefetch" or
|
|
"STtraceData" or
|
|
// Wave scheduling priority hint; no effect on results.
|
|
"SSetprio" or
|
|
// NGG shaders bracket their exports with s_sendmsg
|
|
// (GS_ALLOC_REQ/DEALLOC) to reserve hardware export space;
|
|
// exports are translated directly, so the message is moot.
|
|
"SSendmsg")
|
|
{
|
|
return true;
|
|
}
|
|
|
|
if (instruction.Opcode == "SBarrier")
|
|
{
|
|
if (_stage == Gen5SpirvStage.Compute)
|
|
{
|
|
// s_waitcnt vmcnt(0) + s_barrier also publishes buffer and image
|
|
// stores to the workgroup: AcquireRelease over uniform, workgroup
|
|
// and image memory.
|
|
var workgroup = UInt(2);
|
|
var semantics = UInt(0x8 | 0x40 | 0x100 | 0x800);
|
|
_module.AddStatement(
|
|
SpirvOp.ControlBarrier,
|
|
workgroup,
|
|
workgroup,
|
|
semantics);
|
|
}
|
|
return true;
|
|
}
|
|
|
|
if (instruction.Control is Gen5ScalarMemoryControl scalarMemory)
|
|
{
|
|
return TryEmitScalarMemory(instruction, scalarMemory, out error);
|
|
}
|
|
|
|
if (instruction.Control is Gen5InterpolationControl interpolation)
|
|
{
|
|
return TryEmitInterpolation(instruction, interpolation, out error);
|
|
}
|
|
|
|
if (instruction.Control is Gen5ImageControl image)
|
|
{
|
|
return TryEmitImage(instruction, image, out error);
|
|
}
|
|
|
|
if (instruction.Control is Gen5RayIntersectControl rayIntersect)
|
|
{
|
|
_deviceAddressInstructionPc = instruction.Pc;
|
|
EmitRayIntersect(rayIntersect, instruction.Opcode == "ImageBvh64IntersectRay");
|
|
return true;
|
|
}
|
|
|
|
if (instruction.Control is Gen5GlobalMemoryControl globalMemory)
|
|
{
|
|
return TryEmitGlobalMemory(instruction, globalMemory, out error);
|
|
}
|
|
|
|
if (instruction.Control is Gen5BufferMemoryControl bufferMemory)
|
|
{
|
|
return TryEmitBufferMemory(instruction, bufferMemory, out error);
|
|
}
|
|
|
|
if (instruction.Control is Gen5ExportControl export)
|
|
{
|
|
return TryEmitExport(instruction, export, out error);
|
|
}
|
|
|
|
if (instruction.Control is Gen5DataShareControl)
|
|
{
|
|
return TryEmitDataShare(instruction, out error);
|
|
}
|
|
|
|
if (instruction.Encoding is
|
|
Gen5ShaderEncoding.Sop1 or
|
|
Gen5ShaderEncoding.Sop2 or
|
|
Gen5ShaderEncoding.Sopc or
|
|
Gen5ShaderEncoding.Sopk)
|
|
{
|
|
return TryEmitScalarAlu(instruction, out error);
|
|
}
|
|
|
|
if (instruction.Encoding is
|
|
Gen5ShaderEncoding.Sopp or
|
|
Gen5ShaderEncoding.Smrd or
|
|
Gen5ShaderEncoding.Smem)
|
|
{
|
|
return true;
|
|
}
|
|
|
|
return TryEmitVectorAlu(instruction, out error);
|
|
}
|
|
|
|
private bool TryEmitDataShare(
|
|
Gen5ShaderInstruction instruction,
|
|
out string error)
|
|
{
|
|
error = string.Empty;
|
|
if (instruction.Control is Gen5DataShareControl { Gds: true } globalShare)
|
|
{
|
|
return TryEmitGlobalDataShare(instruction, globalShare, out error);
|
|
}
|
|
|
|
if (instruction.Control is not Gen5DataShareControl control)
|
|
{
|
|
error = "invalid LDS instruction";
|
|
return false;
|
|
}
|
|
|
|
switch (instruction.Opcode)
|
|
{
|
|
case "DsSwizzleB32":
|
|
return TryEmitDataShareSwizzle(instruction, control, out error);
|
|
case "DsBpermuteB32":
|
|
return TryEmitDataShareBpermute(instruction, control, out error);
|
|
}
|
|
|
|
if (_lds == 0 || _ldsElementPointer == 0)
|
|
{
|
|
error = "invalid LDS instruction";
|
|
return false;
|
|
}
|
|
|
|
switch (instruction.Opcode)
|
|
{
|
|
case "DsAppend":
|
|
case "DsConsume":
|
|
return TryEmitDataShareWaveCounter(instruction, control, out error);
|
|
case "DsMskorB32":
|
|
return TryEmitDataShareAtomic(instruction, control, out error);
|
|
case "DsWriteB32":
|
|
case "DsWriteAddtidB32":
|
|
{
|
|
if (instruction.Sources.Count < 2)
|
|
{
|
|
error = "missing LDS write source";
|
|
return false;
|
|
}
|
|
|
|
var address = GetRawSource(instruction, 0);
|
|
if (instruction.Opcode == "DsWriteAddtidB32")
|
|
{
|
|
address = IAdd(BitwiseAnd(address, UInt(0xFFFF)), ShiftLeftLogical(GuestWaveLane(), UInt(2)));
|
|
}
|
|
StoreLds(
|
|
LdsPointer(address, control.SingleOffsetBytes),
|
|
GetRawSource(instruction, 1));
|
|
return true;
|
|
}
|
|
case "DsWriteB64":
|
|
{
|
|
if (instruction.Sources.Count < 3)
|
|
{
|
|
error = "missing LDS write64 source";
|
|
return false;
|
|
}
|
|
|
|
var address = GetRawSource(instruction, 0);
|
|
var offset = control.SingleOffsetBytes;
|
|
StoreLds(LdsPointer(address, offset), GetRawSource(instruction, 1));
|
|
StoreLds(
|
|
LdsPointer(address, offset + sizeof(uint)),
|
|
GetRawSource(instruction, 2));
|
|
return true;
|
|
}
|
|
case "DsAddU64":
|
|
case "DsOrB64":
|
|
{
|
|
if (instruction.Sources.Count < 3)
|
|
{
|
|
error = "missing LDS 64-bit atomic source";
|
|
return false;
|
|
}
|
|
|
|
var atomicAddress = GetRawSource(instruction, 0);
|
|
var isOr = instruction.Opcode == "DsOrB64";
|
|
// The LDS allocation is a uint array. Reinterpreting its
|
|
// pointer as ulong with OpBitcast is invalid SPIR-V, so use
|
|
// the dword fallback even when native 64-bit atomics exist.
|
|
// The pair is not atomic against concurrent 64-bit use.
|
|
var lowPointer = LdsPointer(
|
|
atomicAddress,
|
|
control.SingleOffsetBytes);
|
|
var highPointer = LdsPointer(
|
|
atomicAddress,
|
|
control.SingleOffsetBytes + sizeof(uint));
|
|
EmitExecConditional(() =>
|
|
{
|
|
var lowValue = GetRawSource(instruction, 1);
|
|
var highValue = GetRawSource(instruction, 2);
|
|
if (isOr)
|
|
{
|
|
// OR is bitwise-independent: no dword influences the
|
|
// other, so two 32-bit ORs are the 64-bit OR.
|
|
EmitAtomic(
|
|
SpirvOp.AtomicOr,
|
|
_uintType,
|
|
lowPointer,
|
|
scope: 2,
|
|
semantics: 0x108,
|
|
value: () => lowValue,
|
|
comparator: () => lowValue);
|
|
EmitAtomic(
|
|
SpirvOp.AtomicOr,
|
|
_uintType,
|
|
highPointer,
|
|
scope: 2,
|
|
semantics: 0x108,
|
|
value: () => highValue,
|
|
comparator: () => highValue);
|
|
return;
|
|
}
|
|
|
|
// ADD needs the carry out of the low dword folded into
|
|
// the high one. Each lane detects its own wrap from the
|
|
// pre-add value its atomic returned and adds exactly one
|
|
// carry, so the high dword ends up with the true count of
|
|
// wraps however the lanes interleave.
|
|
var originalLow = EmitAtomic(
|
|
SpirvOp.AtomicIAdd,
|
|
_uintType,
|
|
lowPointer,
|
|
scope: 2,
|
|
semantics: 0x108,
|
|
value: () => lowValue,
|
|
comparator: () => lowValue);
|
|
var sumLow = IAdd(originalLow, lowValue);
|
|
var wrapped = _module.AddInstruction(
|
|
SpirvOp.ULessThan,
|
|
_boolType,
|
|
sumLow,
|
|
originalLow);
|
|
var carry = _module.AddInstruction(
|
|
SpirvOp.Select,
|
|
_uintType,
|
|
wrapped,
|
|
UInt(1),
|
|
UInt(0));
|
|
var highWithCarry = IAdd(highValue, carry);
|
|
EmitAtomic(
|
|
SpirvOp.AtomicIAdd,
|
|
_uintType,
|
|
highPointer,
|
|
scope: 2,
|
|
semantics: 0x108,
|
|
value: () => highWithCarry,
|
|
comparator: () => highWithCarry);
|
|
});
|
|
|
|
return true;
|
|
}
|
|
case "DsWriteB96":
|
|
case "DsWriteB128":
|
|
{
|
|
// ds_write_b96 stores 3 consecutive dwords, ds_write_b128
|
|
// stores 4, from data0..data0+N at the address's offset.
|
|
var dwordCount = instruction.Opcode == "DsWriteB128" ? 4 : 3;
|
|
if (instruction.Sources.Count < 1 + dwordCount)
|
|
{
|
|
error = "missing LDS write128 source";
|
|
return false;
|
|
}
|
|
|
|
var address = GetRawSource(instruction, 0);
|
|
var offset = control.SingleOffsetBytes;
|
|
for (var dword = 0; dword < dwordCount; dword++)
|
|
{
|
|
StoreLds(
|
|
LdsPointer(address, offset + (uint)(dword * sizeof(uint))),
|
|
GetRawSource(instruction, 1 + dword));
|
|
}
|
|
|
|
return true;
|
|
}
|
|
case "DsWrite2B64":
|
|
case "DsWrite2St64B64":
|
|
return TryEmitDataShareWritePair64(instruction, control, out error);
|
|
case "DsWrite2B32":
|
|
case "DsWrite2St64B32":
|
|
{
|
|
if (instruction.Sources.Count < 3)
|
|
{
|
|
error = "missing LDS write2 source";
|
|
return false;
|
|
}
|
|
|
|
var st64 = instruction.Opcode == "DsWrite2St64B32";
|
|
var address = GetRawSource(instruction, 0);
|
|
StoreLds(
|
|
LdsPointer(
|
|
address,
|
|
EffectiveDsPairOffsetBytes(control.Offset0, st64)),
|
|
GetRawSource(instruction, 1));
|
|
StoreLds(
|
|
LdsPointer(
|
|
address,
|
|
EffectiveDsPairOffsetBytes(control.Offset1, st64)),
|
|
GetRawSource(instruction, 2));
|
|
return true;
|
|
}
|
|
case "DsReadB32":
|
|
case "DsReadAddtidB32":
|
|
{
|
|
if (instruction.Destinations.Count < 1 ||
|
|
instruction.Sources.Count < 1)
|
|
{
|
|
error = "missing LDS read operand";
|
|
return false;
|
|
}
|
|
|
|
var address = GetRawSource(instruction, 0);
|
|
if (instruction.Opcode == "DsReadAddtidB32")
|
|
{
|
|
address = IAdd(BitwiseAnd(address, UInt(0xFFFF)), ShiftLeftLogical(GuestWaveLane(), UInt(2)));
|
|
}
|
|
var value = Load(
|
|
_uintType,
|
|
LdsPointer(address, control.SingleOffsetBytes));
|
|
StoreV(instruction.Destinations[0].Value, value);
|
|
return true;
|
|
}
|
|
case "DsReadI8":
|
|
{
|
|
if (instruction.Destinations.Count < 1 || instruction.Sources.Count < 1)
|
|
{
|
|
error = "missing LDS signed byte read operand";
|
|
return false;
|
|
}
|
|
|
|
var address = GetRawSource(instruction, 0);
|
|
var byteAddress = control.SingleOffsetBytes == 0
|
|
? address
|
|
: IAdd(address, UInt(control.SingleOffsetBytes));
|
|
var word = Load(_uintType, LdsPointer(address, control.SingleOffsetBytes));
|
|
var shift = ShiftLeftLogical(BitwiseAnd(byteAddress, UInt(3)), UInt(3));
|
|
var packed = ShiftRightLogical(word, shift);
|
|
var signedByte = _module.AddInstruction(
|
|
SpirvOp.BitFieldSExtract,
|
|
_intType,
|
|
Bitcast(_intType, packed),
|
|
UInt(0),
|
|
UInt(8));
|
|
StoreV(instruction.Destinations[0].Value, Bitcast(_uintType, signedByte));
|
|
return true;
|
|
}
|
|
case "DsReadB64":
|
|
case "DsReadB96":
|
|
case "DsReadB128":
|
|
{
|
|
var dwordCount = instruction.Opcode switch { "DsReadB64" => 2, "DsReadB96" => 3, _ => 4 };
|
|
if (instruction.Destinations.Count < dwordCount ||
|
|
instruction.Sources.Count < 1)
|
|
{
|
|
error = "missing LDS read operand";
|
|
return false;
|
|
}
|
|
|
|
var address = GetRawSource(instruction, 0);
|
|
var offset = control.SingleOffsetBytes;
|
|
for (var dword = 0; dword < dwordCount; dword++)
|
|
{
|
|
var value = Load(
|
|
_uintType,
|
|
LdsPointer(address, offset + (uint)(dword * sizeof(uint))));
|
|
StoreV(instruction.Destinations[dword].Value, value);
|
|
}
|
|
|
|
return true;
|
|
}
|
|
case "DsRead2B64":
|
|
return TryEmitDataShareReadPair64(instruction, control, out error);
|
|
case "DsRead2B32":
|
|
case "DsRead2St64B32":
|
|
{
|
|
if (instruction.Destinations.Count < 2 ||
|
|
instruction.Sources.Count < 1)
|
|
{
|
|
error = "missing LDS read2 operand";
|
|
return false;
|
|
}
|
|
|
|
var st64 = instruction.Opcode == "DsRead2St64B32";
|
|
var address = GetRawSource(instruction, 0);
|
|
var first = Load(
|
|
_uintType,
|
|
LdsPointer(
|
|
address,
|
|
EffectiveDsPairOffsetBytes(control.Offset0, st64)));
|
|
var second = Load(
|
|
_uintType,
|
|
LdsPointer(
|
|
address,
|
|
EffectiveDsPairOffsetBytes(control.Offset1, st64)));
|
|
StoreV(instruction.Destinations[0].Value, first);
|
|
StoreV(instruction.Destinations[1].Value, second);
|
|
return true;
|
|
}
|
|
default:
|
|
if (Gen5ShaderTranslator.IsDataShareAtomic(instruction.Opcode))
|
|
{
|
|
return TryEmitDataShareAtomic(instruction, control, out error);
|
|
}
|
|
|
|
error = $"unsupported LDS opcode {instruction.Opcode}";
|
|
return false;
|
|
}
|
|
}
|
|
|
|
private static uint EffectiveDsPairOffsetBytes(uint offset, bool st64 = false) =>
|
|
offset * (st64 ? 256u : sizeof(uint));
|
|
|
|
private bool TryEmitDataShareWaveCounter(
|
|
Gen5ShaderInstruction instruction,
|
|
Gen5DataShareControl control,
|
|
out string error)
|
|
{
|
|
error = string.Empty;
|
|
if (instruction.Sources.Count < 1 || instruction.Destinations.Count < 1)
|
|
{
|
|
error = $"missing {instruction.Opcode} operand";
|
|
return false;
|
|
}
|
|
|
|
var offset = control.SingleOffsetBytes;
|
|
var m0 = GetRawSource(instruction, 0);
|
|
var baseAddress = ShiftRightLogical(m0, UInt(16));
|
|
var sizeBytes = BitwiseAnd(m0, UInt(0xFFFF));
|
|
var inBounds = _module.AddInstruction(
|
|
SpirvOp.ULessThan,
|
|
_boolType,
|
|
UInt(offset + 3),
|
|
sizeBytes);
|
|
var pointer = LdsPointer(baseAddress, offset);
|
|
var destination = instruction.Destinations[0].Value;
|
|
var active = Load(_boolType, _exec);
|
|
|
|
var activeMask = BooleanToWaveMask(active);
|
|
var activeLow = _module.AddInstruction(
|
|
SpirvOp.UConvert,
|
|
_uintType,
|
|
activeMask);
|
|
var activeHigh = _module.AddInstruction(
|
|
SpirvOp.UConvert,
|
|
_uintType,
|
|
ShiftRightLogical64(
|
|
activeMask,
|
|
_module.Constant64(_ulongType, 32)));
|
|
// OpBitCount needs a 32-bit operand without maintenance9.
|
|
var activeCount = IAdd(
|
|
_module.AddInstruction(SpirvOp.BitCount, _uintType, activeLow),
|
|
_module.AddInstruction(SpirvOp.BitCount, _uintType, activeHigh));
|
|
var firstLane = _module.AddInstruction(
|
|
SpirvOp.Select,
|
|
_uintType,
|
|
IsNotZero(activeLow),
|
|
Ext(73, _uintType, activeLow),
|
|
IAdd(Ext(73, _uintType, activeHigh), UInt(32)));
|
|
var isFirstActive = _module.AddInstruction(
|
|
SpirvOp.LogicalAnd,
|
|
_boolType,
|
|
inBounds,
|
|
_module.AddInstruction(
|
|
SpirvOp.LogicalAnd,
|
|
_boolType,
|
|
active,
|
|
_module.AddInstruction(
|
|
SpirvOp.IEqual,
|
|
_boolType,
|
|
GuestWaveLane(),
|
|
firstLane)));
|
|
|
|
EmitConditional(isFirstActive, () =>
|
|
{
|
|
uint original;
|
|
if (_stage == Gen5SpirvStage.Compute)
|
|
{
|
|
original = EmitAtomic(
|
|
instruction.Opcode == "DsAppend"
|
|
? SpirvOp.AtomicIAdd
|
|
: SpirvOp.AtomicISub,
|
|
_uintType,
|
|
pointer,
|
|
scope: 2,
|
|
semantics: 0x108,
|
|
value: () => activeCount,
|
|
comparator: () => UInt(0));
|
|
}
|
|
else
|
|
{
|
|
// Vulkan graphics stages cannot use Workgroup storage.
|
|
// Keep the counter in the first active lane's private LDS
|
|
// model, but preserve the RDNA2 wave operation: change it
|
|
// by the active-lane count and broadcast the old value.
|
|
original = Load(_uintType, pointer);
|
|
var changed = instruction.Opcode == "DsAppend"
|
|
? IAdd(original, activeCount)
|
|
: _module.AddInstruction(
|
|
SpirvOp.ISub,
|
|
_uintType,
|
|
original,
|
|
activeCount);
|
|
Store(pointer, changed);
|
|
}
|
|
|
|
StoreV(destination, original);
|
|
});
|
|
|
|
var firstValue = LoadV(destination);
|
|
uint broadcast;
|
|
if (_emulateWave64)
|
|
{
|
|
broadcast = ExchangeWave64Value(isFirstActive, firstValue);
|
|
}
|
|
else
|
|
{
|
|
broadcast = ShuffleLane(firstValue, firstLane);
|
|
}
|
|
|
|
var validResult = _module.AddInstruction(
|
|
SpirvOp.LogicalAnd,
|
|
_boolType,
|
|
inBounds,
|
|
IsNotZero64(activeMask));
|
|
StoreV(
|
|
destination,
|
|
_module.AddInstruction(
|
|
SpirvOp.Select,
|
|
_uintType,
|
|
validResult,
|
|
broadcast,
|
|
UInt(0)));
|
|
return true;
|
|
}
|
|
|
|
private uint LdsPointer(uint address, uint offsetBytes)
|
|
{
|
|
var addressWithOffset = offsetBytes == 0
|
|
? address
|
|
: IAdd(address, UInt(offsetBytes));
|
|
// Mask the dword index into the array bounds. LDS is a power-of-two
|
|
// dword count, so this is a no-op for in-range compute addresses but
|
|
// prevents out-of-bounds access when a graphics-stage scratch write
|
|
// uses an arbitrary computed byte address.
|
|
var index = BitwiseAnd(
|
|
ShiftRightLogical(addressWithOffset, UInt(2)),
|
|
UInt(_ldsDwordMask));
|
|
return _module.AddInstruction(
|
|
SpirvOp.AccessChain,
|
|
_ldsElementPointer,
|
|
_lds,
|
|
index);
|
|
}
|
|
|
|
private void StoreLds(uint pointer, uint value)
|
|
{
|
|
var active = Load(_boolType, _exec);
|
|
// Inactive lanes must not read and write back another lane's shared value.
|
|
EmitConditional(active, () => Store(pointer, value));
|
|
}
|
|
|
|
private bool TryEmitDataShareAtomic(
|
|
Gen5ShaderInstruction instruction,
|
|
Gen5DataShareControl control,
|
|
out string error)
|
|
{
|
|
error = string.Empty;
|
|
if (instruction.Opcode == "DsMskorB32")
|
|
{
|
|
if (instruction.Sources.Count < 3)
|
|
{
|
|
error = "missing LDS masked-OR source";
|
|
return false;
|
|
}
|
|
|
|
var maskedPointer = LdsPointer(
|
|
GetRawSource(instruction, 0),
|
|
control.SingleOffsetBytes);
|
|
EmitExecConditional(() =>
|
|
{
|
|
var original = Load(_uintType, maskedPointer);
|
|
var updated = BitwiseOr(
|
|
BitwiseAnd(
|
|
original,
|
|
_module.AddInstruction(
|
|
SpirvOp.Not,
|
|
_uintType,
|
|
GetRawSource(instruction, 1))),
|
|
GetRawSource(instruction, 2));
|
|
EmitAtomic(
|
|
SpirvOp.AtomicCompareExchange,
|
|
_uintType,
|
|
maskedPointer,
|
|
scope: 2,
|
|
semantics: 0x108,
|
|
value: () => updated,
|
|
comparator: () => original);
|
|
});
|
|
return true;
|
|
}
|
|
|
|
if (instruction.Opcode is "DsMinF32" or "DsMaxF32")
|
|
{
|
|
if (instruction.Sources.Count < 3)
|
|
{
|
|
error = $"missing LDS operands for {instruction.Opcode}";
|
|
return false;
|
|
}
|
|
|
|
var floatAddress = GetRawSource(instruction, 0);
|
|
var floatPointer = LdsPointer(floatAddress, control.SingleOffsetBytes);
|
|
EmitExecConditional(() =>
|
|
EmitDataShareFloatAtomic(
|
|
floatPointer,
|
|
GetRawSource(instruction, 1),
|
|
GetRawSource(instruction, 2),
|
|
instruction.Opcode == "DsMaxF32",
|
|
scope: 2,
|
|
semantics: 0x108));
|
|
return true;
|
|
}
|
|
|
|
var atomicOp = instruction.Opcode switch
|
|
{
|
|
"DsAddU32" or "DsAddRtnU32" => SpirvOp.AtomicIAdd,
|
|
"DsSubU32" or "DsSubRtnU32" => SpirvOp.AtomicISub,
|
|
"DsIncU32" or "DsIncRtnU32" => SpirvOp.AtomicIIncrement,
|
|
"DsDecU32" or "DsDecRtnU32" => SpirvOp.AtomicIDecrement,
|
|
"DsMinI32" or "DsMinRtnI32" => SpirvOp.AtomicSMin,
|
|
"DsMaxI32" or "DsMaxRtnI32" => SpirvOp.AtomicSMax,
|
|
"DsMinU32" or "DsMinRtnU32" => SpirvOp.AtomicUMin,
|
|
"DsMaxU32" or "DsMaxRtnU32" => SpirvOp.AtomicUMax,
|
|
"DsAndB32" or "DsAndRtnB32" => SpirvOp.AtomicAnd,
|
|
"DsOrB32" or "DsOrRtnB32" => SpirvOp.AtomicOr,
|
|
"DsXorB32" or "DsXorRtnB32" => SpirvOp.AtomicXor,
|
|
"DsWrxchgRtnB32" => SpirvOp.AtomicExchange,
|
|
"DsCmpstB32" or "DsCmpstRtnB32" => SpirvOp.AtomicCompareExchange,
|
|
_ => SpirvOp.Nop,
|
|
};
|
|
if (atomicOp == SpirvOp.Nop)
|
|
{
|
|
error = $"unsupported LDS opcode {instruction.Opcode}";
|
|
return false;
|
|
}
|
|
|
|
var address = GetRawSource(instruction, 0);
|
|
var pointer = LdsPointer(address, control.SingleOffsetBytes);
|
|
EmitExecConditional(() =>
|
|
{
|
|
var original = EmitAtomic(
|
|
atomicOp,
|
|
_uintType,
|
|
pointer,
|
|
scope: 2,
|
|
semantics: 0x108,
|
|
// DS_CMPST sources: DATA0 is the comparator, DATA1 the new value.
|
|
value: () => GetRawSource(
|
|
instruction,
|
|
atomicOp == SpirvOp.AtomicCompareExchange ? 2 : 1),
|
|
comparator: () => GetRawSource(instruction, 1));
|
|
if (instruction.Destinations.Count > 0)
|
|
{
|
|
StoreV(instruction.Destinations[0].Value, original);
|
|
}
|
|
});
|
|
|
|
return true;
|
|
}
|
|
|
|
private void EmitDataShareFloatAtomic(
|
|
uint pointer,
|
|
uint data,
|
|
uint compare,
|
|
bool maxValue,
|
|
uint scope,
|
|
uint semantics)
|
|
{
|
|
var preheader = _module.AllocateId();
|
|
var header = _module.AllocateId();
|
|
var continueLabel = _module.AllocateId();
|
|
var mergeLabel = _module.AllocateId();
|
|
var exchanged = _module.AllocateId();
|
|
|
|
_module.AddStatement(SpirvOp.Branch, preheader);
|
|
_module.AddLabel(preheader);
|
|
var initial = _module.AddInstruction(
|
|
SpirvOp.AtomicLoad,
|
|
_uintType,
|
|
pointer,
|
|
UInt(scope),
|
|
UInt((semantics & ~0xCu) | 0x2u));
|
|
_module.AddStatement(SpirvOp.Branch, header);
|
|
|
|
_module.AddLabel(header);
|
|
var observed = _module.AddInstruction(
|
|
SpirvOp.Phi,
|
|
_uintType,
|
|
initial,
|
|
preheader,
|
|
exchanged,
|
|
continueLabel);
|
|
var observedFloat = Bitcast(_floatType, observed);
|
|
var compareFloat = Bitcast(_floatType, compare);
|
|
var replace = _module.AddInstruction(
|
|
maxValue ? SpirvOp.FOrdGreaterThan : SpirvOp.FOrdLessThan,
|
|
_boolType,
|
|
maxValue ? observedFloat : compareFloat,
|
|
maxValue ? compareFloat : observedFloat);
|
|
var next = _module.AddInstruction(SpirvOp.Select, _uintType, replace, data, observed);
|
|
|
|
_module.AddStatement(
|
|
SpirvOp.AtomicCompareExchange,
|
|
_uintType,
|
|
exchanged,
|
|
pointer,
|
|
UInt(scope),
|
|
UInt(semantics),
|
|
UInt((semantics & ~0x8u) | 0x2u),
|
|
next,
|
|
observed);
|
|
var success = _module.AddInstruction(SpirvOp.IEqual, _boolType, exchanged, observed);
|
|
_module.AddStatement(SpirvOp.LoopMerge, mergeLabel, continueLabel, 0);
|
|
_module.AddStatement(SpirvOp.BranchConditional, success, mergeLabel, continueLabel);
|
|
_module.AddLabel(continueLabel);
|
|
_module.AddStatement(SpirvOp.Branch, header);
|
|
_module.AddLabel(mergeLabel);
|
|
}
|
|
|
|
private uint EmitBufferFloatAtomic(
|
|
uint pointer,
|
|
uint value,
|
|
bool maxValue,
|
|
uint scope,
|
|
uint semantics)
|
|
{
|
|
var preheader = _module.AllocateId();
|
|
var header = _module.AllocateId();
|
|
var continueLabel = _module.AllocateId();
|
|
var mergeLabel = _module.AllocateId();
|
|
var exchanged = _module.AllocateId();
|
|
|
|
_module.AddStatement(SpirvOp.Branch, preheader);
|
|
_module.AddLabel(preheader);
|
|
var initial = _module.AddInstruction(
|
|
SpirvOp.AtomicLoad,
|
|
_uintType,
|
|
pointer,
|
|
UInt(scope),
|
|
UInt((semantics & ~0xCu) | 0x2u));
|
|
_module.AddStatement(SpirvOp.Branch, header);
|
|
|
|
_module.AddLabel(header);
|
|
var observed = _module.AddInstruction(
|
|
SpirvOp.Phi,
|
|
_uintType,
|
|
initial,
|
|
preheader,
|
|
exchanged,
|
|
continueLabel);
|
|
var observedFloat = Bitcast(_floatType, observed);
|
|
var valueFloat = Bitcast(_floatType, value);
|
|
var replace = _module.AddInstruction(
|
|
maxValue ? SpirvOp.FOrdGreaterThan : SpirvOp.FOrdLessThan,
|
|
_boolType,
|
|
valueFloat,
|
|
observedFloat);
|
|
var next = _module.AddInstruction(SpirvOp.Select, _uintType, replace, value, observed);
|
|
|
|
_module.AddStatement(
|
|
SpirvOp.AtomicCompareExchange,
|
|
_uintType,
|
|
exchanged,
|
|
pointer,
|
|
UInt(scope),
|
|
UInt(semantics),
|
|
UInt((semantics & ~0x8u) | 0x2u),
|
|
next,
|
|
observed);
|
|
var success = _module.AddInstruction(SpirvOp.IEqual, _boolType, exchanged, observed);
|
|
_module.AddStatement(SpirvOp.LoopMerge, mergeLabel, continueLabel, 0);
|
|
_module.AddStatement(SpirvOp.BranchConditional, success, mergeLabel, continueLabel);
|
|
_module.AddLabel(continueLabel);
|
|
_module.AddStatement(SpirvOp.Branch, header);
|
|
_module.AddLabel(mergeLabel);
|
|
return observed;
|
|
}
|
|
|
|
// Maps the AMD atomic-op name suffix shared by buffer/image atomics to a SPIR-V opcode.
|
|
// Inc/Dec approximate the AMD wrap-clamp semantics (MEM = tmp >= DATA ? 0 : tmp + 1),
|
|
// which is exact for the common 0xFFFFFFFF clamp operand.
|
|
private static bool TryGetAtomicOp(string name, out SpirvOp op)
|
|
{
|
|
op = name switch
|
|
{
|
|
"Swap" => SpirvOp.AtomicExchange,
|
|
"Cmpswap" => SpirvOp.AtomicCompareExchange,
|
|
"Add" => SpirvOp.AtomicIAdd,
|
|
"Sub" => SpirvOp.AtomicISub,
|
|
"Smin" => SpirvOp.AtomicSMin,
|
|
"Umin" => SpirvOp.AtomicUMin,
|
|
"Smax" => SpirvOp.AtomicSMax,
|
|
"Umax" or "UMax" => SpirvOp.AtomicUMax,
|
|
"And" => SpirvOp.AtomicAnd,
|
|
"Or" => SpirvOp.AtomicOr,
|
|
"Xor" => SpirvOp.AtomicXor,
|
|
"Inc" => SpirvOp.AtomicIIncrement,
|
|
"Dec" => SpirvOp.AtomicIDecrement,
|
|
_ => SpirvOp.Nop,
|
|
};
|
|
return op != SpirvOp.Nop;
|
|
}
|
|
|
|
private uint EmitAtomic(
|
|
SpirvOp op,
|
|
uint type,
|
|
uint pointer,
|
|
uint scope,
|
|
uint semantics,
|
|
Func<uint> value,
|
|
Func<uint> comparator)
|
|
{
|
|
if (op is SpirvOp.AtomicIIncrement or SpirvOp.AtomicIDecrement)
|
|
{
|
|
return _module.AddInstruction(
|
|
op,
|
|
type,
|
|
pointer,
|
|
UInt(scope),
|
|
UInt(semantics));
|
|
}
|
|
|
|
if (op == SpirvOp.AtomicCompareExchange)
|
|
{
|
|
// The unequal semantics must not contain Release; downgrade it to Acquire.
|
|
return _module.AddInstruction(
|
|
op,
|
|
type,
|
|
pointer,
|
|
UInt(scope),
|
|
UInt(semantics),
|
|
UInt((semantics & ~0x8u) | 0x2u),
|
|
value(),
|
|
comparator());
|
|
}
|
|
|
|
return _module.AddInstruction(
|
|
op,
|
|
type,
|
|
pointer,
|
|
UInt(scope),
|
|
UInt(semantics),
|
|
value());
|
|
}
|
|
|
|
private bool TryEmitInterpolation(
|
|
Gen5ShaderInstruction instruction,
|
|
Gen5InterpolationControl interpolation,
|
|
out string error)
|
|
{
|
|
error = string.Empty;
|
|
if (_stage != Gen5SpirvStage.Pixel ||
|
|
!_pixelInputs.TryGetValue(interpolation.Attribute, out var input) ||
|
|
!TryGetVectorDestination(instruction, out var destination))
|
|
{
|
|
error = "invalid interpolated attribute";
|
|
return false;
|
|
}
|
|
|
|
if (_perVertexAttributes.Contains(interpolation.Attribute))
|
|
{
|
|
return TryEmitInterpolationParameter(instruction, interpolation, input, destination, out error);
|
|
}
|
|
|
|
if (_perVertexSourcedAttributes.TryGetValue(interpolation.Attribute, out var flatSource))
|
|
{
|
|
EmitPerVertexSourcedInterpolation(interpolation, input, destination, flatSource);
|
|
return true;
|
|
}
|
|
|
|
var vector = Load(_vec4Type, input);
|
|
var component = _module.AddInstruction(
|
|
SpirvOp.CompositeExtract,
|
|
_floatType,
|
|
vector,
|
|
interpolation.Channel);
|
|
StoreV(destination, Bitcast(_uintType, component));
|
|
return true;
|
|
}
|
|
|
|
private bool TryEmitScalarMemory(
|
|
Gen5ShaderInstruction instruction,
|
|
Gen5ScalarMemoryControl control,
|
|
out string error)
|
|
{
|
|
error = string.Empty;
|
|
{
|
|
return TryEmitLayoutScalarMemory(instruction, control, out error);
|
|
}
|
|
}
|
|
|
|
private bool TryEmitGlobalMemory(
|
|
Gen5ShaderInstruction instruction,
|
|
Gen5GlobalMemoryControl control,
|
|
out string error)
|
|
{
|
|
error = string.Empty;
|
|
{
|
|
return TryEmitLayoutGlobalMemory(instruction, control, out error);
|
|
}
|
|
}
|
|
|
|
private bool TryEmitBufferMemory(
|
|
Gen5ShaderInstruction instruction,
|
|
Gen5BufferMemoryControl control,
|
|
out string error)
|
|
{
|
|
error = string.Empty;
|
|
if (instruction.Opcode is "BufferWbinvl1" or "BufferWbinvl1Vol")
|
|
{
|
|
// The host storage path is already coherent; the guest instruction
|
|
// invalidates a cache level and does not access the buffer itself.
|
|
return true;
|
|
}
|
|
|
|
if (control.Typed && instruction.Opcode.Contains("D16", StringComparison.Ordinal))
|
|
{
|
|
error = $"unsupported buffer opcode {instruction.Opcode}";
|
|
return false;
|
|
}
|
|
|
|
if (_stage == Gen5SpirvStage.Vertex &&
|
|
_vertexInputsByPc.TryGetValue(instruction.Pc, out var vertexInput))
|
|
{
|
|
return TryEmitVertexInputFetch(control, vertexInput, out error);
|
|
}
|
|
|
|
if (_request.Memory.Find(instruction.Pc) is { DeviceDescriptor: true })
|
|
{
|
|
return TryEmitDeviceDescriptorBufferMemory(instruction, control, out error);
|
|
}
|
|
|
|
if (_request.Memory.TryGetIndex(instruction.Pc, 0, out var accessMemoryIndex))
|
|
{
|
|
var accessMemory = _request.Memory[accessMemoryIndex];
|
|
if (accessMemory.PlanningOnly)
|
|
{
|
|
// The access belongs to the linearized tail of a fused
|
|
// shader. Its descriptor is supplied by the continuation
|
|
// object and is intentionally absent from this plan; keep
|
|
// the dead linear instruction side-effect free.
|
|
return true;
|
|
}
|
|
|
|
var strategy = accessMemory.BufferDescriptor?.ChooseStrategy(
|
|
control.Typed,
|
|
accessMemory.Formatted,
|
|
accessMemory.Access == MemoryAccess.Atomic)
|
|
?? BufferLoweringStrategy.NativeBinding;
|
|
if (strategy == BufferLoweringStrategy.PhysicalStorageBuffer)
|
|
{
|
|
return TryEmitPhysicalStorageBufferMemory(instruction, control, out error);
|
|
}
|
|
|
|
if (strategy == BufferLoweringStrategy.BoundedCandidateTable)
|
|
{
|
|
return TryEmitBoundedCandidateTableMemory(instruction, control, accessMemoryIndex, out error);
|
|
}
|
|
}
|
|
|
|
if (!TryResolveLayoutBuffer(instruction.Pc, out var bindingIndex, out _))
|
|
{
|
|
error = "missing buffer-memory binding";
|
|
return false;
|
|
}
|
|
|
|
return EmitResolvedBufferMemory(instruction, control, bindingIndex, out error);
|
|
}
|
|
|
|
// One buffer operation against a resolved candidate binding: the dense binding path
|
|
// and every arm of a bounded candidate table share it, so the operation itself
|
|
// stays independent of how its descriptor was selected.
|
|
private bool EmitResolvedBufferMemory(
|
|
Gen5ShaderInstruction instruction,
|
|
Gen5BufferMemoryControl control,
|
|
int bindingIndex,
|
|
out string error)
|
|
{
|
|
error = string.Empty;
|
|
var info = _request.Resources.Info;
|
|
if (bindingIndex < 0 || bindingIndex >= info.Buffers.Count)
|
|
{
|
|
error = $"buffer binding {bindingIndex} is out of range";
|
|
return false;
|
|
}
|
|
|
|
var specialized = info.Buffers[bindingIndex];
|
|
var stride = UInt(specialized.PackedStride & 0x3FFF);
|
|
var descriptorFormat = specialized.DescriptorFormat;
|
|
var descriptorWord3 = UInt((specialized.DescriptorFormat << 12) | (specialized.DescriptorSwizzle & 0xFFF));
|
|
|
|
var scalarOffset = instruction.Sources.Count > 2
|
|
? GetRawSource(instruction, 2)
|
|
: UInt(0);
|
|
var vectorIndex = control.IndexEnabled
|
|
? LoadV(control.VectorAddress)
|
|
: UInt(0);
|
|
var vectorOffset = control.OffsetEnabled
|
|
? LoadV(control.VectorAddress + (control.IndexEnabled ? 1u : 0u))
|
|
: UInt(0);
|
|
var byteAddress = IAdd(
|
|
UInt(unchecked((uint)control.OffsetBytes)),
|
|
scalarOffset);
|
|
byteAddress = IAdd(byteAddress, vectorOffset);
|
|
byteAddress = IAdd(
|
|
byteAddress,
|
|
_module.AddInstruction(SpirvOp.IMul, _uintType, vectorIndex, stride));
|
|
byteAddress = ApplyGuestBufferByteBias(bindingIndex, byteAddress);
|
|
var dwordAddress = ShiftRightLogical(byteAddress, UInt(2));
|
|
|
|
if (instruction.Opcode is "BufferAtomicSwapX2" or "BufferAtomicOrX2")
|
|
{
|
|
EmitExecConditional(() =>
|
|
{
|
|
var firstInRange = IsBufferWordInRange(bindingIndex, dwordAddress);
|
|
var secondAddress = IAdd(dwordAddress, UInt(1));
|
|
var secondInRange = IsBufferWordInRange(bindingIndex, secondAddress);
|
|
EmitConditional(LogicalAnd(firstInRange, secondInRange), () =>
|
|
{
|
|
var atomicOp = instruction.Opcode == "BufferAtomicSwapX2"
|
|
? SpirvOp.AtomicExchange
|
|
: SpirvOp.AtomicOr;
|
|
var first = EmitAtomic(
|
|
atomicOp,
|
|
_uintType,
|
|
BufferWordPointer(bindingIndex, dwordAddress),
|
|
scope: 1,
|
|
semantics: 0x48,
|
|
value: () => LoadV(control.VectorData),
|
|
comparator: () => UInt(0));
|
|
var second = EmitAtomic(
|
|
atomicOp,
|
|
_uintType,
|
|
BufferWordPointer(bindingIndex, secondAddress),
|
|
scope: 1,
|
|
semantics: 0x48,
|
|
value: () => LoadV(control.VectorData + 1),
|
|
comparator: () => UInt(0));
|
|
if (control.Glc)
|
|
{
|
|
StoreV(control.VectorData, first);
|
|
StoreV(control.VectorData + 1, second);
|
|
}
|
|
});
|
|
});
|
|
return true;
|
|
}
|
|
|
|
if (instruction.Opcode.StartsWith("BufferAtomic", StringComparison.Ordinal))
|
|
{
|
|
var atomicSuffix = instruction.Opcode["BufferAtomic".Length..];
|
|
if (atomicSuffix is "SwapX2" or "OrX2")
|
|
{
|
|
var wideAtomicOp = atomicSuffix == "SwapX2"
|
|
? SpirvOp.AtomicExchange
|
|
: SpirvOp.AtomicOr;
|
|
EmitExecConditional(() =>
|
|
{
|
|
var secondAddress = IAdd(dwordAddress, UInt(1));
|
|
var inRange = _module.AddInstruction(
|
|
SpirvOp.LogicalAnd,
|
|
_boolType,
|
|
IsBufferWordInRange(bindingIndex, dwordAddress),
|
|
IsBufferWordInRange(bindingIndex, secondAddress));
|
|
EmitConditional(inRange, () =>
|
|
{
|
|
var originalLow = EmitAtomic(
|
|
wideAtomicOp,
|
|
_uintType,
|
|
BufferWordPointer(bindingIndex, dwordAddress),
|
|
scope: 1,
|
|
semantics: 0x48,
|
|
value: () => LoadV(control.VectorData),
|
|
comparator: () => UInt(0));
|
|
var originalHigh = EmitAtomic(
|
|
wideAtomicOp,
|
|
_uintType,
|
|
BufferWordPointer(bindingIndex, secondAddress),
|
|
scope: 1,
|
|
semantics: 0x48,
|
|
value: () => LoadV(control.VectorData + 1),
|
|
comparator: () => UInt(0));
|
|
if (control.Glc)
|
|
{
|
|
StoreV(control.VectorData, originalLow);
|
|
StoreV(control.VectorData + 1, originalHigh);
|
|
}
|
|
});
|
|
});
|
|
|
|
return true;
|
|
}
|
|
|
|
if (atomicSuffix is "Fmin" or "Fmax")
|
|
{
|
|
EmitExecConditional(() =>
|
|
{
|
|
var inRange = IsBufferWordInRange(bindingIndex, dwordAddress);
|
|
EmitConditional(inRange, () =>
|
|
{
|
|
var original = EmitBufferFloatAtomic(
|
|
BufferWordPointer(bindingIndex, dwordAddress),
|
|
LoadV(control.VectorData),
|
|
maxValue: atomicSuffix == "Fmax",
|
|
scope: 1,
|
|
semantics: 0x48);
|
|
if (control.Glc)
|
|
{
|
|
StoreV(control.VectorData, original);
|
|
}
|
|
});
|
|
});
|
|
|
|
return true;
|
|
}
|
|
|
|
if (!TryGetAtomicOp(atomicSuffix, out var atomicOp))
|
|
{
|
|
error = $"unsupported buffer opcode {instruction.Opcode}";
|
|
return false;
|
|
}
|
|
|
|
EmitExecConditional(() =>
|
|
{
|
|
var inRange = IsBufferWordInRange(bindingIndex, dwordAddress);
|
|
EmitConditional(inRange, () =>
|
|
{
|
|
var original = EmitAtomic(
|
|
atomicOp,
|
|
_uintType,
|
|
BufferWordPointer(bindingIndex, dwordAddress),
|
|
scope: 1,
|
|
semantics: 0x48,
|
|
value: () => LoadV(control.VectorData),
|
|
comparator: () => LoadV(control.VectorData + 1));
|
|
if (control.Glc)
|
|
{
|
|
StoreV(control.VectorData, original);
|
|
}
|
|
});
|
|
});
|
|
|
|
return true;
|
|
}
|
|
|
|
if (instruction.Opcode is "BufferStoreFormatX" or "BufferStoreFormatXy" or
|
|
"BufferStoreFormatXyz" or "BufferStoreFormatXyzw")
|
|
{
|
|
if (descriptorFormat == 0)
|
|
{
|
|
return true;
|
|
}
|
|
|
|
if (!Gfx10UnifiedFormat.TryDecode(descriptorFormat, out _, out _))
|
|
{
|
|
error = $"unsupported buffer store format {descriptorFormat}";
|
|
return false;
|
|
}
|
|
|
|
EmitExecConditional(() => TryEmitBufferFormatStore(
|
|
bindingIndex, byteAddress, control, descriptorWord3, descriptorFormat));
|
|
return true;
|
|
}
|
|
|
|
if (instruction.Opcode.StartsWith("BufferStoreDword", StringComparison.Ordinal) ||
|
|
instruction.Opcode.StartsWith("BufferStoreFormat", StringComparison.Ordinal) ||
|
|
instruction.Opcode.StartsWith("TBufferStoreFormat", StringComparison.Ordinal) ||
|
|
instruction.Opcode.StartsWith("BufferStoreByte", StringComparison.Ordinal) ||
|
|
instruction.Opcode.StartsWith("BufferStoreShort", StringComparison.Ordinal))
|
|
{
|
|
EmitExecConditional(() =>
|
|
{
|
|
if (control.Typed && TryEmitBufferFormatStore(bindingIndex, byteAddress, control, descriptorWord3, control.TypedFormat))
|
|
{
|
|
return;
|
|
}
|
|
|
|
if (TryGetSubdwordStoreInfo(
|
|
instruction.Opcode,
|
|
out var byteCount,
|
|
out var sourceShift))
|
|
{
|
|
StoreBufferBytes(
|
|
bindingIndex,
|
|
byteAddress,
|
|
LoadV(control.VectorData),
|
|
byteCount,
|
|
sourceShift);
|
|
return;
|
|
}
|
|
|
|
// BUFFER_STORE/LOAD_DWORD(x2/x3/x4) are dword-aligned by the GCN ISA, same as the GLOBAL case above — no per-byte reassembly needed.
|
|
for (uint index = 0; index < control.DwordCount; index++)
|
|
{
|
|
var indexedDwordAddress = index == 0
|
|
? dwordAddress
|
|
: IAdd(dwordAddress, UInt(index));
|
|
StoreBufferWord(
|
|
bindingIndex,
|
|
indexedDwordAddress,
|
|
LoadV(control.VectorData + index));
|
|
}
|
|
});
|
|
|
|
return true;
|
|
}
|
|
|
|
if (TryGetSubdwordLoadInfo(
|
|
instruction.Opcode,
|
|
out var loadByteCount,
|
|
out var signExtend,
|
|
out var d16,
|
|
out var d16High))
|
|
{
|
|
StoreV(
|
|
control.VectorData,
|
|
LoadSubdwordBufferValue(
|
|
bindingIndex,
|
|
byteAddress,
|
|
LoadV(control.VectorData),
|
|
loadByteCount,
|
|
signExtend,
|
|
d16,
|
|
d16High));
|
|
return true;
|
|
}
|
|
|
|
if (!instruction.Opcode.StartsWith("BufferLoad", StringComparison.Ordinal) &&
|
|
!instruction.Opcode.StartsWith("TBufferLoad", StringComparison.Ordinal))
|
|
{
|
|
error = $"unsupported buffer opcode {instruction.Opcode}";
|
|
return false;
|
|
}
|
|
|
|
// A typed load converts with the instruction format, a formatted untyped load
|
|
// with the descriptor format and swizzle; raw dword loads take the path below.
|
|
if (IsFormatBufferLoad(instruction.Opcode))
|
|
{
|
|
if (!control.Typed)
|
|
{
|
|
EmitBufferFormatLoad(
|
|
bindingIndex,
|
|
byteAddress,
|
|
descriptorWord3,
|
|
control.VectorData,
|
|
control.DwordCount);
|
|
return true;
|
|
}
|
|
|
|
if (TryEmitTypedBufferFormatLoad(bindingIndex, byteAddress, control, descriptorWord3))
|
|
{
|
|
return true;
|
|
}
|
|
}
|
|
|
|
for (uint index = 0; index < control.DwordCount; index++)
|
|
{
|
|
var indexedDwordAddress = index == 0
|
|
? dwordAddress
|
|
: IAdd(dwordAddress, UInt(index));
|
|
StoreV(
|
|
control.VectorData + index,
|
|
LoadBufferWord(bindingIndex, indexedDwordAddress));
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
// BufferLoweringStrategy.BoundedCandidateTable: a runtime V# whose descriptors cannot
|
|
// be reconstructed from its raw words. The runtime V# sits in the scalar resource
|
|
// registers; its base-address dword is the probe key into the flattened candidate
|
|
// mapping, and each arm binds a native candidate and runs the ordinary buffer op.
|
|
private bool TryEmitBoundedCandidateTableMemory(
|
|
Gen5ShaderInstruction instruction,
|
|
Gen5BufferMemoryControl control,
|
|
int memoryIndex,
|
|
out string error)
|
|
{
|
|
error = string.Empty;
|
|
if (!_request.BufferCandidateTableByMemoryIndex.TryGetValue(memoryIndex, out var table))
|
|
{
|
|
// A formatted load needs a statically known Vulkan view format. Most
|
|
// runtime V#s prove a bounded SRT candidate set above, but a few Yotei
|
|
// material-lookup paths merge descriptors through control flow and have
|
|
// no finite, safe candidate table. Do not reinterpret an arbitrary
|
|
// device address with a guessed format: a null read is the defined
|
|
// fallback, matching the bindless-image fallback in ResourceTracker.
|
|
if (instruction.Opcode.StartsWith("BufferLoadFormat", StringComparison.Ordinal))
|
|
{
|
|
for (uint index = 0; index < control.DwordCount; index++)
|
|
{
|
|
StoreV(control.VectorData + index, UInt(0));
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
error = $"runtime buffer descriptor has no candidate table for {instruction.Opcode}";
|
|
return false;
|
|
}
|
|
|
|
if (table.CandidateCount == 0)
|
|
{
|
|
error = "runtime buffer descriptor candidate table is empty";
|
|
return false;
|
|
}
|
|
|
|
if (table.CandidateCount == 1)
|
|
{
|
|
return EmitResolvedBufferMemory(instruction, control, (int)table.FirstCandidate, out error);
|
|
}
|
|
|
|
if (!HasFlattenedTable)
|
|
{
|
|
error = "runtime buffer descriptor candidate table without a flattened table binding";
|
|
return false;
|
|
}
|
|
|
|
if (instruction.Sources.Count < 2 || instruction.Sources[1].Kind != Gen5OperandKind.ScalarRegister)
|
|
{
|
|
error = "runtime buffer descriptor has no scalar resource base";
|
|
return false;
|
|
}
|
|
|
|
var probeKey = LoadS(instruction.Sources[1].Value);
|
|
var selector = SelectBufferCandidate(table, probeKey);
|
|
var emitted = true;
|
|
var caseError = string.Empty;
|
|
for (uint index = 0; index < table.CandidateCount && emitted; index++)
|
|
{
|
|
var candidate = table.FirstCandidate + index;
|
|
EmitConditional(_module.AddInstruction(SpirvOp.IEqual, _boolType, selector, UInt(index)), () =>
|
|
{
|
|
if (!EmitResolvedBufferMemory(instruction, control, (int)candidate, out caseError))
|
|
{
|
|
emitted = false;
|
|
}
|
|
});
|
|
}
|
|
|
|
error = caseError;
|
|
return emitted;
|
|
}
|
|
|
|
// Searches the sorted base-address mapping of a candidate table for the runtime V#'s
|
|
// probe key; the result is the candidate-local index, candidate 0 when absent.
|
|
private uint SelectBufferCandidate(BufferCandidateTableUse table, uint key)
|
|
{
|
|
var mapping = UInt(table.MappingOffset);
|
|
var count = LoadFlattenedWord(mapping);
|
|
var low = UInt(0);
|
|
var high = count;
|
|
for (uint iteration = 0; iteration < table.SearchIterations; iteration++)
|
|
{
|
|
var span = _module.AddInstruction(SpirvOp.ISub, _uintType, high, low);
|
|
var middle = IAdd(low, ShiftRightLogical(span, UInt(1)));
|
|
var probeSlot = IAdd(IAdd(mapping, UInt(1)), ShiftLeftLogical(middle, UInt(1)));
|
|
var probeKey = LoadFlattenedWord(probeSlot);
|
|
var moveUp = LogicalAnd(
|
|
_module.AddInstruction(SpirvOp.ULessThan, _boolType, probeKey, key),
|
|
_module.AddInstruction(SpirvOp.ULessThan, _boolType, low, high));
|
|
low = _module.AddInstruction(SpirvOp.Select, _uintType, moveUp, IAdd(middle, UInt(1)), low);
|
|
high = _module.AddInstruction(SpirvOp.Select, _uintType, moveUp, high, middle);
|
|
}
|
|
|
|
var foundSlot = IAdd(IAdd(mapping, UInt(1)), ShiftLeftLogical(low, UInt(1)));
|
|
var found = LogicalAnd(
|
|
_module.AddInstruction(SpirvOp.ULessThan, _boolType, low, count),
|
|
_module.AddInstruction(SpirvOp.IEqual, _boolType, LoadFlattenedWord(foundSlot), key));
|
|
var mapped = LoadFlattenedWord(IAdd(foundSlot, UInt(1)));
|
|
var inRange = LogicalAnd(
|
|
found,
|
|
_module.AddInstruction(SpirvOp.ULessThan, _boolType, mapped, UInt(table.CandidateCount)));
|
|
return _module.AddInstruction(SpirvOp.Select, _uintType, inRange, mapped, UInt(0));
|
|
}
|
|
|
|
private bool TryEmitDeviceDescriptorBufferMemory(
|
|
Gen5ShaderInstruction instruction,
|
|
Gen5BufferMemoryControl control,
|
|
out string error)
|
|
{
|
|
error = string.Empty;
|
|
if (control.Typed || instruction.Opcode.StartsWith("TBuffer", StringComparison.Ordinal))
|
|
{
|
|
error = $"device buffer descriptor operation {instruction.Opcode} is not supported";
|
|
return false;
|
|
}
|
|
|
|
_deviceAddressInstructionPc = instruction.Pc;
|
|
var (baseAddress, size, stride, descriptorWord3) = LoadDeviceBufferDescriptor(control.ScalarResource);
|
|
var scalarOffset = instruction.Sources.Count > 2
|
|
? GetRawSource(instruction, 2)
|
|
: UInt(0);
|
|
var vectorIndex = control.IndexEnabled
|
|
? LoadV(control.VectorAddress)
|
|
: UInt(0);
|
|
var vectorOffset = control.OffsetEnabled
|
|
? LoadV(control.VectorAddress + (control.IndexEnabled ? 1u : 0u))
|
|
: UInt(0);
|
|
var byteAddress = IAdd(UInt(unchecked((uint)control.OffsetBytes)), scalarOffset);
|
|
byteAddress = IAdd(byteAddress, vectorOffset);
|
|
byteAddress = IAdd(byteAddress, _module.AddInstruction(SpirvOp.IMul, _uintType, vectorIndex, stride));
|
|
|
|
if (IsFormatBufferLoad(instruction.Opcode))
|
|
{
|
|
EmitDeviceBufferFormatLoad(
|
|
baseAddress,
|
|
size,
|
|
byteAddress,
|
|
descriptorWord3,
|
|
control.VectorData,
|
|
control.DwordCount);
|
|
return true;
|
|
}
|
|
|
|
if (instruction.Opcode.Contains("Format", StringComparison.Ordinal))
|
|
{
|
|
error = $"device buffer descriptor operation {instruction.Opcode} is not supported";
|
|
return false;
|
|
}
|
|
|
|
if (instruction.Opcode.StartsWith("BufferAtomic", StringComparison.Ordinal))
|
|
{
|
|
return TryEmitDeviceDescriptorBufferAtomic(instruction, control, baseAddress, size, byteAddress, out error);
|
|
}
|
|
|
|
if (TryGetSubdwordStoreInfo(instruction.Opcode, out var storeByteCount, out var sourceShift))
|
|
{
|
|
EmitExecConditional(() =>
|
|
{
|
|
var inRange = IsDeviceBufferByteRangeInRange(size, byteAddress, storeByteCount);
|
|
var address = And64(IAdd64(baseAddress, Widen(byteAddress)), ULong(DeviceAddressMask));
|
|
EmitConditional(inRange, () =>
|
|
StoreDeviceBytes(address, LoadV(control.VectorData), storeByteCount, sourceShift, _module.ConstantBool(true)));
|
|
});
|
|
return true;
|
|
}
|
|
|
|
if (instruction.Opcode.StartsWith("BufferStoreDword", StringComparison.Ordinal))
|
|
{
|
|
EmitExecConditional(() =>
|
|
{
|
|
for (uint index = 0; index < control.DwordCount; index++)
|
|
{
|
|
var componentAddress = index == 0
|
|
? byteAddress
|
|
: IAdd(byteAddress, UInt(index * sizeof(uint)));
|
|
StoreDeviceBufferWord(
|
|
baseAddress,
|
|
size,
|
|
componentAddress,
|
|
LoadV(control.VectorData + index));
|
|
}
|
|
});
|
|
return true;
|
|
}
|
|
|
|
if (TryGetSubdwordLoadInfo(
|
|
instruction.Opcode,
|
|
out var loadByteCount,
|
|
out var signExtend,
|
|
out var d16,
|
|
out var d16High))
|
|
{
|
|
var inRange = IsDeviceBufferByteRangeInRange(size, byteAddress, loadByteCount);
|
|
var address = And64(IAdd64(baseAddress, Widen(byteAddress)), ULong(DeviceAddressMask));
|
|
Store(_deviceBufferWordScratch, UInt(0));
|
|
EmitConditional(inRange, () => Store(
|
|
_deviceBufferWordScratch,
|
|
LoadSubdwordDeviceValue(
|
|
address,
|
|
LoadV(control.VectorData),
|
|
loadByteCount,
|
|
signExtend,
|
|
d16,
|
|
d16High)));
|
|
StoreV(control.VectorData, Load(_uintType, _deviceBufferWordScratch));
|
|
return true;
|
|
}
|
|
|
|
if (instruction.Opcode.StartsWith("BufferLoadDword", StringComparison.Ordinal))
|
|
{
|
|
for (uint index = 0; index < control.DwordCount; index++)
|
|
{
|
|
var componentAddress = index == 0
|
|
? byteAddress
|
|
: IAdd(byteAddress, UInt(index * sizeof(uint)));
|
|
StoreV(
|
|
control.VectorData + index,
|
|
LoadDeviceBufferWord(baseAddress, size, componentAddress));
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
error = $"unsupported device buffer descriptor opcode {instruction.Opcode}";
|
|
return false;
|
|
}
|
|
|
|
private bool TryEmitDeviceDescriptorBufferAtomic(
|
|
Gen5ShaderInstruction instruction,
|
|
Gen5BufferMemoryControl control,
|
|
uint baseAddress,
|
|
uint size,
|
|
uint byteAddress,
|
|
out string error)
|
|
{
|
|
error = string.Empty;
|
|
var atomicSuffix = instruction.Opcode["BufferAtomic".Length..];
|
|
if (atomicSuffix is "SwapX2" or "OrX2")
|
|
{
|
|
var atomicOp = atomicSuffix == "SwapX2" ? SpirvOp.AtomicExchange : SpirvOp.AtomicOr;
|
|
EmitExecConditional(() =>
|
|
{
|
|
var valid = IsDeviceBufferByteRangeInRange(size, byteAddress, 2 * sizeof(uint));
|
|
var firstAddress = And64(IAdd64(baseAddress, Widen(byteAddress)), ULong(DeviceAddressMask & ~3ul));
|
|
var secondAddress = And64(IAdd64(baseAddress, Widen(IAdd(byteAddress, UInt(sizeof(uint))))), ULong(DeviceAddressMask & ~3ul));
|
|
EmitConditional(valid, () =>
|
|
{
|
|
var (firstPointer, firstMapped) = ResolveDeviceAddress(firstAddress);
|
|
var (secondPointer, secondMapped) = ResolveDeviceAddress(secondAddress);
|
|
var bothMapped = LogicalAnd(firstMapped, secondMapped);
|
|
EmitConditional(bothMapped, () =>
|
|
{
|
|
var originalLow = EmitAtomic(
|
|
atomicOp, _uintType, DeviceWordPointer(firstPointer), 1, 0x48,
|
|
() => LoadV(control.VectorData), () => UInt(0));
|
|
var originalHigh = EmitAtomic(
|
|
atomicOp, _uintType, DeviceWordPointer(secondPointer), 1, 0x48,
|
|
() => LoadV(control.VectorData + 1), () => UInt(0));
|
|
if (control.Glc)
|
|
{
|
|
StoreV(control.VectorData, originalLow);
|
|
StoreV(control.VectorData + 1, originalHigh);
|
|
}
|
|
});
|
|
});
|
|
});
|
|
return true;
|
|
}
|
|
|
|
if (atomicSuffix is "Fmin" or "Fmax")
|
|
{
|
|
EmitExecConditional(() =>
|
|
{
|
|
var valid = IsDeviceBufferByteRangeInRange(size, byteAddress, sizeof(uint));
|
|
var address = And64(IAdd64(baseAddress, Widen(byteAddress)), ULong(DeviceAddressMask & ~3ul));
|
|
EmitConditional(valid, () =>
|
|
{
|
|
var (pointer, mapped) = ResolveDeviceAddress(address);
|
|
EmitConditional(mapped, () =>
|
|
{
|
|
var original = EmitBufferFloatAtomic(
|
|
DeviceWordPointer(pointer),
|
|
LoadV(control.VectorData),
|
|
maxValue: atomicSuffix == "Fmax",
|
|
scope: 1,
|
|
semantics: 0x48);
|
|
if (control.Glc)
|
|
StoreV(control.VectorData, original);
|
|
});
|
|
});
|
|
});
|
|
return true;
|
|
}
|
|
|
|
if (!TryGetAtomicOp(atomicSuffix, out var atomicOperation))
|
|
{
|
|
error = $"unsupported buffer atomic opcode {instruction.Opcode}";
|
|
return false;
|
|
}
|
|
|
|
EmitExecConditional(() =>
|
|
{
|
|
var valid = IsDeviceBufferByteRangeInRange(size, byteAddress, sizeof(uint));
|
|
var address = And64(IAdd64(baseAddress, Widen(byteAddress)), ULong(DeviceAddressMask & ~3ul));
|
|
EmitConditional(valid, () =>
|
|
{
|
|
var (pointer, mapped) = ResolveDeviceAddress(address);
|
|
EmitConditional(mapped, () =>
|
|
{
|
|
var original = EmitAtomic(
|
|
atomicOperation,
|
|
_uintType,
|
|
DeviceWordPointer(pointer),
|
|
1,
|
|
0x48,
|
|
() => LoadV(control.VectorData),
|
|
() => LoadV(control.VectorData + 1));
|
|
if (control.Glc)
|
|
StoreV(control.VectorData, original);
|
|
});
|
|
});
|
|
});
|
|
return true;
|
|
}
|
|
|
|
private void EmitDeviceBufferFormatLoad(
|
|
uint baseAddress,
|
|
uint size,
|
|
uint byteAddress,
|
|
uint descriptorWord3,
|
|
uint vectorData,
|
|
uint componentCount)
|
|
{
|
|
var unifiedFormat = BitwiseAnd(
|
|
ShiftRightLogical(descriptorWord3, UInt(12)),
|
|
UInt(0x7F));
|
|
var (dataFormat, numberFormat) = DecodeGfx10BufferFormat(unifiedFormat);
|
|
|
|
var canonical = new uint[4];
|
|
var componentBounds = new uint[4];
|
|
for (var component = 0; component < canonical.Length; component++)
|
|
{
|
|
canonical[component] = LoadGfx10DeviceBufferFormatComponent(
|
|
baseAddress,
|
|
size,
|
|
byteAddress,
|
|
dataFormat,
|
|
numberFormat,
|
|
component,
|
|
out componentBounds[component]);
|
|
}
|
|
|
|
var selectors = new uint[componentCount];
|
|
var inBounds = _module.ConstantBool(true);
|
|
for (uint destination = 0; destination < componentCount; destination++)
|
|
{
|
|
var selector = BitwiseAnd(
|
|
ShiftRightLogical(descriptorWord3, UInt(destination * 3)),
|
|
UInt(7));
|
|
selectors[destination] = selector;
|
|
var selectedInBounds = _module.ConstantBool(true);
|
|
for (uint component = 0; component < 4; component++)
|
|
{
|
|
selectedInBounds = _module.AddInstruction(
|
|
SpirvOp.Select,
|
|
_boolType,
|
|
_module.AddInstruction(SpirvOp.IEqual, _boolType, selector, UInt(component + 4)),
|
|
componentBounds[component],
|
|
selectedInBounds);
|
|
}
|
|
|
|
inBounds = _module.AddInstruction(SpirvOp.LogicalAnd, _boolType, inBounds, selectedInBounds);
|
|
}
|
|
|
|
var one = Gfx10FormatOne(numberFormat);
|
|
for (uint destination = 0; destination < componentCount; destination++)
|
|
{
|
|
var selector = selectors[destination];
|
|
var constant = SelectUInt(selector, 1, one, UInt(0));
|
|
var value = constant;
|
|
value = SelectUInt(selector, 4, canonical[0], value);
|
|
value = SelectUInt(selector, 5, canonical[1], value);
|
|
value = SelectUInt(selector, 6, canonical[2], value);
|
|
value = SelectUInt(selector, 7, canonical[3], value);
|
|
StoreV(
|
|
vectorData + destination,
|
|
_module.AddInstruction(SpirvOp.Select, _uintType, inBounds, value, constant));
|
|
}
|
|
}
|
|
|
|
private uint LoadGfx10DeviceBufferFormatComponent(
|
|
uint baseAddress,
|
|
uint size,
|
|
uint elementAddress,
|
|
uint dataFormat,
|
|
uint numberFormat,
|
|
int component,
|
|
out uint componentInBounds)
|
|
{
|
|
var byteOffset = UInt(0);
|
|
var bitOffset = UInt(0);
|
|
var bitCount = UInt(0);
|
|
|
|
void SetLayout(uint format, uint bytes, uint bits, uint count)
|
|
{
|
|
var matches = _module.AddInstruction(SpirvOp.IEqual, _boolType, dataFormat, UInt(format));
|
|
byteOffset = _module.AddInstruction(SpirvOp.Select, _uintType, matches, UInt(bytes), byteOffset);
|
|
bitOffset = _module.AddInstruction(SpirvOp.Select, _uintType, matches, UInt(bits), bitOffset);
|
|
bitCount = _module.AddInstruction(SpirvOp.Select, _uintType, matches, UInt(count), bitCount);
|
|
}
|
|
|
|
foreach (var layout in Gfx10UnifiedFormat.ComponentLayouts)
|
|
{
|
|
if (layout.Component == (uint)component)
|
|
SetLayout(layout.DataFormat, layout.ByteOffset, layout.BitOffset, layout.BitCount);
|
|
}
|
|
|
|
var componentAddress = IAdd(elementAddress, byteOffset);
|
|
var componentBytes = ShiftRightLogical(IAdd(IAdd(bitOffset, bitCount), UInt(7)), UInt(3));
|
|
var lastByteOffset = _module.AddInstruction(SpirvOp.ISub, _uintType, componentBytes, UInt(1));
|
|
var hasComponent = _module.AddInstruction(SpirvOp.INotEqual, _boolType, bitCount, UInt(0));
|
|
var inRange = IsDeviceBufferElementInRange(size, componentAddress, lastByteOffset);
|
|
var accessAllowed = LogicalAnd(hasComponent, inRange);
|
|
componentInBounds = _module.AddInstruction(
|
|
SpirvOp.LogicalOr,
|
|
_boolType,
|
|
LogicalNot(hasComponent),
|
|
inRange);
|
|
var packed = UInt(0);
|
|
for (uint index = 0; index < sizeof(uint); index++)
|
|
{
|
|
var address = index == 0 ? componentAddress : IAdd(componentAddress, UInt(index));
|
|
var word = LoadDeviceBufferWord(baseAddress, size, address, accessAllowed);
|
|
var shift = ShiftLeftLogical(BitwiseAnd(address, UInt(3)), UInt(3));
|
|
var value = BitwiseAnd(ShiftRightLogical(word, shift), UInt(0xFF));
|
|
packed = BitwiseOr(packed, ShiftLeftLogical(value, UInt(index * 8)));
|
|
}
|
|
|
|
var raw = _module.AddInstruction(SpirvOp.BitFieldUExtract, _uintType, packed, bitOffset, bitCount);
|
|
var converted = ConvertGfx10BufferComponent(raw, bitCount, numberFormat, dataFormat);
|
|
var valid = _module.AddInstruction(SpirvOp.INotEqual, _boolType, bitCount, UInt(0));
|
|
return _module.AddInstruction(
|
|
SpirvOp.Select,
|
|
_uintType,
|
|
valid,
|
|
converted,
|
|
component == 3 ? Gfx10FormatOne(numberFormat) : UInt(0));
|
|
}
|
|
|
|
private void EmitBufferFormatLoad(
|
|
int bindingIndex,
|
|
uint byteAddress,
|
|
uint descriptorWord3,
|
|
uint vectorData,
|
|
uint componentCount)
|
|
{
|
|
var unifiedFormat = BitwiseAnd(
|
|
ShiftRightLogical(descriptorWord3, UInt(12)),
|
|
UInt(0x7F));
|
|
var (dataFormat, numberFormat) = DecodeGfx10BufferFormat(unifiedFormat);
|
|
|
|
var canonical = new uint[4];
|
|
var componentBounds = new uint[4];
|
|
for (var component = 0; component < canonical.Length; component++)
|
|
{
|
|
canonical[component] = LoadGfx10BufferFormatComponent(
|
|
bindingIndex,
|
|
byteAddress,
|
|
dataFormat,
|
|
numberFormat,
|
|
component,
|
|
out componentBounds[component]);
|
|
}
|
|
|
|
// Only selected memory components contribute to the shared bounds check.
|
|
var selectors = new uint[componentCount];
|
|
var inBounds = _module.ConstantBool(true);
|
|
for (uint destination = 0; destination < componentCount; destination++)
|
|
{
|
|
var selector = BitwiseAnd(
|
|
ShiftRightLogical(descriptorWord3, UInt(destination * 3)), UInt(7));
|
|
selectors[destination] = selector;
|
|
var selectedInBounds = _module.ConstantBool(true);
|
|
for (uint component = 0; component < 4; component++)
|
|
{
|
|
selectedInBounds = _module.AddInstruction(
|
|
SpirvOp.Select, _boolType,
|
|
_module.AddInstruction(SpirvOp.IEqual, _boolType, selector, UInt(component + 4)),
|
|
componentBounds[component], selectedInBounds);
|
|
}
|
|
|
|
inBounds = _module.AddInstruction(SpirvOp.LogicalAnd, _boolType, inBounds, selectedInBounds);
|
|
}
|
|
|
|
var one = Gfx10FormatOne(numberFormat);
|
|
for (uint destination = 0; destination < componentCount; destination++)
|
|
{
|
|
var selector = selectors[destination];
|
|
var constant = SelectUInt(selector, 1, one, UInt(0));
|
|
var value = constant;
|
|
value = SelectUInt(selector, 4, canonical[0], value);
|
|
value = SelectUInt(selector, 5, canonical[1], value);
|
|
value = SelectUInt(selector, 6, canonical[2], value);
|
|
value = SelectUInt(selector, 7, canonical[3], value);
|
|
StoreV(
|
|
vectorData + destination,
|
|
_module.AddInstruction(SpirvOp.Select, _uintType, inBounds, value, constant));
|
|
}
|
|
}
|
|
|
|
// Check the first and last dwords of the required range together.
|
|
private uint IsBufferElementInRange(int bindingIndex, uint byteAddress, uint lastByteOffset)
|
|
{
|
|
var first = IsBufferWordInRange(bindingIndex, ShiftRightLogical(byteAddress, UInt(2)));
|
|
var last = IsBufferWordInRange(
|
|
bindingIndex,
|
|
ShiftRightLogical(IAdd(byteAddress, lastByteOffset), UInt(2)));
|
|
return _module.AddInstruction(SpirvOp.LogicalAnd, _boolType, first, last);
|
|
}
|
|
|
|
// Component i of a typed load comes from memory component i; components the
|
|
// format does not have read as zero. An unbound descriptor reads as zero.
|
|
private bool TryEmitTypedBufferFormatLoad(
|
|
int bindingIndex,
|
|
uint byteAddress,
|
|
Gen5BufferMemoryControl control,
|
|
uint descriptorWord3)
|
|
{
|
|
if (!Gfx10UnifiedFormat.TryDecode(control.TypedFormat, out var dataFormat, out var numberFormat))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
var componentCount = Gfx10UnifiedFormat.ComponentCount(dataFormat);
|
|
if (componentCount == 0)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
// All transferred components must be bound and inside the binding.
|
|
var valid = _module.AddInstruction(
|
|
SpirvOp.LogicalAnd,
|
|
_boolType,
|
|
IsDescriptorBound(descriptorWord3),
|
|
IsBufferElementInRange(
|
|
bindingIndex,
|
|
byteAddress,
|
|
UInt(Gfx10UnifiedFormat.GetAccessByteSize(dataFormat, control.DwordCount) - 1)));
|
|
var dataFormatId = UInt(dataFormat);
|
|
var numberFormatId = UInt(numberFormat);
|
|
for (uint destination = 0; destination < control.DwordCount; destination++)
|
|
{
|
|
var value = destination < componentCount
|
|
? LoadGfx10BufferFormatComponent(
|
|
bindingIndex,
|
|
byteAddress,
|
|
dataFormatId,
|
|
numberFormatId,
|
|
(int)destination,
|
|
out _)
|
|
: UInt(0);
|
|
StoreV(
|
|
control.VectorData + destination,
|
|
_module.AddInstruction(SpirvOp.Select, _uintType, valid, value, UInt(0)));
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
// A formatted store converts each register with the selected number format and places
|
|
// the bits at the component's offset; all transferred components are stored or dropped.
|
|
private bool TryEmitBufferFormatStore(
|
|
int bindingIndex,
|
|
uint byteAddress,
|
|
Gen5BufferMemoryControl control,
|
|
uint descriptorWord3,
|
|
uint unifiedFormat)
|
|
{
|
|
if (!Gfx10UnifiedFormat.TryDecode(unifiedFormat, out var dataFormat, out var numberFormat))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
var componentCount = Math.Min(control.DwordCount, Gfx10UnifiedFormat.ComponentCount(dataFormat));
|
|
if (componentCount == 0)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
var elementBytes = Gfx10UnifiedFormat.GetAccessByteSize(dataFormat, componentCount);
|
|
var allowed = _module.AddInstruction(
|
|
SpirvOp.LogicalAnd,
|
|
_boolType,
|
|
IsDescriptorBound(descriptorWord3),
|
|
IsBufferElementInRange(bindingIndex, byteAddress, UInt(elementBytes - 1)));
|
|
EmitConditional(allowed, () =>
|
|
{
|
|
if (Gfx10UnifiedFormat.HasWholeDwordComponents(dataFormat))
|
|
{
|
|
// Dword components are dword aligned and keep their register bits.
|
|
var dwordAddress = ShiftRightLogical(byteAddress, UInt(2));
|
|
for (uint component = 0; component < componentCount; component++)
|
|
{
|
|
Gfx10UnifiedFormat.TryGetComponentLayout(dataFormat, component, out var byteOffset, out _, out _);
|
|
StoreBufferWord(
|
|
bindingIndex,
|
|
byteOffset == 0 ? dwordAddress : IAdd(dwordAddress, UInt(byteOffset / 4)),
|
|
LoadV(control.VectorData + component));
|
|
}
|
|
|
|
return;
|
|
}
|
|
|
|
var element = new (uint Value, uint Mask)[(elementBytes + 3) / 4];
|
|
for (var index = 0; index < element.Length; index++)
|
|
{
|
|
element[index] = (UInt(0), 0);
|
|
}
|
|
|
|
for (uint component = 0; component < componentCount; component++)
|
|
{
|
|
Gfx10UnifiedFormat.TryGetComponentLayout(dataFormat, component, out var byteOffset, out var bitOffset, out var bitCount);
|
|
var encoded = EncodeGfx10BufferComponent(
|
|
LoadV(control.VectorData + component),
|
|
bitCount,
|
|
numberFormat,
|
|
dataFormat);
|
|
var dword = (int)(byteOffset / 4);
|
|
var bit = ((byteOffset & 3) * 8) + bitOffset;
|
|
var (value, mask) = element[dword];
|
|
element[dword] = (
|
|
BitwiseOr(value, bit == 0 ? encoded : ShiftLeftLogical(encoded, UInt(bit))),
|
|
mask | (((1u << (int)bitCount) - 1) << (int)bit));
|
|
}
|
|
|
|
StoreBufferElementBits(bindingIndex, byteAddress, element);
|
|
});
|
|
return true;
|
|
}
|
|
|
|
// Converts one register to the bits its component stores, per the number format.
|
|
private uint EncodeGfx10BufferComponent(uint value, uint bitCount, uint numberFormat, uint dataFormat)
|
|
{
|
|
if (bitCount == 32)
|
|
{
|
|
return value;
|
|
}
|
|
|
|
var mask = (1u << (int)bitCount) - 1;
|
|
var signedMaximum = mask >> 1;
|
|
var signedMinimum = -(int)signedMaximum - 1;
|
|
switch (numberFormat)
|
|
{
|
|
case 0:
|
|
return RoundedFloatToUnsigned(value, 0f, 1f, mask);
|
|
case 1:
|
|
return BitwiseAnd(RoundedFloatToSigned(value, -1f, 1f, signedMaximum), UInt(mask));
|
|
case 2:
|
|
return RoundedFloatToUnsigned(value, 0f, mask, 1f);
|
|
case 3:
|
|
return BitwiseAnd(RoundedFloatToSigned(value, signedMinimum, signedMaximum, 1f), UInt(mask));
|
|
case 4:
|
|
return Ext(38, _uintType, value, UInt(mask));
|
|
case 5:
|
|
return BitwiseAnd(
|
|
Bitcast(
|
|
_uintType,
|
|
Ext(
|
|
45,
|
|
_intType,
|
|
Bitcast(_intType, value),
|
|
_module.Constant(_intType, unchecked((uint)signedMinimum)),
|
|
_module.Constant(_intType, signedMaximum))),
|
|
UInt(mask));
|
|
case 7:
|
|
if (dataFormat is 6 or 7)
|
|
{
|
|
return EncodeUnsignedMiniFloat(value, bitCount);
|
|
}
|
|
|
|
if (bitCount == 16)
|
|
{
|
|
var pair = _module.AddInstruction(
|
|
SpirvOp.CompositeConstruct,
|
|
_vec2Type,
|
|
Bitcast(_floatType, value),
|
|
Float(0f));
|
|
return BitwiseAnd(Ext(58, _uintType, pair), UInt(0xFFFF));
|
|
}
|
|
|
|
return BitwiseAnd(value, UInt(mask));
|
|
default:
|
|
return BitwiseAnd(value, UInt(mask));
|
|
}
|
|
}
|
|
|
|
// NaN stores as zero; the value is clamped, scaled and rounded to nearest even.
|
|
private uint ClampedScaledFloat(uint value, float minimum, float maximum, float scale)
|
|
{
|
|
var input = Bitcast(_floatType, value);
|
|
input = _module.AddInstruction(
|
|
SpirvOp.Select,
|
|
_floatType,
|
|
_module.AddInstruction(SpirvOp.IsNan, _boolType, input),
|
|
Float(0f),
|
|
input);
|
|
var clamped = Ext(43, _floatType, input, Float(minimum), Float(maximum));
|
|
if (scale != 1f)
|
|
{
|
|
clamped = _module.AddInstruction(SpirvOp.FMul, _floatType, clamped, Float(scale));
|
|
}
|
|
|
|
return Ext(2, _floatType, clamped);
|
|
}
|
|
|
|
private uint RoundedFloatToUnsigned(uint value, float minimum, float maximum, float scale) =>
|
|
_module.AddInstruction(
|
|
SpirvOp.ConvertFToU,
|
|
_uintType,
|
|
ClampedScaledFloat(value, minimum, maximum, scale));
|
|
|
|
private uint RoundedFloatToSigned(uint value, float minimum, float maximum, float scale) =>
|
|
Bitcast(
|
|
_uintType,
|
|
_module.AddInstruction(
|
|
SpirvOp.ConvertFToS,
|
|
_intType,
|
|
ClampedScaledFloat(value, minimum, maximum, scale)));
|
|
|
|
// Encodes a float as an unsigned 10 or 11 bit mini-float, rounding to nearest even.
|
|
private uint EncodeUnsignedMiniFloat(uint value, uint bitCount)
|
|
{
|
|
var mantissaBits = (int)bitCount - 5;
|
|
var shift = 23 - mantissaBits;
|
|
var mantissaMask = (1u << mantissaBits) - 1;
|
|
var input = Bitcast(_floatType, value);
|
|
var isNan = _module.AddInstruction(SpirvOp.IsNan, _boolType, input);
|
|
var positive = _module.AddInstruction(SpirvOp.FOrdGreaterThan, _boolType, input, Float(0f));
|
|
var exponent = BitwiseAnd(ShiftRightLogical(value, UInt(23)), UInt(0xFF));
|
|
|
|
// Below 2^-14 the result is a denormal: the mantissa alone scales the value.
|
|
var denormal = _module.AddInstruction(
|
|
SpirvOp.ConvertFToU,
|
|
_uintType,
|
|
Ext(
|
|
2,
|
|
_floatType,
|
|
_module.AddInstruction(
|
|
SpirvOp.FMul,
|
|
_floatType,
|
|
input,
|
|
Float(1u << (14 + mantissaBits)))));
|
|
|
|
// Rounding the dropped mantissa bits may carry into the exponent.
|
|
var roundBias = IAdd(
|
|
UInt((1u << (shift - 1)) - 1),
|
|
BitwiseAnd(ShiftRightLogical(value, UInt((uint)shift)), UInt(1)));
|
|
var rounded = IAdd(value, roundBias);
|
|
var roundedExponent = BitwiseAnd(ShiftRightLogical(rounded, UInt(23)), UInt(0xFF));
|
|
var normal = BitwiseOr(
|
|
ShiftLeftLogical(
|
|
_module.AddInstruction(SpirvOp.ISub, _uintType, roundedExponent, UInt(112)),
|
|
UInt((uint)mantissaBits)),
|
|
BitwiseAnd(ShiftRightLogical(rounded, UInt((uint)shift)), UInt(mantissaMask)));
|
|
var overflow = _module.AddInstruction(
|
|
SpirvOp.UGreaterThanEqual,
|
|
_boolType,
|
|
roundedExponent,
|
|
UInt(143));
|
|
var result = _module.AddInstruction(
|
|
SpirvOp.Select,
|
|
_uintType,
|
|
overflow,
|
|
UInt(31u << mantissaBits),
|
|
normal);
|
|
result = _module.AddInstruction(
|
|
SpirvOp.Select,
|
|
_uintType,
|
|
_module.AddInstruction(SpirvOp.ULessThan, _boolType, exponent, UInt(113)),
|
|
denormal,
|
|
result);
|
|
result = _module.AddInstruction(SpirvOp.Select, _uintType, positive, result, UInt(0));
|
|
return _module.AddInstruction(
|
|
SpirvOp.Select,
|
|
_uintType,
|
|
isNan,
|
|
UInt((1u << (int)bitCount) - 1),
|
|
result);
|
|
}
|
|
|
|
// A descriptor whose format field is INVALID is not bound.
|
|
private uint IsDescriptorBound(uint descriptorWord3)
|
|
{
|
|
var descriptorFormat = BitwiseAnd(
|
|
ShiftRightLogical(descriptorWord3, UInt(12)),
|
|
UInt(0x7F));
|
|
return _module.AddInstruction(SpirvOp.INotEqual, _boolType, descriptorFormat, UInt(0));
|
|
}
|
|
|
|
private (uint DataFormat, uint NumberFormat) DecodeGfx10BufferFormat(
|
|
uint unifiedFormat)
|
|
{
|
|
// A specialized descriptor makes the format a translation-time constant.
|
|
if (_module.TryGetConstantValue(unifiedFormat, out var knownFormat) && knownFormat < 128)
|
|
{
|
|
Gfx10UnifiedFormat.TryDecode(knownFormat, out var knownDataFormat, out var knownNumberFormat);
|
|
return (UInt(knownDataFormat), UInt(knownNumberFormat));
|
|
}
|
|
|
|
// The descriptor is loaded at execution time, so format decoding
|
|
// must remain dynamic too. Generate one module-level lookup table
|
|
// from the same authoritative decoder used by descriptor
|
|
// evaluation rather than specializing the shader to the SRD seen
|
|
// at compile time (compiled compute shaders may be reused with new
|
|
// SRDs). A table also avoids emitting 77 compares at every format
|
|
// load site, which matters in buffer-heavy compute kernels.
|
|
if (_gfx10BufferFormatTable == 0)
|
|
{
|
|
const uint formatCount = 128;
|
|
var entries = new uint[formatCount];
|
|
for (uint format = 0; format < formatCount; format++)
|
|
{
|
|
Gfx10UnifiedFormat.TryDecode(
|
|
format,
|
|
out var decodedDataFormat,
|
|
out var decodedNumberFormat);
|
|
entries[format] = UInt(
|
|
decodedDataFormat | (decodedNumberFormat << 8));
|
|
}
|
|
|
|
var tableType = _module.TypeArray(_uintType, formatCount);
|
|
var tablePointer = _module.TypePointer(
|
|
SpirvStorageClass.Private,
|
|
tableType);
|
|
_gfx10BufferFormatTable = _module.AddGlobalVariable(
|
|
tablePointer,
|
|
SpirvStorageClass.Private,
|
|
_module.ConstantComposite(tableType, entries));
|
|
_module.AddName(_gfx10BufferFormatTable, "gfx10BufferFormats");
|
|
_interfaces.Add(_gfx10BufferFormatTable);
|
|
}
|
|
|
|
var entryPointer = _module.AddInstruction(
|
|
SpirvOp.AccessChain,
|
|
_privateUintPointer,
|
|
_gfx10BufferFormatTable,
|
|
unifiedFormat);
|
|
var entry = Load(_uintType, entryPointer);
|
|
return (
|
|
BitwiseAnd(entry, UInt(0xFF)),
|
|
BitwiseAnd(
|
|
ShiftRightLogical(entry, UInt(8)),
|
|
UInt(0xFF)));
|
|
}
|
|
|
|
private uint LoadGfx10BufferFormatComponent(
|
|
int bindingIndex,
|
|
uint elementAddress,
|
|
uint dataFormat,
|
|
uint numberFormat,
|
|
int component,
|
|
out uint componentInBounds)
|
|
{
|
|
var byteOffset = UInt(0);
|
|
var bitOffset = UInt(0);
|
|
var bitCount = UInt(0);
|
|
|
|
void SetLayout(uint format, uint bytes, uint bits, uint count)
|
|
{
|
|
var matches = _module.AddInstruction(
|
|
SpirvOp.IEqual,
|
|
_boolType,
|
|
dataFormat,
|
|
UInt(format));
|
|
byteOffset = _module.AddInstruction(
|
|
SpirvOp.Select,
|
|
_uintType,
|
|
matches,
|
|
UInt(bytes),
|
|
byteOffset);
|
|
bitOffset = _module.AddInstruction(
|
|
SpirvOp.Select,
|
|
_uintType,
|
|
matches,
|
|
UInt(bits),
|
|
bitOffset);
|
|
bitCount = _module.AddInstruction(
|
|
SpirvOp.Select,
|
|
_uintType,
|
|
matches,
|
|
UInt(count),
|
|
bitCount);
|
|
}
|
|
|
|
// The layout is selected at run time from the descriptor's data format.
|
|
foreach (var layout in Gfx10UnifiedFormat.ComponentLayouts)
|
|
{
|
|
if (layout.Component == (uint)component)
|
|
{
|
|
SetLayout(layout.DataFormat, layout.ByteOffset, layout.BitOffset, layout.BitCount);
|
|
}
|
|
}
|
|
|
|
var packed = LoadUnalignedBufferWord(
|
|
bindingIndex,
|
|
IAdd(elementAddress, byteOffset));
|
|
var componentBytes = ShiftRightLogical(IAdd(IAdd(bitOffset, bitCount), UInt(7)), UInt(3));
|
|
componentInBounds = _module.AddInstruction(
|
|
SpirvOp.LogicalOr, _boolType,
|
|
_module.AddInstruction(SpirvOp.IEqual, _boolType, bitCount, UInt(0)),
|
|
IsBufferElementInRange(bindingIndex, IAdd(elementAddress, byteOffset),
|
|
_module.AddInstruction(SpirvOp.ISub, _uintType, componentBytes, UInt(1))));
|
|
var raw = _module.AddInstruction(
|
|
SpirvOp.BitFieldUExtract,
|
|
_uintType,
|
|
packed,
|
|
bitOffset,
|
|
bitCount);
|
|
var converted = ConvertGfx10BufferComponent(
|
|
raw,
|
|
bitCount,
|
|
numberFormat,
|
|
dataFormat);
|
|
var valid = _module.AddInstruction(
|
|
SpirvOp.INotEqual,
|
|
_boolType,
|
|
bitCount,
|
|
UInt(0));
|
|
return _module.AddInstruction(
|
|
SpirvOp.Select,
|
|
_uintType,
|
|
valid,
|
|
converted,
|
|
component == 3 ? Gfx10FormatOne(numberFormat) : UInt(0));
|
|
}
|
|
|
|
private uint ConvertGfx10BufferComponent(
|
|
uint raw,
|
|
uint bitCount,
|
|
uint numberFormat,
|
|
uint dataFormat)
|
|
{
|
|
var widthIs32 = _module.AddInstruction(
|
|
SpirvOp.IEqual,
|
|
_boolType,
|
|
bitCount,
|
|
UInt(32));
|
|
var lowMask = _module.AddInstruction(
|
|
SpirvOp.ISub,
|
|
_uintType,
|
|
ShiftLeftLogical(UInt(1), bitCount),
|
|
UInt(1));
|
|
lowMask = _module.AddInstruction(
|
|
SpirvOp.Select,
|
|
_uintType,
|
|
widthIs32,
|
|
UInt(uint.MaxValue),
|
|
lowMask);
|
|
|
|
var signedRaw = _module.AddInstruction(
|
|
SpirvOp.BitFieldSExtract,
|
|
_intType,
|
|
Bitcast(_intType, raw),
|
|
UInt(0),
|
|
bitCount);
|
|
var signedBits = Bitcast(_uintType, signedRaw);
|
|
var unsignedFloat = _module.AddInstruction(
|
|
SpirvOp.ConvertUToF,
|
|
_floatType,
|
|
raw);
|
|
var signedFloat = _module.AddInstruction(
|
|
SpirvOp.ConvertSToF,
|
|
_floatType,
|
|
signedRaw);
|
|
|
|
var unorm = Bitcast(
|
|
_uintType,
|
|
_module.AddInstruction(
|
|
SpirvOp.FDiv,
|
|
_floatType,
|
|
unsignedFloat,
|
|
_module.AddInstruction(
|
|
SpirvOp.ConvertUToF,
|
|
_floatType,
|
|
lowMask)));
|
|
var signedMaximum = ShiftRightLogical(lowMask, UInt(1));
|
|
var snormFloat = _module.AddInstruction(
|
|
SpirvOp.FDiv,
|
|
_floatType,
|
|
signedFloat,
|
|
_module.AddInstruction(
|
|
SpirvOp.ConvertUToF,
|
|
_floatType,
|
|
signedMaximum));
|
|
snormFloat = _module.AddInstruction(
|
|
SpirvOp.Select,
|
|
_floatType,
|
|
_module.AddInstruction(
|
|
SpirvOp.FOrdLessThan,
|
|
_boolType,
|
|
snormFloat,
|
|
Float(-1f)),
|
|
Float(-1f),
|
|
snormFloat);
|
|
var snorm = Bitcast(_uintType, snormFloat);
|
|
var uscaled = Bitcast(_uintType, unsignedFloat);
|
|
var sscaled = Bitcast(_uintType, signedFloat);
|
|
|
|
var unpackedHalf = Ext(62, _vec2Type, BitwiseAnd(raw, UInt(0xFFFF)));
|
|
var half = Bitcast(
|
|
_uintType,
|
|
_module.AddInstruction(
|
|
SpirvOp.CompositeExtract,
|
|
_floatType,
|
|
unpackedHalf,
|
|
0));
|
|
var floating = _module.AddInstruction(
|
|
SpirvOp.Select,
|
|
_uintType,
|
|
_module.AddInstruction(
|
|
SpirvOp.IEqual,
|
|
_boolType,
|
|
bitCount,
|
|
UInt(16)),
|
|
half,
|
|
raw);
|
|
|
|
// DATA_FORMAT 10_11_11 and 11_11_10 use unsigned mini-floats
|
|
// when NUM_FORMAT is FLOAT, not ordinary integer bit patterns.
|
|
var isPackedFloat = _module.AddInstruction(
|
|
SpirvOp.LogicalOr,
|
|
_boolType,
|
|
_module.AddInstruction(
|
|
SpirvOp.IEqual,
|
|
_boolType,
|
|
dataFormat,
|
|
UInt(6)),
|
|
_module.AddInstruction(
|
|
SpirvOp.IEqual,
|
|
_boolType,
|
|
dataFormat,
|
|
UInt(7)));
|
|
floating = _module.AddInstruction(
|
|
SpirvOp.Select,
|
|
_uintType,
|
|
isPackedFloat,
|
|
DecodeUnsignedMiniFloat(raw, bitCount),
|
|
floating);
|
|
|
|
var result = raw;
|
|
result = SelectUInt(numberFormat, 0, unorm, result);
|
|
result = SelectUInt(numberFormat, 1, snorm, result);
|
|
result = SelectUInt(numberFormat, 2, uscaled, result);
|
|
result = SelectUInt(numberFormat, 3, sscaled, result);
|
|
result = SelectUInt(numberFormat, 4, raw, result);
|
|
result = SelectUInt(numberFormat, 5, signedBits, result);
|
|
result = SelectUInt(numberFormat, 7, floating, result);
|
|
return result;
|
|
}
|
|
|
|
private uint DecodeUnsignedMiniFloat(uint raw, uint bitCount)
|
|
{
|
|
var mantissaBits = _module.AddInstruction(
|
|
SpirvOp.ISub,
|
|
_uintType,
|
|
bitCount,
|
|
UInt(5));
|
|
var mantissaMask = _module.AddInstruction(
|
|
SpirvOp.ISub,
|
|
_uintType,
|
|
ShiftLeftLogical(UInt(1), mantissaBits),
|
|
UInt(1));
|
|
var mantissa = BitwiseAnd(raw, mantissaMask);
|
|
var exponent = BitwiseAnd(
|
|
ShiftRightLogical(raw, mantissaBits),
|
|
UInt(0x1F));
|
|
var mantissaShift = _module.AddInstruction(
|
|
SpirvOp.ISub,
|
|
_uintType,
|
|
UInt(23),
|
|
mantissaBits);
|
|
var normalBits = BitwiseOr(
|
|
ShiftLeftLogical(IAdd(exponent, UInt(112)), UInt(23)),
|
|
ShiftLeftLogical(mantissa, mantissaShift));
|
|
var subnormal = Bitcast(
|
|
_uintType,
|
|
_module.AddInstruction(
|
|
SpirvOp.FMul,
|
|
_floatType,
|
|
_module.AddInstruction(
|
|
SpirvOp.ConvertUToF,
|
|
_floatType,
|
|
mantissa),
|
|
_module.AddInstruction(
|
|
SpirvOp.Select,
|
|
_floatType,
|
|
_module.AddInstruction(
|
|
SpirvOp.IEqual,
|
|
_boolType,
|
|
mantissaBits,
|
|
UInt(6)),
|
|
Float(1f / 1_048_576f), // 2^-20 for 11-bit UFLOAT
|
|
Float(1f / 524_288f)))); // 2^-19 for 10-bit UFLOAT
|
|
var special = BitwiseOr(
|
|
UInt(0x7F800000),
|
|
ShiftLeftLogical(mantissa, mantissaShift));
|
|
var result = _module.AddInstruction(
|
|
SpirvOp.Select,
|
|
_uintType,
|
|
_module.AddInstruction(
|
|
SpirvOp.IEqual,
|
|
_boolType,
|
|
exponent,
|
|
UInt(0)),
|
|
subnormal,
|
|
normalBits);
|
|
return _module.AddInstruction(
|
|
SpirvOp.Select,
|
|
_uintType,
|
|
_module.AddInstruction(
|
|
SpirvOp.IEqual,
|
|
_boolType,
|
|
exponent,
|
|
UInt(31)),
|
|
special,
|
|
result);
|
|
}
|
|
|
|
private uint Gfx10FormatOne(uint numberFormat)
|
|
{
|
|
var isUint = _module.AddInstruction(
|
|
SpirvOp.IEqual,
|
|
_boolType,
|
|
numberFormat,
|
|
UInt(4));
|
|
var isSint = _module.AddInstruction(
|
|
SpirvOp.IEqual,
|
|
_boolType,
|
|
numberFormat,
|
|
UInt(5));
|
|
return _module.AddInstruction(
|
|
SpirvOp.Select,
|
|
_uintType,
|
|
_module.AddInstruction(
|
|
SpirvOp.LogicalOr,
|
|
_boolType,
|
|
isUint,
|
|
isSint),
|
|
UInt(1),
|
|
UInt(0x3F800000));
|
|
}
|
|
|
|
private uint SelectUInt(
|
|
uint selector,
|
|
uint expected,
|
|
uint whenTrue,
|
|
uint whenFalse) =>
|
|
_module.AddInstruction(
|
|
SpirvOp.Select,
|
|
_uintType,
|
|
_module.AddInstruction(
|
|
SpirvOp.IEqual,
|
|
_boolType,
|
|
selector,
|
|
UInt(expected)),
|
|
whenTrue,
|
|
whenFalse);
|
|
|
|
private uint LoadUnalignedBufferWord(int bindingIndex, uint byteAddress)
|
|
{
|
|
var result = UInt(0);
|
|
for (uint index = 0; index < 4; index++)
|
|
{
|
|
var address = index == 0
|
|
? byteAddress
|
|
: IAdd(byteAddress, UInt(index));
|
|
var dwordAddress = ShiftRightLogical(address, UInt(2));
|
|
var bitOffset = ShiftLeftLogical(BitwiseAnd(address, UInt(3)), UInt(3));
|
|
var value = BitwiseAnd(
|
|
ShiftRightLogical(LoadBufferWord(bindingIndex, dwordAddress), bitOffset),
|
|
UInt(0xFF));
|
|
result = BitwiseOr(result, ShiftLeftLogical(value, UInt(index * 8)));
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
private uint LoadSubdwordBufferValue(
|
|
int bindingIndex,
|
|
uint byteAddress,
|
|
uint previous,
|
|
uint byteCount,
|
|
bool signExtend,
|
|
bool d16,
|
|
bool d16High)
|
|
{
|
|
var width = byteCount * 8;
|
|
var raw = BitwiseAnd(
|
|
LoadUnalignedBufferWord(bindingIndex, byteAddress),
|
|
UInt(byteCount == 1 ? 0xFFu : 0xFFFFu));
|
|
if (signExtend)
|
|
{
|
|
raw = Bitcast(
|
|
_uintType,
|
|
_module.AddInstruction(
|
|
SpirvOp.BitFieldSExtract,
|
|
_intType,
|
|
Bitcast(_intType, raw),
|
|
UInt(0),
|
|
UInt(width)));
|
|
}
|
|
|
|
if (!d16)
|
|
{
|
|
return raw;
|
|
}
|
|
|
|
var half = BitwiseAnd(raw, UInt(0xFFFF));
|
|
return d16High
|
|
? BitwiseOr(
|
|
BitwiseAnd(previous, UInt(0x0000_FFFF)),
|
|
ShiftLeftLogical(half, UInt(16)))
|
|
: BitwiseOr(
|
|
BitwiseAnd(previous, UInt(0xFFFF_0000)),
|
|
half);
|
|
}
|
|
|
|
// A byte or short store merges into its dword atomically, so neighbouring
|
|
// sub-word stores from other invocations keep their bytes.
|
|
private void StoreBufferBytes(
|
|
int bindingIndex,
|
|
uint byteAddress,
|
|
uint value,
|
|
uint byteCount,
|
|
uint sourceShift)
|
|
{
|
|
if (sourceShift != 0)
|
|
{
|
|
value = ShiftRightLogical(value, UInt(sourceShift));
|
|
}
|
|
|
|
StoreBufferElementBits(
|
|
bindingIndex,
|
|
byteAddress,
|
|
[(value, byteCount == 1 ? 0xFFu : 0xFFFFu)]);
|
|
}
|
|
|
|
// Stores an element of up to four dwords at any byte alignment. Each touched
|
|
// dword keeps the bits outside the element's mask.
|
|
private void StoreBufferElementBits(
|
|
int bindingIndex,
|
|
uint byteAddress,
|
|
IReadOnlyList<(uint Value, uint Mask)> element)
|
|
{
|
|
var alignment = BitwiseAnd(byteAddress, UInt(3));
|
|
var shift = ShiftLeftLogical(alignment, UInt(3));
|
|
var aligned = _module.AddInstruction(SpirvOp.IEqual, _boolType, alignment, UInt(0));
|
|
var carryShift = _module.AddInstruction(SpirvOp.ISub, _uintType, UInt(32), shift);
|
|
var firstDword = ShiftRightLogical(byteAddress, UInt(2));
|
|
for (var index = 0; index <= element.Count; index++)
|
|
{
|
|
var value = UInt(0);
|
|
var mask = UInt(0);
|
|
if (index < element.Count)
|
|
{
|
|
var (elementValue, elementMask) = element[index];
|
|
value = ShiftLeftLogical(BitwiseAnd(elementValue, UInt(elementMask)), shift);
|
|
mask = ShiftLeftLogical(UInt(elementMask), shift);
|
|
}
|
|
|
|
if (index > 0)
|
|
{
|
|
// An unaligned element carries its high bits into the next dword.
|
|
var (previousValue, previousMask) = element[index - 1];
|
|
value = BitwiseOr(
|
|
value,
|
|
_module.AddInstruction(
|
|
SpirvOp.Select,
|
|
_uintType,
|
|
aligned,
|
|
UInt(0),
|
|
ShiftRightLogical(BitwiseAnd(previousValue, UInt(previousMask)), carryShift)));
|
|
mask = BitwiseOr(
|
|
mask,
|
|
_module.AddInstruction(
|
|
SpirvOp.Select,
|
|
_uintType,
|
|
aligned,
|
|
UInt(0),
|
|
ShiftRightLogical(UInt(previousMask), carryShift)));
|
|
}
|
|
|
|
StoreBufferMaskedWord(
|
|
bindingIndex,
|
|
index == 0 ? firstDword : IAdd(firstDword, UInt((uint)index)),
|
|
value,
|
|
mask);
|
|
}
|
|
}
|
|
|
|
// Writes the masked bits of one dword: a plain store for a full mask, else an
|
|
// atomic read-modify-write that keeps the other bits.
|
|
private void StoreBufferMaskedWord(int bindingIndex, uint dwordAddress, uint value, uint mask)
|
|
{
|
|
var touched = _module.AddInstruction(
|
|
SpirvOp.LogicalAnd,
|
|
_boolType,
|
|
_module.AddInstruction(SpirvOp.INotEqual, _boolType, mask, UInt(0)),
|
|
IsBufferWordInRange(bindingIndex, dwordAddress));
|
|
EmitConditional(touched, () =>
|
|
{
|
|
var pointer = BufferWordPointer(bindingIndex, dwordAddress);
|
|
var full = _module.AddInstruction(SpirvOp.IEqual, _boolType, mask, UInt(uint.MaxValue));
|
|
EmitConditional(
|
|
full,
|
|
() => Store(pointer, value),
|
|
() => EmitAtomicWordUpdate(
|
|
pointer,
|
|
observed => BitwiseOr(
|
|
BitwiseAnd(observed, _module.AddInstruction(SpirvOp.Not, _uintType, mask)),
|
|
value)));
|
|
});
|
|
}
|
|
|
|
// Compare-exchange loop: the merge runs on the observed word until the
|
|
// exchange succeeds. Returns the observed (pre-exchange) word at the
|
|
// point the loop exits, which is a valid use here because it is
|
|
// defined by the OpPhi in `header`, and `header` dominates `mergeLabel`.
|
|
private uint EmitAtomicWordUpdate(uint pointer, Func<uint, uint> merge)
|
|
{
|
|
var preheader = _module.AllocateId();
|
|
var header = _module.AllocateId();
|
|
var continueLabel = _module.AllocateId();
|
|
var mergeLabel = _module.AllocateId();
|
|
var exchanged = _module.AllocateId();
|
|
_module.AddStatement(SpirvOp.Branch, preheader);
|
|
_module.AddLabel(preheader);
|
|
var initial = _module.AddInstruction(SpirvOp.AtomicLoad, _uintType, pointer, UInt(1), UInt(0));
|
|
_module.AddStatement(SpirvOp.Branch, header);
|
|
_module.AddLabel(header);
|
|
var observed = _module.AddInstruction(
|
|
SpirvOp.Phi,
|
|
_uintType,
|
|
initial,
|
|
preheader,
|
|
exchanged,
|
|
continueLabel);
|
|
var next = merge(observed);
|
|
// The phi above names this result before it is emitted.
|
|
_module.AddStatement(
|
|
SpirvOp.AtomicCompareExchange,
|
|
_uintType,
|
|
exchanged,
|
|
pointer,
|
|
UInt(1),
|
|
UInt(0),
|
|
UInt(0),
|
|
next,
|
|
observed);
|
|
var success = _module.AddInstruction(SpirvOp.IEqual, _boolType, exchanged, observed);
|
|
_module.AddStatement(SpirvOp.LoopMerge, mergeLabel, continueLabel, 0);
|
|
_module.AddStatement(SpirvOp.BranchConditional, success, mergeLabel, continueLabel);
|
|
_module.AddLabel(continueLabel);
|
|
_module.AddStatement(SpirvOp.Branch, header);
|
|
_module.AddLabel(mergeLabel);
|
|
return observed;
|
|
}
|
|
|
|
private static bool TryGetSubdwordLoadInfo(
|
|
string opcode,
|
|
out uint byteCount,
|
|
out bool signExtend,
|
|
out bool d16,
|
|
out bool d16High)
|
|
{
|
|
byteCount = opcode.Contains("byte", StringComparison.OrdinalIgnoreCase) ? 1u : 2u;
|
|
signExtend = opcode.Contains("Sbyte", StringComparison.Ordinal) ||
|
|
opcode.Contains("Sshort", StringComparison.Ordinal);
|
|
d16 = opcode.Contains("D16", StringComparison.Ordinal);
|
|
d16High = opcode.EndsWith("D16Hi", StringComparison.Ordinal);
|
|
return opcode.Contains("LoadUbyte", StringComparison.Ordinal) ||
|
|
opcode.Contains("LoadSbyte", StringComparison.Ordinal) ||
|
|
opcode.Contains("LoadUshort", StringComparison.Ordinal) ||
|
|
opcode.Contains("LoadSshort", StringComparison.Ordinal) ||
|
|
opcode.Contains("LoadShortD16", StringComparison.Ordinal);
|
|
}
|
|
|
|
private static bool TryGetSubdwordStoreInfo(
|
|
string opcode,
|
|
out uint byteCount,
|
|
out uint sourceShift)
|
|
{
|
|
byteCount = opcode.Contains("StoreByte", StringComparison.Ordinal) ? 1u : 2u;
|
|
sourceShift = opcode.EndsWith("D16Hi", StringComparison.Ordinal) ? 16u : 0u;
|
|
return opcode.Contains("StoreByte", StringComparison.Ordinal) ||
|
|
opcode.Contains("StoreShort", StringComparison.Ordinal);
|
|
}
|
|
|
|
private static bool IsFormatBufferLoad(string opcode) =>
|
|
opcode.StartsWith("BufferLoadFormat", StringComparison.Ordinal) ||
|
|
opcode.StartsWith("TBufferLoadFormat", StringComparison.Ordinal);
|
|
|
|
private static bool UsesSampler(string opcode) =>
|
|
opcode.StartsWith("ImageSample", StringComparison.Ordinal) ||
|
|
opcode.StartsWith("ImageGather", StringComparison.Ordinal) ||
|
|
opcode == "ImageGetLod";
|
|
|
|
private bool TryEmitVertexInputFetch(
|
|
Gen5BufferMemoryControl control,
|
|
SpirvVertexInput input,
|
|
out string error)
|
|
{
|
|
error = string.Empty;
|
|
if (control.DwordCount == 0)
|
|
{
|
|
error =
|
|
$"invalid vertex input fetch components={control.DwordCount}";
|
|
return false;
|
|
}
|
|
|
|
var loaded = Load(input.Type, input.Variable);
|
|
for (uint component = 0; component < control.DwordCount; component++)
|
|
{
|
|
uint raw;
|
|
var selector = (input.DestinationSelect >> (int)(component * 3)) & 0x7u;
|
|
if (selector == 0)
|
|
{
|
|
raw = UInt(0);
|
|
}
|
|
else if (selector == 1)
|
|
{
|
|
raw = UInt(input.NumberFormat is 4u or 5u ? 1u : 0x3F80_0000u);
|
|
}
|
|
else if (selector is >= 4u and <= 7u)
|
|
{
|
|
var sourceComponent = selector - 4u;
|
|
if (sourceComponent >= input.ComponentCount)
|
|
{
|
|
error =
|
|
$"vertex input destination selector exceeds source components: selector={selector} components={input.ComponentCount}";
|
|
return false;
|
|
}
|
|
|
|
var value = input.ComponentCount == 1
|
|
? loaded
|
|
: _module.AddInstruction(
|
|
SpirvOp.CompositeExtract,
|
|
input.ComponentType,
|
|
loaded,
|
|
sourceComponent);
|
|
raw = input.ComponentKind == VertexInputComponentKind.Uint
|
|
? value
|
|
: Bitcast(_uintType, value);
|
|
}
|
|
else
|
|
{
|
|
error = $"unsupported vertex input destination selector={selector}";
|
|
return false;
|
|
}
|
|
|
|
StoreV(control.VectorData + component, raw);
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
private bool TryEmitImage(
|
|
Gen5ShaderInstruction instruction,
|
|
Gen5ImageControl image,
|
|
out string error)
|
|
{
|
|
error = string.Empty;
|
|
SpirvImageResource resource;
|
|
uint imageObject;
|
|
uint dstSelect;
|
|
uint mipLevel;
|
|
{
|
|
if (TryGetImageElementCases(instruction, image, out var selector, out var elements, out error))
|
|
{
|
|
// One case per descriptor over a constant element, like a switch on the selector.
|
|
var emitted = true;
|
|
var caseError = string.Empty;
|
|
for (var index = 0; index < elements.Count && emitted; index++)
|
|
{
|
|
var elementCase = elements[index];
|
|
EmitConditional(_module.AddInstruction(SpirvOp.IEqual, _boolType, selector, UInt((uint)index)), () =>
|
|
{
|
|
if (!TryResolveLayoutImage(instruction, image, out var caseResource, out var caseImageObject, out var caseDstSelect, out caseError,
|
|
elementCase))
|
|
{
|
|
emitted = false;
|
|
return;
|
|
}
|
|
|
|
// A sampled mip load keeps its mip operand; a storage case is already its own mip view.
|
|
var caseMipLevel = instruction.Opcode == "ImageLoadMip" && !caseResource.IsStorage
|
|
? LoadImageIntegerAddress(image, (int)ImageCoordinateComponentCount(caseResource))
|
|
: UInt(0);
|
|
if (!EmitImageOperation(instruction, image, caseResource, caseImageObject, caseDstSelect, caseMipLevel, out caseError))
|
|
{
|
|
emitted = false;
|
|
}
|
|
});
|
|
}
|
|
|
|
error = caseError;
|
|
return emitted;
|
|
}
|
|
|
|
if (error.Length != 0)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
if (!TryResolveLayoutImage(instruction, image, out resource, out imageObject, out dstSelect, out error))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
// A sampled mip load reads its level from the address operand after the coordinates.
|
|
mipLevel = instruction.Opcode == "ImageLoadMip"
|
|
? LoadImageIntegerAddress(image, (int)ImageCoordinateComponentCount(resource))
|
|
: UInt(0);
|
|
}
|
|
|
|
return EmitImageOperation(instruction, image, resource, imageObject, dstSelect, mipLevel, out error);
|
|
}
|
|
|
|
// One image operation over a resolved image object; its results are register writes.
|
|
private bool EmitImageOperation(
|
|
Gen5ShaderInstruction instruction,
|
|
Gen5ImageControl image,
|
|
SpirvImageResource resource,
|
|
uint imageObject,
|
|
uint dstSelect,
|
|
uint mipLevel,
|
|
out string error)
|
|
{
|
|
error = string.Empty;
|
|
if (instruction.Opcode == "ImageGetResinfo")
|
|
{
|
|
var sizeComponentCount = ImageCoordinateComponentCount(resource);
|
|
var queryImage = imageObject;
|
|
var size = _module.AddInstruction(
|
|
resource.IsStorage || resource.Multisampled
|
|
? SpirvOp.ImageQuerySize
|
|
: SpirvOp.ImageQuerySizeLod,
|
|
ImageIntegerCoordinateType(sizeComponentCount),
|
|
resource.IsStorage || resource.Multisampled
|
|
? [queryImage]
|
|
: [queryImage, LoadImageIntegerAddress(image, 0)]);
|
|
// NVIDIA's compiler crashes on an unused level query of a 3D image.
|
|
var levels = (image.Dmask & 0x8u) != 0 && !resource.IsStorage && !resource.Multisampled
|
|
? _module.AddInstruction(
|
|
SpirvOp.ImageQueryLevels,
|
|
_uintType,
|
|
queryImage)
|
|
: UInt(1);
|
|
uint outputIndex = 0;
|
|
for (uint component = 0; component < 4; component++)
|
|
{
|
|
if ((image.Dmask & (1u << (int)component)) == 0)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
uint value;
|
|
if (component < sizeComponentCount)
|
|
{
|
|
var signedValue = sizeComponentCount == 1
|
|
? size
|
|
: _module.AddInstruction(
|
|
SpirvOp.CompositeExtract,
|
|
_intType,
|
|
size,
|
|
component);
|
|
value = Bitcast(_uintType, signedValue);
|
|
}
|
|
else if (component == 3)
|
|
{
|
|
value = levels;
|
|
}
|
|
else
|
|
{
|
|
value = UInt(0);
|
|
}
|
|
|
|
StoreV(image.VectorData + outputIndex++, value);
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
if (instruction.Opcode == "ImageGetLod")
|
|
{
|
|
var coordinateComponentCount = ImageCoordinateComponentCount(resource);
|
|
var coordinates = BuildFloatCoordinates(
|
|
image,
|
|
0,
|
|
coordinateComponentCount,
|
|
resource);
|
|
var queried = _module.AddInstruction(
|
|
SpirvOp.ImageQueryLod,
|
|
_vec2Type,
|
|
imageObject,
|
|
coordinates);
|
|
uint outputIndex = 0;
|
|
var mask = image.Dmask != 0 ? image.Dmask : 1u;
|
|
for (uint component = 0; component < 2; component++)
|
|
{
|
|
if ((mask & (1u << (int)component)) == 0)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
var value = _module.AddInstruction(
|
|
SpirvOp.CompositeExtract,
|
|
_floatType,
|
|
queried,
|
|
component);
|
|
StoreV(image.VectorData + outputIndex++, Bitcast(_uintType, value));
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
if (instruction.Opcode is "ImageStore" or "ImageStoreMip")
|
|
{
|
|
if (!resource.IsStorage)
|
|
{
|
|
error = "image store is not bound as storage";
|
|
return false;
|
|
}
|
|
|
|
var coordinateComponentCount =
|
|
ImageCoordinateComponentCount(resource);
|
|
var coordinates = BuildIntegerCoordinates(
|
|
image,
|
|
0,
|
|
coordinateComponentCount);
|
|
var components = new uint[4];
|
|
for (var component = 0; component < components.Length; component++)
|
|
{
|
|
var sourceIndex = Gen5ShaderTranslator.GetImageStoreSourceIndex(
|
|
dstSelect,
|
|
image.Dmask,
|
|
component);
|
|
if (sourceIndex >= 0)
|
|
{
|
|
var raw = LoadImageStoreComponent(
|
|
image,
|
|
resource,
|
|
(uint)sourceIndex);
|
|
components[component] = resource.ComponentKind switch
|
|
{
|
|
ImageComponentKind.Sint => Bitcast(_intType, raw),
|
|
ImageComponentKind.Uint => raw,
|
|
_ => Bitcast(_floatType, raw),
|
|
};
|
|
}
|
|
else
|
|
{
|
|
components[component] = resource.ComponentKind switch
|
|
{
|
|
ImageComponentKind.Sint =>
|
|
_module.Constant(_intType, 0),
|
|
ImageComponentKind.Uint => UInt(0),
|
|
_ => Float(0),
|
|
};
|
|
}
|
|
}
|
|
|
|
var texel = _module.AddInstruction(
|
|
SpirvOp.CompositeConstruct,
|
|
resource.VectorType,
|
|
components);
|
|
texel = PackImageTexel(resource, texel);
|
|
EmitExecConditional(() =>
|
|
_module.AddStatement(
|
|
SpirvOp.ImageWrite,
|
|
imageObject,
|
|
coordinates,
|
|
texel));
|
|
|
|
return true;
|
|
}
|
|
|
|
if (instruction.Opcode.StartsWith("ImageAtomic", StringComparison.Ordinal))
|
|
{
|
|
if (!resource.IsStorage)
|
|
{
|
|
error = "image atomic is not bound as storage";
|
|
return false;
|
|
}
|
|
|
|
// IMAGE_ATOMIC_FMIN/FMAX target float-format storage images and
|
|
// have no native SPIR-V storage-image atomic without pulling in
|
|
// SPV_EXT_shader_atomic_float_min_max. Lower them as a
|
|
// compare-and-swap loop on the underlying bit pattern instead
|
|
// (same technique used by other float-atomic emulations that
|
|
// avoid that extension dependency).
|
|
if (instruction.Opcode is "ImageAtomicFmax" or "ImageAtomicFmin")
|
|
{
|
|
var isMax = instruction.Opcode == "ImageAtomicFmax";
|
|
var floatCoordinateCount = ImageCoordinateComponentCount(resource);
|
|
var floatCoordinates = BuildIntegerCoordinates(
|
|
image,
|
|
0,
|
|
floatCoordinateCount);
|
|
EmitExecConditional(() =>
|
|
{
|
|
var pointer = _module.AddInstruction(
|
|
SpirvOp.ImageTexelPointer,
|
|
_module.TypePointer(SpirvStorageClass.Image, _uintType),
|
|
resource.Variable,
|
|
floatCoordinates,
|
|
UInt(0));
|
|
var srcBits = Bitcast(_uintType, LoadV(image.VectorData));
|
|
var old = EmitAtomicWordUpdate(pointer, observed =>
|
|
{
|
|
var oldFloat = Bitcast(_floatType, observed);
|
|
var srcFloat = Bitcast(_floatType, srcBits);
|
|
var pickSrc = _module.AddInstruction(
|
|
isMax ? SpirvOp.FOrdLessThan : SpirvOp.FOrdGreaterThan,
|
|
_boolType,
|
|
oldFloat,
|
|
srcFloat);
|
|
var chosen = _module.AddInstruction(
|
|
SpirvOp.Select, _floatType, pickSrc, srcFloat, oldFloat);
|
|
return Bitcast(_uintType, chosen);
|
|
});
|
|
if (image.Glc)
|
|
{
|
|
StoreV(image.VectorData, old);
|
|
}
|
|
});
|
|
return true;
|
|
}
|
|
|
|
if (resource.ComponentKind == ImageComponentKind.Float ||
|
|
!TryGetAtomicOp(instruction.Opcode["ImageAtomic".Length..], out var atomicOp))
|
|
{
|
|
error = $"unsupported storage image opcode {instruction.Opcode}";
|
|
return false;
|
|
}
|
|
|
|
var signed = resource.ComponentKind == ImageComponentKind.Sint;
|
|
var coordinateComponentCount =
|
|
ImageCoordinateComponentCount(resource);
|
|
var coordinates = BuildIntegerCoordinates(
|
|
image,
|
|
0,
|
|
coordinateComponentCount);
|
|
EmitExecConditional(() =>
|
|
{
|
|
var pointer = _module.AddInstruction(
|
|
SpirvOp.ImageTexelPointer,
|
|
_module.TypePointer(SpirvStorageClass.Image, resource.ComponentType),
|
|
resource.Variable,
|
|
coordinates,
|
|
UInt(0));
|
|
uint LoadData(uint register) => signed
|
|
? Bitcast(_intType, LoadV(register))
|
|
: LoadV(register);
|
|
var original = EmitAtomic(
|
|
atomicOp,
|
|
resource.ComponentType,
|
|
pointer,
|
|
scope: 1,
|
|
semantics: 0x808,
|
|
value: () => LoadData(image.VectorData),
|
|
comparator: () => LoadData(image.VectorData + 1));
|
|
if (image.Glc)
|
|
{
|
|
StoreV(
|
|
image.VectorData,
|
|
signed ? Bitcast(_uintType, original) : original);
|
|
}
|
|
});
|
|
|
|
return true;
|
|
}
|
|
|
|
if (resource.IsStorage &&
|
|
instruction.Opcode is not ("ImageLoad" or "ImageLoadMip"))
|
|
{
|
|
error = $"unsupported storage image opcode {instruction.Opcode}";
|
|
return false;
|
|
}
|
|
|
|
uint sampled;
|
|
var writeAllComponents = false;
|
|
if (instruction.Opcode is "ImageLoad" or "ImageLoadMip")
|
|
{
|
|
if (resource.IsStorage)
|
|
{
|
|
var coordinateComponentCount =
|
|
ImageCoordinateComponentCount(resource);
|
|
var coordinates = BuildIntegerCoordinates(
|
|
image,
|
|
0,
|
|
coordinateComponentCount);
|
|
sampled = _module.AddInstruction(
|
|
SpirvOp.ImageRead,
|
|
resource.VectorType,
|
|
imageObject,
|
|
coordinates);
|
|
}
|
|
else
|
|
{
|
|
// A sampled image is fetched through its image type; a request has no sampler here.
|
|
var fetchedImage = imageObject;
|
|
var coordinateComponentCount =
|
|
ImageCoordinateComponentCount(resource);
|
|
var coordinates = BuildIntegerCoordinates(
|
|
image,
|
|
0,
|
|
coordinateComponentCount);
|
|
if (resource.Multisampled)
|
|
{
|
|
var sample = LoadImageIntegerAddress(
|
|
image,
|
|
(int)coordinateComponentCount);
|
|
sampled = _module.AddInstruction(
|
|
SpirvOp.ImageFetch,
|
|
resource.VectorType,
|
|
fetchedImage,
|
|
coordinates,
|
|
0x40,
|
|
sample);
|
|
}
|
|
else
|
|
{
|
|
sampled = _module.AddInstruction(
|
|
SpirvOp.ImageFetch,
|
|
resource.VectorType,
|
|
fetchedImage,
|
|
coordinates,
|
|
2,
|
|
mipLevel);
|
|
}
|
|
}
|
|
|
|
sampled = UnpackImageTexel(resource, sampled);
|
|
}
|
|
else if (instruction.Opcode.StartsWith(
|
|
"ImageSample",
|
|
StringComparison.Ordinal))
|
|
{
|
|
var hasOffset =
|
|
instruction.Opcode.EndsWith("O", StringComparison.Ordinal);
|
|
var hasCompare =
|
|
instruction.Opcode.Contains("SampleC", StringComparison.Ordinal) &&
|
|
!instruction.Opcode.StartsWith("ImageSampleCd", StringComparison.Ordinal);
|
|
var hasGradients =
|
|
instruction.Opcode.Contains("SampleD", StringComparison.Ordinal) ||
|
|
instruction.Opcode.Contains("SampleCd", StringComparison.Ordinal) ||
|
|
instruction.Opcode.Contains("SampleCCd", StringComparison.Ordinal);
|
|
var hasZeroLod =
|
|
instruction.Opcode.Contains("Lz", StringComparison.Ordinal);
|
|
var hasLod = !hasZeroLod &&
|
|
instruction.Opcode.Contains("SampleL", StringComparison.Ordinal);
|
|
var hasBias =
|
|
instruction.Opcode.Contains("SampleB", StringComparison.Ordinal);
|
|
|
|
if (hasCompare && resource.ConversionFormat != GuestImageFormat.Invalid)
|
|
{
|
|
error = "image sample uses depth comparison with a packed integer image";
|
|
return false;
|
|
}
|
|
|
|
// RDNA MIMG address operands are ordered as
|
|
// {offset}{bias/lod}{z-compare}{derivatives}{body}. The old
|
|
// lowering treated SAMPLE_D as body-first and consequently
|
|
// sampled gradients as coordinates in every captured
|
|
// derivative operation.
|
|
var spatialComponentCount =
|
|
ImageSpatialComponentCount(resource);
|
|
var coordinateComponentCount =
|
|
ImageCoordinateComponentCount(resource);
|
|
var addressCursor = 0;
|
|
var offset = 0u;
|
|
if (hasOffset)
|
|
{
|
|
addressCursor = AlignFullImageAddress(image, addressCursor);
|
|
offset = BuildImageOffset(
|
|
image,
|
|
addressCursor,
|
|
spatialComponentCount);
|
|
addressCursor += ImageFullAddressSlots(image);
|
|
}
|
|
|
|
// SAMPLE_B prefixes the body with a bias. SAMPLE_L instead
|
|
// carries LOD as the final body component (x, y, lod for 2D),
|
|
// per the RDNA image-address table.
|
|
var lodOrBias = hasBias
|
|
? LoadImageFloatAddress(image, addressCursor++)
|
|
: 0u;
|
|
var reference = 0u;
|
|
if (hasCompare)
|
|
{
|
|
// PCF references remain full-width even when A16 packs the
|
|
// ordinary address components two per VGPR.
|
|
addressCursor = AlignFullImageAddress(image, addressCursor);
|
|
reference = Bitcast(
|
|
_floatType,
|
|
LoadV(image.GetAddressRegister(
|
|
ImageAddressRegister(image, addressCursor))));
|
|
addressCursor += ImageFullAddressSlots(image);
|
|
}
|
|
|
|
var gradientX = hasGradients
|
|
? BuildFloatCoordinates(
|
|
image,
|
|
addressCursor,
|
|
spatialComponentCount,
|
|
resource)
|
|
: 0u;
|
|
var gradientY = hasGradients
|
|
? BuildFloatCoordinates(
|
|
image,
|
|
addressCursor + (int)spatialComponentCount,
|
|
spatialComponentCount,
|
|
resource)
|
|
: 0u;
|
|
if (hasGradients)
|
|
{
|
|
addressCursor += checked((int)(spatialComponentCount * 2));
|
|
}
|
|
|
|
var coordinates = BuildFloatCoordinates(
|
|
image,
|
|
addressCursor,
|
|
coordinateComponentCount,
|
|
resource);
|
|
// Non-pixel samples require explicit derivatives or a level of detail.
|
|
// Use level zero when the instruction supplies neither.
|
|
var explicitLod = hasGradients || hasZeroLod || hasLod ||
|
|
_stage != Gen5SpirvStage.Pixel;
|
|
var lod = hasZeroLod
|
|
? Float(0)
|
|
: hasLod
|
|
? LoadImageFloatAddress(
|
|
image,
|
|
addressCursor + (int)coordinateComponentCount)
|
|
: explicitLod
|
|
? Float(0)
|
|
: lodOrBias;
|
|
if (hasOffset)
|
|
{
|
|
// Vulkan before maintenance8 forbids the dynamic Offset
|
|
// image operand on non-gather sampling operations. RDNA
|
|
// offsets are per-lane VGPR values, so ConstOffset is not
|
|
// equivalent. Fold the texel offset into normalized sample
|
|
// coordinates using the queried mip extent instead.
|
|
var offsetLod = explicitLod && !hasGradients
|
|
? lod
|
|
: Float(0);
|
|
coordinates = ApplyDynamicSampleOffset(
|
|
resource,
|
|
imageObject,
|
|
coordinates,
|
|
offset,
|
|
offsetLod);
|
|
}
|
|
|
|
var imageOperands =
|
|
hasGradients ? 4u : explicitLod ? 2u : hasBias ? 1u : 0u;
|
|
var operands = new List<uint>
|
|
{
|
|
imageObject,
|
|
coordinates,
|
|
};
|
|
|
|
if (imageOperands != 0)
|
|
{
|
|
operands.Add(imageOperands);
|
|
if (hasGradients)
|
|
{
|
|
operands.Add(gradientX);
|
|
operands.Add(gradientY);
|
|
}
|
|
else if (explicitLod)
|
|
{
|
|
operands.Add(lod);
|
|
}
|
|
else if (hasBias)
|
|
{
|
|
operands.Add(lodOrBias);
|
|
}
|
|
|
|
}
|
|
|
|
if (hasCompare && resource.EmulatedCompareFunction >= 0)
|
|
{
|
|
// A color format cannot back a Vulkan depth-compare view; compare
|
|
// the sampled first channel like RDNA does for such formats.
|
|
var texel = _module.AddInstruction(
|
|
explicitLod ? SpirvOp.ImageSampleExplicitLod : SpirvOp.ImageSampleImplicitLod,
|
|
resource.VectorType,
|
|
[.. operands]);
|
|
var depth = EmulatedDepthCompare(
|
|
reference,
|
|
_module.AddInstruction(SpirvOp.CompositeExtract, _floatType, texel, 0u),
|
|
resource.EmulatedCompareFunction);
|
|
sampled = _module.AddInstruction(
|
|
SpirvOp.CompositeConstruct,
|
|
resource.VectorType,
|
|
depth,
|
|
depth,
|
|
depth,
|
|
Float(1f));
|
|
}
|
|
else if (hasCompare)
|
|
{
|
|
// The sampler carries the compare; the depth result fills x, y, z.
|
|
var drefOperands = new List<uint> { imageObject, coordinates, reference };
|
|
drefOperands.AddRange(operands.Skip(2));
|
|
var depth = _module.AddInstruction(
|
|
explicitLod ? SpirvOp.ImageSampleDrefExplicitLod : SpirvOp.ImageSampleDrefImplicitLod,
|
|
_floatType,
|
|
[.. drefOperands]);
|
|
sampled = _module.AddInstruction(
|
|
SpirvOp.CompositeConstruct,
|
|
resource.VectorType,
|
|
depth,
|
|
depth,
|
|
depth,
|
|
Float(1f));
|
|
}
|
|
else
|
|
{
|
|
sampled = _module.AddInstruction(
|
|
explicitLod
|
|
? SpirvOp.ImageSampleExplicitLod
|
|
: SpirvOp.ImageSampleImplicitLod,
|
|
resource.VectorType,
|
|
[.. operands]);
|
|
sampled = UnpackImageTexel(resource, sampled);
|
|
}
|
|
}
|
|
else if (instruction.Opcode.StartsWith(
|
|
"ImageGather4",
|
|
StringComparison.Ordinal))
|
|
{
|
|
var hasOffset =
|
|
instruction.Opcode.EndsWith("O", StringComparison.Ordinal);
|
|
var hasCompare =
|
|
instruction.Opcode.Contains("Gather4C", StringComparison.Ordinal);
|
|
var gatherHorizontal =
|
|
instruction.Opcode == "ImageGather4H";
|
|
|
|
if (hasCompare && resource.ConversionFormat != GuestImageFormat.Invalid)
|
|
{
|
|
error = "image gather uses depth comparison with a packed integer image";
|
|
return false;
|
|
}
|
|
var spatialComponentCount =
|
|
ImageSpatialComponentCount(resource);
|
|
var coordinateComponentCount =
|
|
ImageCoordinateComponentCount(resource);
|
|
var addressCursor = 0;
|
|
var offset = 0u;
|
|
if (hasOffset)
|
|
{
|
|
offset = BuildImageOffset(
|
|
image,
|
|
addressCursor,
|
|
spatialComponentCount);
|
|
addressCursor += ImageFullAddressSlots(image);
|
|
}
|
|
|
|
var reference = 0u;
|
|
if (hasCompare)
|
|
{
|
|
addressCursor = AlignFullImageAddress(image, addressCursor);
|
|
reference = Bitcast(
|
|
_floatType,
|
|
LoadV(image.GetAddressRegister(
|
|
ImageAddressRegister(image, addressCursor))));
|
|
addressCursor += ImageFullAddressSlots(image);
|
|
}
|
|
|
|
var coordinates = BuildFloatCoordinates(
|
|
image,
|
|
addressCursor,
|
|
coordinateComponentCount,
|
|
resource);
|
|
|
|
if (resource.Dimension == SpirvImageDim.Dim1D)
|
|
{
|
|
var levelZero =
|
|
instruction.Opcode.Contains("Lz", StringComparison.Ordinal);
|
|
if (resource.Arrayed ||
|
|
hasCompare ||
|
|
hasOffset ||
|
|
gatherHorizontal ||
|
|
!levelZero)
|
|
{
|
|
error = resource.Arrayed
|
|
? "unsupported 1D-array image gather"
|
|
: "unsupported 1D image gather variant";
|
|
return false;
|
|
}
|
|
|
|
sampled = EmitOneDimensionalGatherLz(
|
|
image,
|
|
resource,
|
|
imageObject,
|
|
coordinates);
|
|
sampled = UnpackImageGather(resource, image.Dmask, sampled);
|
|
writeAllComponents = true;
|
|
goto GatherComplete;
|
|
}
|
|
|
|
var operands = new List<uint>
|
|
{
|
|
imageObject,
|
|
coordinates,
|
|
};
|
|
var emulatedCompare = hasCompare && resource.EmulatedCompareFunction >= 0;
|
|
if (emulatedCompare)
|
|
{
|
|
// Gather the first channel and compare each texel in the shader.
|
|
operands.Add(UInt(0));
|
|
}
|
|
else if (hasCompare)
|
|
{
|
|
operands.Add(reference);
|
|
}
|
|
else
|
|
{
|
|
uint component = 0;
|
|
if (resource.ConversionFormat == GuestImageFormat.Invalid)
|
|
{
|
|
while (component < 3 &&
|
|
(image.Dmask & (1u << (int)component)) == 0)
|
|
{
|
|
component++;
|
|
}
|
|
}
|
|
|
|
operands.Add(UInt(component));
|
|
}
|
|
|
|
if (hasOffset)
|
|
{
|
|
operands.Add(0x10u);
|
|
operands.Add(offset);
|
|
}
|
|
else if (gatherHorizontal)
|
|
{
|
|
if (resource.Dimension == SpirvImageDim.Dim1D)
|
|
{
|
|
error = "unsupported 1D horizontal image gather";
|
|
return false;
|
|
}
|
|
|
|
operands.Add(0x20u);
|
|
operands.Add(BuildHorizontalGatherOffsets());
|
|
}
|
|
|
|
sampled = _module.AddInstruction(
|
|
hasCompare && !emulatedCompare ? SpirvOp.ImageDrefGather : SpirvOp.ImageGather,
|
|
resource.VectorType,
|
|
[.. operands]);
|
|
if (emulatedCompare)
|
|
{
|
|
var gathered = sampled;
|
|
var compared = new uint[4];
|
|
for (var texel = 0u; texel < 4; texel++)
|
|
{
|
|
compared[texel] = EmulatedDepthCompare(
|
|
reference,
|
|
_module.AddInstruction(SpirvOp.CompositeExtract, _floatType, gathered, texel),
|
|
resource.EmulatedCompareFunction);
|
|
}
|
|
|
|
sampled = _module.AddInstruction(SpirvOp.CompositeConstruct, resource.VectorType, compared);
|
|
}
|
|
else if (!hasCompare)
|
|
{
|
|
sampled = UnpackImageGather(resource, image.Dmask, sampled);
|
|
}
|
|
|
|
writeAllComponents = true;
|
|
GatherComplete:;
|
|
}
|
|
else
|
|
{
|
|
error = $"unsupported image opcode {instruction.Opcode}";
|
|
return false;
|
|
}
|
|
|
|
var outputValues = new List<uint>(4);
|
|
for (uint component = 0; component < 4; component++)
|
|
{
|
|
if (!writeAllComponents &&
|
|
(image.Dmask & (1u << (int)component)) == 0)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
var value = _module.AddInstruction(
|
|
SpirvOp.CompositeExtract,
|
|
resource.ComponentType,
|
|
sampled,
|
|
component);
|
|
var raw = resource.ComponentKind switch
|
|
{
|
|
ImageComponentKind.Uint => value,
|
|
_ => Bitcast(_uintType, value),
|
|
};
|
|
outputValues.Add(raw);
|
|
}
|
|
|
|
if (_stage == Gen5SpirvStage.Pixel &&
|
|
PixelImageCaptureAddressMatches() &&
|
|
uint.TryParse(
|
|
Environment.GetEnvironmentVariable(
|
|
"SHARPEMU_CAPTURE_PIXEL_IMAGE_PC"),
|
|
out var captureImagePc) &&
|
|
instruction.Pc == captureImagePc)
|
|
{
|
|
var captureBase = 248u;
|
|
if (uint.TryParse(
|
|
Environment.GetEnvironmentVariable(
|
|
"SHARPEMU_CAPTURE_PIXEL_IMAGE_VGPR_BASE"),
|
|
out var requestedCaptureBase))
|
|
{
|
|
captureBase = requestedCaptureBase;
|
|
}
|
|
captureBase = captureBase <= 252 ? captureBase : 248u;
|
|
for (var component = 0; component < 4; component++)
|
|
{
|
|
StoreV(
|
|
captureBase + (uint)component,
|
|
component < outputValues.Count
|
|
? outputValues[component]
|
|
: Bitcast(_uintType, Float(1)));
|
|
}
|
|
}
|
|
|
|
if (image.D16)
|
|
{
|
|
for (var index = 0; index < outputValues.Count; index += 2)
|
|
{
|
|
var low = outputValues[index];
|
|
var high = index + 1 < outputValues.Count
|
|
? outputValues[index + 1]
|
|
: UInt(0);
|
|
StoreV(
|
|
image.VectorData + (uint)(index / 2),
|
|
PackImageD16(resource, low, high));
|
|
}
|
|
}
|
|
else
|
|
{
|
|
for (var index = 0; index < outputValues.Count; index++)
|
|
{
|
|
StoreV(image.VectorData + (uint)index, outputValues[index]);
|
|
}
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
private uint EmitOneDimensionalGatherLz(
|
|
Gen5ImageControl image,
|
|
SpirvImageResource resource,
|
|
uint sampledImage,
|
|
uint coordinate)
|
|
{
|
|
var imageValue = Load(resource.ImageType, resource.Variable);
|
|
var width = _module.AddInstruction(
|
|
SpirvOp.ImageQuerySizeLod,
|
|
_intType,
|
|
imageValue,
|
|
_module.Constant(_intType, 0));
|
|
var widthFloat = _module.AddInstruction(
|
|
SpirvOp.ConvertSToF,
|
|
_floatType,
|
|
width);
|
|
var centered = _module.AddInstruction(
|
|
SpirvOp.FSub,
|
|
_floatType,
|
|
_module.AddInstruction(
|
|
SpirvOp.FMul,
|
|
_floatType,
|
|
coordinate,
|
|
widthFloat),
|
|
Float(0.5f));
|
|
var left = Ext(8, _floatType, centered);
|
|
|
|
uint component = 0;
|
|
if (resource.ConversionFormat == GuestImageFormat.Invalid)
|
|
{
|
|
while (component < 3 &&
|
|
(image.Dmask & (1u << (int)component)) == 0)
|
|
{
|
|
component++;
|
|
}
|
|
}
|
|
|
|
var values = new uint[2];
|
|
for (var index = 0; index < values.Length; index++)
|
|
{
|
|
var sampleCoordinate = _module.AddInstruction(
|
|
SpirvOp.FDiv,
|
|
_floatType,
|
|
_module.AddInstruction(
|
|
SpirvOp.FAdd,
|
|
_floatType,
|
|
left,
|
|
Float(index == 0 ? 0.5f : 1.5f)),
|
|
widthFloat);
|
|
var texel = _module.AddInstruction(
|
|
SpirvOp.ImageSampleExplicitLod,
|
|
resource.VectorType,
|
|
sampledImage,
|
|
sampleCoordinate,
|
|
2u,
|
|
Float(0));
|
|
values[index] = _module.AddInstruction(
|
|
SpirvOp.CompositeExtract,
|
|
resource.ComponentType,
|
|
texel,
|
|
component);
|
|
}
|
|
|
|
return _module.AddInstruction(
|
|
SpirvOp.CompositeConstruct,
|
|
resource.VectorType,
|
|
values[0],
|
|
values[1],
|
|
values[1],
|
|
values[0]);
|
|
}
|
|
|
|
private static bool TryGetPackedImageConversion(
|
|
SpirvImageResource resource,
|
|
out int componentCount,
|
|
out ReadOnlySpan<uint> componentBits,
|
|
out ReadOnlySpan<uint> componentOffsets)
|
|
{
|
|
if (resource.ConversionFormat == GuestImageFormat.Format11x2x10Uint)
|
|
{
|
|
componentCount = 3;
|
|
componentBits = [11u, 11u, 10u];
|
|
componentOffsets = [0u, 11u, 22u];
|
|
return true;
|
|
}
|
|
|
|
componentCount = 0;
|
|
componentBits = default;
|
|
componentOffsets = default;
|
|
return false;
|
|
}
|
|
|
|
private uint UnpackImageTexel(
|
|
SpirvImageResource resource,
|
|
uint texel)
|
|
{
|
|
if (resource.ConversionFormat is GuestImageFormat.Format8Uscaled or GuestImageFormat.Format8x2Uscaled)
|
|
{
|
|
var scaled = _module.AddInstruction(SpirvOp.FMul, resource.VectorType, texel,
|
|
_module.AddInstruction(SpirvOp.CompositeConstruct, resource.VectorType,
|
|
Float(255), Float(255), Float(1), Float(1)));
|
|
var channels = new uint[4];
|
|
for (var component = 0; component < channels.Length; component++)
|
|
{
|
|
var selector = (resource.ShaderSwizzle >> (component * 3)) & 7u;
|
|
channels[component] = selector switch
|
|
{
|
|
1u => Float(1),
|
|
>= 4u => _module.AddInstruction(SpirvOp.CompositeExtract, _floatType, scaled, selector - 4u),
|
|
_ => Float(0),
|
|
};
|
|
}
|
|
return _module.AddInstruction(SpirvOp.CompositeConstruct, resource.VectorType, channels);
|
|
}
|
|
|
|
if (!TryGetPackedImageConversion(
|
|
resource,
|
|
out var componentCount,
|
|
out var componentBits,
|
|
out var componentOffsets))
|
|
{
|
|
return texel;
|
|
}
|
|
|
|
var packed = _module.AddInstruction(
|
|
SpirvOp.CompositeExtract,
|
|
_uintType,
|
|
texel,
|
|
0u);
|
|
var components = new uint[4];
|
|
for (var component = 0; component < componentCount; component++)
|
|
{
|
|
components[component] = _module.AddInstruction(
|
|
SpirvOp.BitFieldUExtract,
|
|
_uintType,
|
|
packed,
|
|
UInt(componentOffsets[component]),
|
|
UInt(componentBits[component]));
|
|
}
|
|
|
|
for (var component = componentCount; component < components.Length; component++)
|
|
{
|
|
components[component] = components[component % componentCount];
|
|
}
|
|
|
|
var selected = new uint[4];
|
|
for (var component = 0; component < selected.Length; component++)
|
|
{
|
|
var selector = (resource.ShaderSwizzle >> (component * 3)) & 7u;
|
|
selected[component] = selector switch
|
|
{
|
|
1u => UInt(1),
|
|
>= 4u => components[(selector - 4u) % (uint)componentCount],
|
|
_ => UInt(0),
|
|
};
|
|
}
|
|
|
|
return _module.AddInstruction(
|
|
SpirvOp.CompositeConstruct,
|
|
resource.VectorType,
|
|
selected);
|
|
}
|
|
|
|
private uint UnpackImageGather(
|
|
SpirvImageResource resource,
|
|
uint dmask,
|
|
uint gathered)
|
|
{
|
|
if (!TryGetPackedImageConversion(
|
|
resource,
|
|
out var componentCount,
|
|
out var componentBits,
|
|
out var componentOffsets))
|
|
{
|
|
return gathered;
|
|
}
|
|
|
|
uint component = 0;
|
|
while (component < 3 && (dmask & (1u << (int)component)) == 0)
|
|
{
|
|
component++;
|
|
}
|
|
|
|
var selector = (resource.ShaderSwizzle >> ((int)component * 3)) & 7u;
|
|
if (selector < 4u)
|
|
{
|
|
var value = selector == 1u ? UInt(1) : UInt(0);
|
|
return _module.AddInstruction(
|
|
SpirvOp.CompositeConstruct,
|
|
resource.VectorType,
|
|
value,
|
|
value,
|
|
value,
|
|
value);
|
|
}
|
|
|
|
var physical = (selector - 4u) % (uint)componentCount;
|
|
var values = new uint[4];
|
|
for (var lane = 0; lane < values.Length; lane++)
|
|
{
|
|
var packed = _module.AddInstruction(
|
|
SpirvOp.CompositeExtract,
|
|
_uintType,
|
|
gathered,
|
|
(uint)lane);
|
|
values[lane] = _module.AddInstruction(
|
|
SpirvOp.BitFieldUExtract,
|
|
_uintType,
|
|
packed,
|
|
UInt(componentOffsets[(int)physical]),
|
|
UInt(componentBits[(int)physical]));
|
|
}
|
|
|
|
return _module.AddInstruction(
|
|
SpirvOp.CompositeConstruct,
|
|
resource.VectorType,
|
|
values);
|
|
}
|
|
|
|
private uint PackImageTexel(
|
|
SpirvImageResource resource,
|
|
uint texel)
|
|
{
|
|
if (!TryGetPackedImageConversion(
|
|
resource,
|
|
out var componentCount,
|
|
out var componentBits,
|
|
out var componentOffsets))
|
|
{
|
|
return texel;
|
|
}
|
|
|
|
var packed = UInt(0);
|
|
for (var component = 0; component < componentCount; component++)
|
|
{
|
|
var value = _module.AddInstruction(
|
|
SpirvOp.CompositeExtract,
|
|
_uintType,
|
|
texel,
|
|
(uint)component);
|
|
var maximum = (1u << (int)componentBits[component]) - 1u;
|
|
var within = _module.AddInstruction(
|
|
SpirvOp.ULessThan,
|
|
_boolType,
|
|
value,
|
|
UInt(maximum));
|
|
var clamped = _module.AddInstruction(
|
|
SpirvOp.Select,
|
|
_uintType,
|
|
within,
|
|
value,
|
|
UInt(maximum));
|
|
var shifted = componentOffsets[component] == 0
|
|
? clamped
|
|
: ShiftLeftLogical(clamped, UInt(componentOffsets[component]));
|
|
packed = BitwiseOr(packed, shifted);
|
|
}
|
|
|
|
return _module.AddInstruction(
|
|
SpirvOp.CompositeConstruct,
|
|
resource.VectorType,
|
|
packed,
|
|
UInt(0),
|
|
UInt(0),
|
|
UInt(0));
|
|
}
|
|
|
|
private uint BuildHorizontalGatherOffsets()
|
|
{
|
|
var vec2Int = _module.TypeVector(_intType, 2);
|
|
var offsets = new uint[4];
|
|
for (var index = 0; index < offsets.Length; index++)
|
|
{
|
|
offsets[index] = _module.ConstantComposite(
|
|
vec2Int,
|
|
_module.Constant(_intType, unchecked((uint)(index - 1))),
|
|
_module.Constant(_intType, 0));
|
|
}
|
|
|
|
return _module.ConstantComposite(
|
|
_module.TypeArray(vec2Int, 4),
|
|
offsets);
|
|
}
|
|
|
|
private static uint ImageSpatialComponentCount(
|
|
SpirvImageResource resource) =>
|
|
ImageSpatialComponentCountOf(resource.Dimension);
|
|
|
|
private static uint ImageCoordinateComponentCount(
|
|
SpirvImageResource resource) =>
|
|
ImageSpatialComponentCount(resource) + (resource.Arrayed ? 1u : 0u);
|
|
|
|
private uint ImageIntegerCoordinateType(uint componentCount) =>
|
|
componentCount == 1
|
|
? _intType
|
|
: _module.TypeVector(_intType, componentCount);
|
|
|
|
private uint IntegerTypeForComponents(uint componentCount) =>
|
|
componentCount == 1 ? _intType : _module.TypeVector(_intType, componentCount);
|
|
|
|
private uint FloatTypeForComponents(uint componentCount) =>
|
|
componentCount == 1 ? _floatType : _module.TypeVector(_floatType, componentCount);
|
|
|
|
private uint BuildFloatCoordinates(
|
|
Gen5ImageControl image,
|
|
int start,
|
|
uint componentCount,
|
|
SpirvImageResource resource)
|
|
{
|
|
var components = new uint[checked((int)componentCount)];
|
|
for (var component = 0; component < components.Length; component++)
|
|
{
|
|
components[component] = LoadImageFloatAddress(
|
|
image,
|
|
start + component);
|
|
}
|
|
|
|
if (resource.Cube && components.Length >= 2)
|
|
{
|
|
components[0] = _module.AddInstruction(SpirvOp.FSub, _floatType, components[0], Float(1));
|
|
components[1] = _module.AddInstruction(SpirvOp.FSub, _floatType, components[1], Float(1));
|
|
if (components.Length >= 3)
|
|
{
|
|
var guestLayer = _module.AddInstruction(SpirvOp.ConvertFToU, _uintType, components[2]);
|
|
var padding = ShiftLeftLogical(ShiftRightLogical(guestLayer, UInt(3)), UInt(1));
|
|
var hostLayer = _module.AddInstruction(SpirvOp.ISub, _uintType, guestLayer, padding);
|
|
components[2] = _module.AddInstruction(SpirvOp.ConvertUToF, _floatType, hostLayer);
|
|
}
|
|
}
|
|
|
|
if (componentCount == 1)
|
|
{
|
|
return components[0];
|
|
}
|
|
|
|
return _module.AddInstruction(
|
|
SpirvOp.CompositeConstruct,
|
|
_module.TypeVector(_floatType, componentCount),
|
|
components);
|
|
}
|
|
|
|
private static int ImageAddressRegister(
|
|
Gen5ImageControl image,
|
|
int component) => image.A16 ? component / 2 : component;
|
|
|
|
private static int ImageFullAddressSlots(Gen5ImageControl image) =>
|
|
image.A16 ? 2 : 1;
|
|
|
|
private static int AlignFullImageAddress(
|
|
Gen5ImageControl image,
|
|
int component) => image.A16 ? (component + 1) & ~1 : component;
|
|
|
|
private uint LoadImageFloatAddress(Gen5ImageControl image, int component)
|
|
{
|
|
var raw = LoadV(image.GetAddressRegister(
|
|
ImageAddressRegister(image, component)));
|
|
if (!image.A16)
|
|
{
|
|
return Bitcast(_floatType, raw);
|
|
}
|
|
|
|
var unpacked = Ext(62, _vec2Type, raw);
|
|
return _module.AddInstruction(
|
|
SpirvOp.CompositeExtract,
|
|
_floatType,
|
|
unpacked,
|
|
(uint)(component & 1));
|
|
}
|
|
|
|
private uint LoadImageIntegerAddress(Gen5ImageControl image, int component)
|
|
{
|
|
var raw = LoadV(image.GetAddressRegister(
|
|
ImageAddressRegister(image, component)));
|
|
if (!image.A16)
|
|
{
|
|
return raw;
|
|
}
|
|
|
|
return BitwiseAnd(
|
|
ShiftRightLogical(raw, UInt((uint)((component & 1) * 16))),
|
|
UInt(0xFFFF));
|
|
}
|
|
|
|
private uint LoadImageStoreComponent(
|
|
Gen5ImageControl image,
|
|
SpirvImageResource resource,
|
|
uint component)
|
|
{
|
|
if (!image.D16)
|
|
{
|
|
return LoadV(image.VectorData + component);
|
|
}
|
|
|
|
var packed = LoadV(image.VectorData + component / 2);
|
|
if (resource.ComponentKind == ImageComponentKind.Float)
|
|
{
|
|
var unpacked = Ext(62, _vec2Type, packed);
|
|
return Bitcast(
|
|
_uintType,
|
|
_module.AddInstruction(
|
|
SpirvOp.CompositeExtract,
|
|
_floatType,
|
|
unpacked,
|
|
component & 1));
|
|
}
|
|
|
|
var shifted = ShiftRightLogical(packed, UInt((component & 1) * 16));
|
|
var low = BitwiseAnd(shifted, UInt(0xFFFF));
|
|
if (resource.ComponentKind != ImageComponentKind.Sint)
|
|
{
|
|
return low;
|
|
}
|
|
|
|
return Bitcast(
|
|
_uintType,
|
|
_module.AddInstruction(
|
|
SpirvOp.BitFieldSExtract,
|
|
_intType,
|
|
Bitcast(_intType, low),
|
|
UInt(0),
|
|
UInt(16)));
|
|
}
|
|
|
|
private uint PackImageD16(
|
|
SpirvImageResource resource,
|
|
uint low,
|
|
uint high)
|
|
{
|
|
if (resource.ComponentKind == ImageComponentKind.Float)
|
|
{
|
|
var pair = _module.AddInstruction(
|
|
SpirvOp.CompositeConstruct,
|
|
_vec2Type,
|
|
Bitcast(_floatType, low),
|
|
Bitcast(_floatType, high));
|
|
return Ext(58, _uintType, pair);
|
|
}
|
|
|
|
return BitwiseOr(
|
|
BitwiseAnd(low, UInt(0xFFFF)),
|
|
ShiftLeftLogical(BitwiseAnd(high, UInt(0xFFFF)), UInt(16)));
|
|
}
|
|
|
|
private uint BuildIntegerCoordinates(
|
|
Gen5ImageControl image,
|
|
int start,
|
|
uint componentCount)
|
|
{
|
|
var components = new uint[checked((int)componentCount)];
|
|
for (var component = 0; component < components.Length; component++)
|
|
{
|
|
components[component] = Bitcast(
|
|
_intType,
|
|
LoadImageIntegerAddress(image, start + component));
|
|
}
|
|
|
|
if (componentCount == 1)
|
|
{
|
|
return components[0];
|
|
}
|
|
|
|
return _module.AddInstruction(
|
|
SpirvOp.CompositeConstruct,
|
|
ImageIntegerCoordinateType(componentCount),
|
|
components);
|
|
}
|
|
|
|
private uint BuildImageOffset(
|
|
Gen5ImageControl image,
|
|
int component,
|
|
uint componentCount)
|
|
{
|
|
var packed = Bitcast(
|
|
_intType,
|
|
LoadV(image.GetAddressRegister(
|
|
ImageAddressRegister(image, component))));
|
|
var components = new uint[checked((int)componentCount)];
|
|
for (var index = 0; index < components.Length; index++)
|
|
{
|
|
components[index] = _module.AddInstruction(
|
|
SpirvOp.BitFieldSExtract,
|
|
_intType,
|
|
packed,
|
|
UInt((uint)(index * 8)),
|
|
UInt(6));
|
|
}
|
|
|
|
if (componentCount == 1)
|
|
{
|
|
return components[0];
|
|
}
|
|
|
|
return _module.AddInstruction(
|
|
SpirvOp.CompositeConstruct,
|
|
_module.TypeVector(_intType, componentCount),
|
|
components);
|
|
}
|
|
|
|
private uint ApplyDynamicSampleOffset(
|
|
SpirvImageResource resource,
|
|
uint sampledImage,
|
|
uint coordinates,
|
|
uint texelOffset,
|
|
uint lod)
|
|
{
|
|
var spatialComponentCount = ImageSpatialComponentCount(resource);
|
|
var coordinateComponentCount =
|
|
ImageCoordinateComponentCount(resource);
|
|
var spatialIntegerType =
|
|
spatialComponentCount == 1
|
|
? _intType
|
|
: _module.TypeVector(_intType, spatialComponentCount);
|
|
var spatialFloatType =
|
|
spatialComponentCount == 1
|
|
? _floatType
|
|
: _module.TypeVector(_floatType, spatialComponentCount);
|
|
var queryComponentCount = resource.Arrayed
|
|
? coordinateComponentCount
|
|
: spatialComponentCount;
|
|
var queryIntegerType =
|
|
queryComponentCount == 1
|
|
? _intType
|
|
: _module.TypeVector(_intType, queryComponentCount);
|
|
var image = _module.AddInstruction(
|
|
SpirvOp.Image,
|
|
resource.ImageType,
|
|
sampledImage);
|
|
var signedLod = _module.AddInstruction(
|
|
SpirvOp.ConvertFToS,
|
|
_intType,
|
|
lod);
|
|
var lodIsNegative = _module.AddInstruction(
|
|
SpirvOp.SLessThan,
|
|
_boolType,
|
|
signedLod,
|
|
_module.Constant(_intType, 0));
|
|
var clampedLod = _module.AddInstruction(
|
|
SpirvOp.Select,
|
|
_intType,
|
|
lodIsNegative,
|
|
_module.Constant(_intType, 0),
|
|
signedLod);
|
|
var size = _module.AddInstruction(
|
|
SpirvOp.ImageQuerySizeLod,
|
|
queryIntegerType,
|
|
image,
|
|
clampedLod);
|
|
if (resource.Arrayed)
|
|
{
|
|
if (spatialComponentCount == 1)
|
|
{
|
|
size = _module.AddInstruction(
|
|
SpirvOp.CompositeExtract,
|
|
_intType,
|
|
size,
|
|
0u);
|
|
}
|
|
else
|
|
{
|
|
var spatialSizeComponents =
|
|
new uint[checked((int)spatialComponentCount)];
|
|
for (uint component = 0;
|
|
component < spatialComponentCount;
|
|
component++)
|
|
{
|
|
spatialSizeComponents[component] = component;
|
|
}
|
|
|
|
size = _module.AddInstruction(
|
|
SpirvOp.VectorShuffle,
|
|
spatialIntegerType,
|
|
[size, size, .. spatialSizeComponents]);
|
|
}
|
|
}
|
|
|
|
var sizeFloat = _module.AddInstruction(
|
|
SpirvOp.ConvertSToF,
|
|
spatialFloatType,
|
|
size);
|
|
var offsetFloat = _module.AddInstruction(
|
|
SpirvOp.ConvertSToF,
|
|
spatialFloatType,
|
|
texelOffset);
|
|
var normalizedOffset = _module.AddInstruction(
|
|
SpirvOp.FDiv,
|
|
spatialFloatType,
|
|
offsetFloat,
|
|
sizeFloat);
|
|
if (!resource.Arrayed)
|
|
{
|
|
return _module.AddInstruction(
|
|
SpirvOp.FAdd,
|
|
spatialFloatType,
|
|
coordinates,
|
|
normalizedOffset);
|
|
}
|
|
|
|
var arrayOffsetComponents =
|
|
new uint[checked((int)coordinateComponentCount)];
|
|
for (uint component = 0;
|
|
component < spatialComponentCount;
|
|
component++)
|
|
{
|
|
arrayOffsetComponents[component] =
|
|
spatialComponentCount == 1
|
|
? normalizedOffset
|
|
: _module.AddInstruction(
|
|
SpirvOp.CompositeExtract,
|
|
_floatType,
|
|
normalizedOffset,
|
|
component);
|
|
}
|
|
arrayOffsetComponents[coordinateComponentCount - 1] = Float(0);
|
|
var arrayOffset = _module.AddInstruction(
|
|
SpirvOp.CompositeConstruct,
|
|
_module.TypeVector(_floatType, coordinateComponentCount),
|
|
arrayOffsetComponents);
|
|
return _module.AddInstruction(
|
|
SpirvOp.FAdd,
|
|
_module.TypeVector(_floatType, coordinateComponentCount),
|
|
coordinates,
|
|
arrayOffset);
|
|
}
|
|
|
|
private bool TryEmitExport(
|
|
Gen5ShaderInstruction instruction,
|
|
Gen5ExportControl export,
|
|
out string error)
|
|
{
|
|
error = string.Empty;
|
|
if (instruction.Sources.Count < 4)
|
|
{
|
|
error = "missing export sources";
|
|
return false;
|
|
}
|
|
|
|
if (_stage == Gen5SpirvStage.Pixel)
|
|
{
|
|
// RDNA2 EXP.VM communicates the current EXEC mask even for a
|
|
// NULL export or an MRT not bound by this host pass. Record it
|
|
// before resolving the data target; the final VM export wins.
|
|
if (export.ValidMask && _pixelValidMaskActive != 0)
|
|
{
|
|
Store(_pixelValidMaskActive, Load(_boolType, _exec));
|
|
}
|
|
|
|
if (!_pixelOutputs.TryGetValue(export.Target, out var output))
|
|
{
|
|
return true;
|
|
}
|
|
|
|
Store(_reachedPixelExport, _module.ConstantBool(true));
|
|
|
|
var values = new uint[4];
|
|
for (var component = 0; component < 4; component++)
|
|
{
|
|
var enabled = (export.EnableMask & (1u << component)) != 0;
|
|
if (!enabled)
|
|
{
|
|
var outputComponent = (uint)component;
|
|
if (!output.ComponentMapping.IsIdentity)
|
|
{
|
|
for (var physicalComponent = 0u; physicalComponent < 4; physicalComponent++)
|
|
{
|
|
if (output.ComponentMapping.Map(physicalComponent) == component)
|
|
{
|
|
outputComponent = physicalComponent;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
values[component] = _module.AddInstruction(
|
|
SpirvOp.CompositeExtract,
|
|
output.Kind switch
|
|
{
|
|
Gen5PixelOutputKind.Uint => _uintType,
|
|
Gen5PixelOutputKind.Sint => _intType,
|
|
_ => _floatType,
|
|
},
|
|
Load(output.Type, output.Variable),
|
|
outputComponent);
|
|
continue;
|
|
}
|
|
|
|
if (export.Compressed)
|
|
{
|
|
var value = LoadCompressedExportComponent(
|
|
instruction,
|
|
component);
|
|
values[component] = output.Kind switch
|
|
{
|
|
Gen5PixelOutputKind.Uint => _module.AddInstruction(
|
|
SpirvOp.ConvertFToU,
|
|
_uintType,
|
|
value),
|
|
Gen5PixelOutputKind.Sint => _module.AddInstruction(
|
|
SpirvOp.ConvertFToS,
|
|
_intType,
|
|
value),
|
|
_ => value,
|
|
};
|
|
continue;
|
|
}
|
|
|
|
var raw = LoadV(instruction.Sources[component].Value);
|
|
values[component] = output.Kind switch
|
|
{
|
|
Gen5PixelOutputKind.Uint => raw,
|
|
Gen5PixelOutputKind.Sint => Bitcast(_intType, raw),
|
|
_ => Bitcast(_floatType, raw),
|
|
};
|
|
}
|
|
|
|
var vector = _module.AddInstruction(
|
|
SpirvOp.CompositeConstruct,
|
|
output.Type,
|
|
values);
|
|
if (output.Kind == Gen5PixelOutputKind.Float &&
|
|
PixelExportVgprAddressMatches() &&
|
|
uint.TryParse(
|
|
Environment.GetEnvironmentVariable(
|
|
"SHARPEMU_FORCE_PIXEL_EXPORT_VGPR_BASE"),
|
|
out var debugVgprBase))
|
|
{
|
|
var registerBase = debugVgprBase + export.Target * 4;
|
|
vector = _module.AddInstruction(
|
|
SpirvOp.CompositeConstruct,
|
|
output.Type,
|
|
Bitcast(_floatType, LoadV(registerBase)),
|
|
Bitcast(_floatType, LoadV(registerBase + 1)),
|
|
Bitcast(_floatType, LoadV(registerBase + 2)),
|
|
Bitcast(_floatType, LoadV(registerBase + 3)));
|
|
}
|
|
if (output.Kind == Gen5PixelOutputKind.Float &&
|
|
PixelExportVgprAddressMatches() &&
|
|
uint.TryParse(
|
|
Environment.GetEnvironmentVariable(
|
|
"SHARPEMU_FORCE_PIXEL_EXPORT_PACK_VGPR_BASE"),
|
|
out var debugPackVgprBase))
|
|
{
|
|
var registerBase = debugPackVgprBase + export.Target * 4;
|
|
var lowPair = _module.AddInstruction(
|
|
SpirvOp.CompositeConstruct,
|
|
_vec2Type,
|
|
TruncateFloat32ForPack(Bitcast(_floatType, LoadV(registerBase))),
|
|
TruncateFloat32ForPack(Bitcast(_floatType, LoadV(registerBase + 1))));
|
|
var highPair = _module.AddInstruction(
|
|
SpirvOp.CompositeConstruct,
|
|
_vec2Type,
|
|
TruncateFloat32ForPack(Bitcast(_floatType, LoadV(registerBase + 2))),
|
|
TruncateFloat32ForPack(Bitcast(_floatType, LoadV(registerBase + 3))));
|
|
var unpackedLow = Ext(62, _vec2Type, Ext(58, _uintType, lowPair));
|
|
var unpackedHigh = Ext(62, _vec2Type, Ext(58, _uintType, highPair));
|
|
vector = _module.AddInstruction(
|
|
SpirvOp.CompositeConstruct,
|
|
output.Type,
|
|
_module.AddInstruction(
|
|
SpirvOp.CompositeExtract,
|
|
_floatType,
|
|
unpackedLow,
|
|
0),
|
|
_module.AddInstruction(
|
|
SpirvOp.CompositeExtract,
|
|
_floatType,
|
|
unpackedLow,
|
|
1),
|
|
_module.AddInstruction(
|
|
SpirvOp.CompositeExtract,
|
|
_floatType,
|
|
unpackedHigh,
|
|
0),
|
|
_module.AddInstruction(
|
|
SpirvOp.CompositeExtract,
|
|
_floatType,
|
|
unpackedHigh,
|
|
1));
|
|
}
|
|
if (_forcePixelMagenta && PixelExportDebugAddressMatches())
|
|
{
|
|
vector = output.Kind switch
|
|
{
|
|
Gen5PixelOutputKind.Float =>
|
|
_module.AddInstruction(
|
|
SpirvOp.CompositeConstruct,
|
|
output.Type,
|
|
Float(1f),
|
|
Float(0f),
|
|
Float(1f),
|
|
Float(1f)),
|
|
Gen5PixelOutputKind.Sint =>
|
|
_module.AddInstruction(
|
|
SpirvOp.CompositeConstruct,
|
|
output.Type,
|
|
Bitcast(_intType, UInt(1)),
|
|
Bitcast(_intType, UInt(0)),
|
|
Bitcast(_intType, UInt(1)),
|
|
Bitcast(_intType, UInt(1))),
|
|
_ =>
|
|
_module.AddInstruction(
|
|
SpirvOp.CompositeConstruct,
|
|
output.Type,
|
|
UInt(1),
|
|
UInt(0),
|
|
UInt(1),
|
|
UInt(1)),
|
|
};
|
|
}
|
|
if (output.ComponentMapping != Gen5ColorComponentMapping.Identity)
|
|
{
|
|
vector = _module.AddInstruction(
|
|
SpirvOp.VectorShuffle,
|
|
output.Type,
|
|
vector,
|
|
vector,
|
|
output.ComponentMapping.Map(0),
|
|
output.ComponentMapping.Map(1),
|
|
output.ComponentMapping.Map(2),
|
|
output.ComponentMapping.Map(3));
|
|
}
|
|
if (Environment.GetEnvironmentVariable(
|
|
"SHARPEMU_FORCE_TITLE_EXPORT_EXEC") == "1" &&
|
|
_request.Program.Address == 0x0000000500781200ul)
|
|
{
|
|
Store(_exec, _module.ConstantBool(true));
|
|
StoreS64(
|
|
126,
|
|
_module.Constant64(_ulongType, 1));
|
|
}
|
|
vector = _module.AddInstruction(
|
|
SpirvOp.Select,
|
|
output.Type,
|
|
Load(_boolType, _exec),
|
|
vector,
|
|
Load(output.Type, output.Variable));
|
|
Store(output.Variable, vector);
|
|
return true;
|
|
}
|
|
|
|
if (_stage != Gen5SpirvStage.Vertex)
|
|
{
|
|
return true;
|
|
}
|
|
|
|
uint outputVariable;
|
|
if (export.Target is >= 12 and < 16)
|
|
{
|
|
if (export.Target != 12)
|
|
{
|
|
EmitAuxPositionExport(instruction, export);
|
|
return true;
|
|
}
|
|
|
|
outputVariable = _positionOutput;
|
|
}
|
|
else if (export.Target is >= 32 and < 64 &&
|
|
_vertexOutputs.TryGetValue(export.Target - 32, out var parameter))
|
|
{
|
|
outputVariable = parameter;
|
|
}
|
|
else
|
|
{
|
|
return true;
|
|
}
|
|
|
|
var components = new uint[4];
|
|
for (var component = 0; component < 4; component++)
|
|
{
|
|
components[component] = (export.EnableMask & (1u << component)) != 0
|
|
? export.Compressed
|
|
? LoadCompressedExportComponent(instruction, component)
|
|
: Bitcast(
|
|
_floatType,
|
|
LoadV(instruction.Sources[component].Value))
|
|
: Float(component == 3 ? 1f : 0f);
|
|
}
|
|
|
|
var outputValue = _module.AddInstruction(
|
|
SpirvOp.CompositeConstruct,
|
|
_vec4Type,
|
|
components);
|
|
if (export.Target == 12 && _request.ClipSpace.Enabled)
|
|
{
|
|
outputValue = ConvertPositionToClipSpace(outputValue);
|
|
}
|
|
if (_request.Program.Address == 0x0000000500780000ul &&
|
|
export.Target is >= 32 and < 36 &&
|
|
Environment.GetEnvironmentVariable(
|
|
"SHARPEMU_FORCE_TITLE_VERTEX_OUTPUTS_ONE") == "1")
|
|
{
|
|
outputValue = _module.AddInstruction(
|
|
SpirvOp.CompositeConstruct,
|
|
_vec4Type,
|
|
Float(1f),
|
|
Float(1f),
|
|
Float(1f),
|
|
Float(1f));
|
|
}
|
|
outputValue = _module.AddInstruction(
|
|
SpirvOp.Select,
|
|
_vec4Type,
|
|
Load(_boolType, _exec),
|
|
outputValue,
|
|
Load(_vec4Type, outputVariable));
|
|
Store(outputVariable, outputValue);
|
|
if (export.Target == 12 && _invalidPositionClipDistance != uint.MaxValue)
|
|
{
|
|
var equal = _module.AddInstruction(
|
|
SpirvOp.FOrdEqual,
|
|
_module.TypeVector(_boolType, 4),
|
|
outputValue,
|
|
_module.ConstantNull(_vec4Type));
|
|
var invalid = _module.AddInstruction(SpirvOp.All, _boolType, equal);
|
|
// A zero position has an undefined perspective divide. Collapse its primitive
|
|
// to the remaining edge, as the guest's clipping-error cull does.
|
|
var distance = _module.AddInstruction(
|
|
SpirvOp.Select, _floatType, invalid, Float(-1f), Float(0f));
|
|
StoreDistanceConditional(_clipDistanceOutput, _invalidPositionClipDistance, distance);
|
|
}
|
|
return true;
|
|
}
|
|
|
|
private readonly record struct PositionExportComponent(
|
|
uint ClipDistance,
|
|
uint CullDistance,
|
|
bool PointSize,
|
|
bool Layer,
|
|
bool Viewport);
|
|
|
|
private static PositionExportComponent DecodePositionExportComponent(
|
|
uint control,
|
|
uint positionIndex,
|
|
uint component)
|
|
{
|
|
if (positionIndex == 0 || component >= 4)
|
|
{
|
|
return new(uint.MaxValue, uint.MaxValue, false, false, false);
|
|
}
|
|
|
|
var slot = positionIndex - 1;
|
|
var vector = 3u;
|
|
for (var index = 0u; index < 3; index++)
|
|
{
|
|
if ((control & (1u << (int)(21 + index))) == 0)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (slot == 0)
|
|
{
|
|
vector = index;
|
|
break;
|
|
}
|
|
|
|
slot--;
|
|
}
|
|
|
|
if (vector == 3)
|
|
{
|
|
return new(uint.MaxValue, uint.MaxValue, false, false, false);
|
|
}
|
|
|
|
if (vector == 0)
|
|
{
|
|
return new(
|
|
uint.MaxValue,
|
|
uint.MaxValue,
|
|
component == 0 && (control & (1u << 16)) != 0,
|
|
component == 2 && (control & (1u << 18)) != 0,
|
|
component == 2 && (control & (1u << 19)) != 0);
|
|
}
|
|
|
|
var scalar = (vector - 1) * 4 + component;
|
|
var lower = scalar == 0 ? 0u : (1u << (int)scalar) - 1u;
|
|
var clip = control & 0xffu;
|
|
var cull = (control >> 8) & 0xffu;
|
|
return new(
|
|
(clip & (1u << (int)scalar)) != 0
|
|
? (uint)System.Numerics.BitOperations.PopCount(clip & lower)
|
|
: uint.MaxValue,
|
|
(cull & (1u << (int)scalar)) != 0
|
|
? (uint)System.Numerics.BitOperations.PopCount(cull & lower)
|
|
: uint.MaxValue,
|
|
false,
|
|
false,
|
|
false);
|
|
}
|
|
|
|
private void DeclareAuxPositionOutputs()
|
|
{
|
|
var needPointSize = false;
|
|
var needLayer = false;
|
|
var needViewport = false;
|
|
var clipCount = 0u;
|
|
var cullCount = 0u;
|
|
|
|
foreach (var export in _request.Program.Instructions
|
|
.Select(static instruction => instruction.Control)
|
|
.OfType<Gen5ExportControl>()
|
|
.Where(static export => export.Target is >= 13 and < 16))
|
|
{
|
|
var positionIndex = export.Target - 12;
|
|
for (var component = 0u; component < 4; component++)
|
|
{
|
|
if ((export.EnableMask & (1u << (int)component)) == 0)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
var output = DecodePositionExportComponent(
|
|
_request.PositionExportControl,
|
|
positionIndex,
|
|
component);
|
|
needPointSize |= output.PointSize;
|
|
needLayer |= output.Layer;
|
|
needViewport |= output.Viewport;
|
|
if (output.ClipDistance != uint.MaxValue)
|
|
{
|
|
clipCount = Math.Max(clipCount, output.ClipDistance + 1);
|
|
}
|
|
if (output.CullDistance != uint.MaxValue)
|
|
{
|
|
cullCount = Math.Max(cullCount, output.CullDistance + 1);
|
|
}
|
|
}
|
|
}
|
|
|
|
if (_request.SupportsClipDistance && clipCount + cullCount < 8 &&
|
|
_request.Program.Instructions.Any(static instruction =>
|
|
instruction.Control is Gen5ExportControl { Target: 12, EnableMask: not 0 }))
|
|
{
|
|
// Use a separate plane so auxiliary position exports keep their own distances.
|
|
_invalidPositionClipDistance = clipCount++;
|
|
}
|
|
|
|
if (needPointSize)
|
|
{
|
|
_pointSizeOutput = DeclareBuiltInOutput(
|
|
_floatType,
|
|
SpirvBuiltIn.PointSize,
|
|
"gl_PointSize");
|
|
}
|
|
if (needLayer || needViewport)
|
|
{
|
|
_module.AddExtension("SPV_EXT_shader_viewport_index_layer");
|
|
_module.AddCapability(SpirvCapability.ShaderViewportIndexLayerExt);
|
|
}
|
|
if (needLayer)
|
|
{
|
|
_layerOutput = DeclareBuiltInOutput(
|
|
_uintType,
|
|
SpirvBuiltIn.Layer,
|
|
"gl_Layer");
|
|
}
|
|
if (needViewport)
|
|
{
|
|
_viewportIndexOutput = DeclareBuiltInOutput(
|
|
_uintType,
|
|
SpirvBuiltIn.ViewportIndex,
|
|
"gl_ViewportIndex");
|
|
}
|
|
if (clipCount != 0)
|
|
{
|
|
_module.AddCapability(SpirvCapability.ClipDistance);
|
|
_clipDistanceCount = clipCount;
|
|
_clipDistanceOutput = DeclareBuiltInOutput(
|
|
_module.TypeArray(_floatType, clipCount),
|
|
SpirvBuiltIn.ClipDistance,
|
|
"gl_ClipDistance");
|
|
}
|
|
if (cullCount != 0)
|
|
{
|
|
_module.AddCapability(SpirvCapability.CullDistance);
|
|
_cullDistanceCount = cullCount;
|
|
_cullDistanceOutput = DeclareBuiltInOutput(
|
|
_module.TypeArray(_floatType, cullCount),
|
|
SpirvBuiltIn.CullDistance,
|
|
"gl_CullDistance");
|
|
}
|
|
}
|
|
|
|
private uint DeclareBuiltInOutput(
|
|
uint type,
|
|
SpirvBuiltIn builtIn,
|
|
string name)
|
|
{
|
|
var pointer = _module.TypePointer(SpirvStorageClass.Output, type);
|
|
var variable = _module.AddGlobalVariable(pointer, SpirvStorageClass.Output);
|
|
_module.AddName(variable, name);
|
|
_module.AddDecoration(
|
|
variable,
|
|
SpirvDecoration.BuiltIn,
|
|
(uint)builtIn);
|
|
_interfaces.Add(variable);
|
|
return variable;
|
|
}
|
|
|
|
private void InitializeDistanceOutput(uint variable, uint count)
|
|
{
|
|
if (variable == 0)
|
|
{
|
|
return;
|
|
}
|
|
|
|
var pointerType = _module.TypePointer(SpirvStorageClass.Output, _floatType);
|
|
for (var index = 0u; index < count; index++)
|
|
{
|
|
var pointer = _module.AddInstruction(
|
|
SpirvOp.AccessChain,
|
|
pointerType,
|
|
variable,
|
|
UInt(index));
|
|
Store(pointer, Float(0f));
|
|
}
|
|
}
|
|
|
|
private void EmitAuxPositionExport(
|
|
Gen5ShaderInstruction instruction,
|
|
Gen5ExportControl export)
|
|
{
|
|
var positionIndex = export.Target - 12;
|
|
for (var component = 0u; component < 4; component++)
|
|
{
|
|
if ((export.EnableMask & (1u << (int)component)) == 0)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
var output = DecodePositionExportComponent(
|
|
_request.PositionExportControl,
|
|
positionIndex,
|
|
component);
|
|
if (!output.PointSize &&
|
|
!output.Layer &&
|
|
!output.Viewport &&
|
|
output.ClipDistance == uint.MaxValue &&
|
|
output.CullDistance == uint.MaxValue)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
var raw = export.Compressed
|
|
? Bitcast(
|
|
_uintType,
|
|
LoadCompressedExportComponent(instruction, (int)component))
|
|
: LoadV(instruction.Sources[(int)component].Value);
|
|
if (output.Layer && _layerOutput != 0)
|
|
{
|
|
StoreConditional(
|
|
_layerOutput,
|
|
BitwiseAnd(raw, UInt(0x7ff)),
|
|
_uintType);
|
|
}
|
|
if (output.Viewport && _viewportIndexOutput != 0)
|
|
{
|
|
var viewport = _module.AddInstruction(
|
|
SpirvOp.BitFieldUExtract,
|
|
_uintType,
|
|
raw,
|
|
UInt(16),
|
|
UInt(4));
|
|
StoreConditional(_viewportIndexOutput, viewport, _uintType);
|
|
}
|
|
|
|
var value = Bitcast(_floatType, raw);
|
|
if (output.PointSize && _pointSizeOutput != 0)
|
|
{
|
|
StoreConditional(_pointSizeOutput, value, _floatType);
|
|
}
|
|
StoreDistanceConditional(
|
|
_clipDistanceOutput,
|
|
output.ClipDistance,
|
|
value);
|
|
StoreDistanceConditional(
|
|
_cullDistanceOutput,
|
|
output.CullDistance,
|
|
value);
|
|
}
|
|
}
|
|
|
|
private void StoreDistanceConditional(
|
|
uint variable,
|
|
uint index,
|
|
uint value)
|
|
{
|
|
if (variable == 0 || index == uint.MaxValue)
|
|
{
|
|
return;
|
|
}
|
|
|
|
var pointer = _module.AddInstruction(
|
|
SpirvOp.AccessChain,
|
|
_module.TypePointer(SpirvStorageClass.Output, _floatType),
|
|
variable,
|
|
UInt(index));
|
|
StoreConditional(pointer, value, _floatType);
|
|
}
|
|
|
|
private void StoreConditional(uint variable, uint value, uint type)
|
|
{
|
|
var selected = _module.AddInstruction(
|
|
SpirvOp.Select,
|
|
type,
|
|
Load(_boolType, _exec),
|
|
value,
|
|
Load(type, variable));
|
|
Store(variable, selected);
|
|
}
|
|
|
|
private uint ConvertPositionToClipSpace(uint position)
|
|
{
|
|
var transform = _request.ClipSpace;
|
|
var components = new uint[4];
|
|
for (var component = 0u; component < 4; component++)
|
|
{
|
|
components[component] = _module.AddInstruction(
|
|
SpirvOp.CompositeExtract,
|
|
_floatType,
|
|
position,
|
|
component);
|
|
}
|
|
|
|
components[0] = ConvertClipCoordinate(
|
|
components[0],
|
|
transform.ScaleX,
|
|
transform.OffsetX,
|
|
transform.HalfExtentX);
|
|
components[1] = ConvertClipCoordinate(
|
|
components[1],
|
|
transform.ScaleY,
|
|
transform.OffsetY,
|
|
transform.HalfExtentY);
|
|
return _module.AddInstruction(
|
|
SpirvOp.CompositeConstruct,
|
|
_vec4Type,
|
|
components);
|
|
}
|
|
|
|
private uint ConvertClipCoordinate(
|
|
uint coordinate,
|
|
float scale,
|
|
float offset,
|
|
float halfExtent)
|
|
{
|
|
var window = _module.AddInstruction(
|
|
SpirvOp.FMul,
|
|
_floatType,
|
|
coordinate,
|
|
Float(scale));
|
|
var biased = _module.AddInstruction(
|
|
SpirvOp.FAdd,
|
|
_floatType,
|
|
window,
|
|
Float(offset));
|
|
var divided = _module.AddInstruction(
|
|
SpirvOp.FDiv,
|
|
_floatType,
|
|
biased,
|
|
Float(halfExtent));
|
|
return _module.AddInstruction(
|
|
SpirvOp.FSub,
|
|
_floatType,
|
|
divided,
|
|
Float(1f));
|
|
}
|
|
|
|
private bool PixelExportDebugAddressMatches()
|
|
{
|
|
var addressFilter = Environment.GetEnvironmentVariable(
|
|
"SHARPEMU_FORCE_PIXEL_EXPORT_ADDRESS");
|
|
if (string.IsNullOrWhiteSpace(addressFilter))
|
|
{
|
|
return true;
|
|
}
|
|
|
|
var span = addressFilter.AsSpan();
|
|
if (span.StartsWith("0x", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
span = span[2..];
|
|
}
|
|
|
|
return ulong.TryParse(
|
|
span,
|
|
System.Globalization.NumberStyles.HexNumber,
|
|
System.Globalization.CultureInfo.InvariantCulture,
|
|
out var address) &&
|
|
_request.Program.Address == address;
|
|
}
|
|
|
|
private bool PixelImageCaptureAddressMatches()
|
|
{
|
|
var addressFilter = Environment.GetEnvironmentVariable(
|
|
"SHARPEMU_CAPTURE_PIXEL_IMAGE_ADDRESS");
|
|
if (string.IsNullOrWhiteSpace(addressFilter))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
var span = addressFilter.AsSpan();
|
|
if (span.StartsWith("0x", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
span = span[2..];
|
|
}
|
|
|
|
return ulong.TryParse(
|
|
span,
|
|
System.Globalization.NumberStyles.HexNumber,
|
|
System.Globalization.CultureInfo.InvariantCulture,
|
|
out var address) &&
|
|
_request.Program.Address == address;
|
|
}
|
|
|
|
private void CapturePixelVgprs(Gen5ShaderInstruction instruction)
|
|
{
|
|
if (_stage != Gen5SpirvStage.Pixel ||
|
|
!PixelVgprCaptureAddressMatches() ||
|
|
!uint.TryParse(
|
|
Environment.GetEnvironmentVariable(
|
|
"SHARPEMU_CAPTURE_PIXEL_VGPR_PC"),
|
|
out var capturePc) ||
|
|
instruction.Pc != capturePc)
|
|
{
|
|
return;
|
|
}
|
|
|
|
var sourceText = Environment.GetEnvironmentVariable(
|
|
"SHARPEMU_CAPTURE_PIXEL_VGPR_SOURCES");
|
|
if (string.IsNullOrWhiteSpace(sourceText))
|
|
{
|
|
return;
|
|
}
|
|
|
|
var destinationBase = 248u;
|
|
if (uint.TryParse(
|
|
Environment.GetEnvironmentVariable(
|
|
"SHARPEMU_CAPTURE_PIXEL_VGPR_DEST_BASE"),
|
|
out var requestedDestinationBase))
|
|
{
|
|
destinationBase = requestedDestinationBase;
|
|
}
|
|
|
|
var sources = sourceText.Split(
|
|
',',
|
|
StringSplitOptions.RemoveEmptyEntries |
|
|
StringSplitOptions.TrimEntries);
|
|
if (sources.Length is 0 or > 4 ||
|
|
destinationBase > 252 ||
|
|
destinationBase + (uint)sources.Length > 256)
|
|
{
|
|
return;
|
|
}
|
|
|
|
for (var index = 0; index < sources.Length; index++)
|
|
{
|
|
if (!uint.TryParse(sources[index], out var source) ||
|
|
source >= 256)
|
|
{
|
|
return;
|
|
}
|
|
}
|
|
|
|
for (var index = 0; index < sources.Length; index++)
|
|
{
|
|
_ = uint.TryParse(sources[index], out var source);
|
|
StoreV(
|
|
destinationBase + (uint)index,
|
|
LoadV(source),
|
|
guardWithExec:
|
|
Environment.GetEnvironmentVariable(
|
|
"SHARPEMU_CAPTURE_PIXEL_VGPR_IGNORE_EXEC") != "1");
|
|
}
|
|
}
|
|
|
|
private bool PixelVgprCaptureAddressMatches()
|
|
{
|
|
var addressFilter = Environment.GetEnvironmentVariable(
|
|
"SHARPEMU_CAPTURE_PIXEL_VGPR_ADDRESS");
|
|
if (string.IsNullOrWhiteSpace(addressFilter))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
var span = addressFilter.AsSpan();
|
|
if (span.StartsWith("0x", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
span = span[2..];
|
|
}
|
|
|
|
return ulong.TryParse(
|
|
span,
|
|
System.Globalization.NumberStyles.HexNumber,
|
|
System.Globalization.CultureInfo.InvariantCulture,
|
|
out var address) &&
|
|
_request.Program.Address == address;
|
|
}
|
|
|
|
private void CapturePixelVgprPoints(Gen5ShaderInstruction instruction)
|
|
{
|
|
if (_stage != Gen5SpirvStage.Pixel ||
|
|
!PixelVgprCaptureAddressMatches())
|
|
{
|
|
return;
|
|
}
|
|
|
|
var captureText = Environment.GetEnvironmentVariable(
|
|
"SHARPEMU_CAPTURE_PIXEL_VGPR_POINTS");
|
|
if (string.IsNullOrWhiteSpace(captureText))
|
|
{
|
|
return;
|
|
}
|
|
|
|
foreach (var capture in captureText.Split(
|
|
',',
|
|
StringSplitOptions.RemoveEmptyEntries |
|
|
StringSplitOptions.TrimEntries))
|
|
{
|
|
var fields = capture.Split(':');
|
|
if (fields.Length != 3 ||
|
|
!uint.TryParse(fields[0], out var pc) ||
|
|
!uint.TryParse(fields[1], out var source) ||
|
|
!uint.TryParse(fields[2], out var destination) ||
|
|
pc != instruction.Pc || source >= 256 || destination >= 256)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
StoreV(
|
|
destination,
|
|
LoadV(source),
|
|
guardWithExec:
|
|
Environment.GetEnvironmentVariable(
|
|
"SHARPEMU_CAPTURE_PIXEL_VGPR_IGNORE_EXEC") != "1");
|
|
}
|
|
}
|
|
|
|
private void MarkPixelPath(Gen5ShaderInstruction instruction)
|
|
{
|
|
if (_stage != Gen5SpirvStage.Pixel ||
|
|
!PixelVgprCaptureAddressMatches())
|
|
{
|
|
return;
|
|
}
|
|
|
|
var markerText = Environment.GetEnvironmentVariable(
|
|
"SHARPEMU_MARK_PIXEL_PCS");
|
|
if (string.IsNullOrWhiteSpace(markerText))
|
|
{
|
|
return;
|
|
}
|
|
|
|
foreach (var marker in markerText.Split(
|
|
',',
|
|
StringSplitOptions.RemoveEmptyEntries |
|
|
StringSplitOptions.TrimEntries))
|
|
{
|
|
var separator = marker.IndexOf(':');
|
|
if (separator <= 0 || separator == marker.Length - 1 ||
|
|
!uint.TryParse(marker.AsSpan(0, separator), out var pc) ||
|
|
!uint.TryParse(marker.AsSpan(separator + 1), out var register) ||
|
|
pc != instruction.Pc || register >= 256)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
StoreV(
|
|
register,
|
|
Bitcast(_uintType, Float(1)),
|
|
guardWithExec: false);
|
|
}
|
|
}
|
|
|
|
private void CapturePixelExec(Gen5ShaderInstruction instruction)
|
|
{
|
|
if (_stage != Gen5SpirvStage.Pixel ||
|
|
!PixelVgprCaptureAddressMatches())
|
|
{
|
|
return;
|
|
}
|
|
|
|
var captureText = Environment.GetEnvironmentVariable(
|
|
"SHARPEMU_CAPTURE_PIXEL_EXEC_PCS");
|
|
if (string.IsNullOrWhiteSpace(captureText))
|
|
{
|
|
return;
|
|
}
|
|
|
|
foreach (var capture in captureText.Split(
|
|
',',
|
|
StringSplitOptions.RemoveEmptyEntries |
|
|
StringSplitOptions.TrimEntries))
|
|
{
|
|
var separator = capture.IndexOf(':');
|
|
if (separator <= 0 || separator == capture.Length - 1 ||
|
|
!uint.TryParse(capture.AsSpan(0, separator), out var pc) ||
|
|
!uint.TryParse(capture.AsSpan(separator + 1), out var register) ||
|
|
pc != instruction.Pc || register >= 256)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
var value = _module.AddInstruction(
|
|
SpirvOp.Select,
|
|
_floatType,
|
|
Load(_boolType, _exec),
|
|
Float(1),
|
|
Float(0));
|
|
StoreV(
|
|
register,
|
|
Bitcast(_uintType, value),
|
|
guardWithExec: false);
|
|
}
|
|
}
|
|
|
|
private bool PixelExportVgprAddressMatches()
|
|
{
|
|
var addressFilter = Environment.GetEnvironmentVariable(
|
|
"SHARPEMU_FORCE_PIXEL_EXPORT_VGPR_ADDRESS");
|
|
if (string.IsNullOrWhiteSpace(addressFilter))
|
|
{
|
|
return PixelExportDebugAddressMatches();
|
|
}
|
|
|
|
var span = addressFilter.AsSpan();
|
|
if (span.StartsWith("0x", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
span = span[2..];
|
|
}
|
|
|
|
return ulong.TryParse(
|
|
span,
|
|
System.Globalization.NumberStyles.HexNumber,
|
|
System.Globalization.CultureInfo.InvariantCulture,
|
|
out var address) &&
|
|
_request.Program.Address == address;
|
|
}
|
|
|
|
private uint LoadCompressedExportComponent(
|
|
Gen5ShaderInstruction instruction,
|
|
int component)
|
|
{
|
|
if (TryLoadPackedHalfExportComponent(
|
|
instruction,
|
|
component,
|
|
out var shadowValue))
|
|
{
|
|
return shadowValue;
|
|
}
|
|
|
|
var packed = LoadV(instruction.Sources[component >> 1].Value);
|
|
var unpacked = Ext(62, _vec2Type, packed);
|
|
return _module.AddInstruction(
|
|
SpirvOp.CompositeExtract,
|
|
_floatType,
|
|
unpacked,
|
|
(uint)(component & 1));
|
|
}
|
|
|
|
private bool TryLoadPackedHalfExportComponent(
|
|
Gen5ShaderInstruction exportInstruction,
|
|
int component,
|
|
out uint value)
|
|
{
|
|
value = 0;
|
|
var packedSource = exportInstruction.Sources[component >> 1];
|
|
var tracePackedExport =
|
|
Environment.GetEnvironmentVariable("SHARPEMU_TRACE_PACKED_EXPORT") == "1" &&
|
|
TraceShaderAddressMatches("SHARPEMU_TRACE_PACKED_EXPORT_ADDRESS");
|
|
if (tracePackedExport)
|
|
{
|
|
Console.Error.WriteLine(
|
|
$"[AGC][PACKED-EXPORT] exp_pc=0x{exportInstruction.Pc:X} " +
|
|
$"component={component} source={packedSource.Kind}:" +
|
|
$"{packedSource.Value}");
|
|
if (component == 0 && exportInstruction.Pc == 0x630)
|
|
{
|
|
foreach (var decoded in _request.Program.Instructions.Where(
|
|
static decoded => decoded.Pc <= 0x640))
|
|
{
|
|
Console.Error.WriteLine(
|
|
$"[AGC][TITLE-IR] 0x{decoded.Pc:X4} " +
|
|
$"{decoded.Opcode} dst=[" +
|
|
string.Join(',', decoded.Destinations) +
|
|
"] src=[" +
|
|
string.Join(',', decoded.Sources) + "] words=[" +
|
|
string.Join(',', decoded.Words.Select(static word => $"{word:X8}")) +
|
|
"] ctrl=" + decoded.Control);
|
|
}
|
|
}
|
|
}
|
|
if (packedSource.Kind != Gen5OperandKind.VectorRegister)
|
|
{
|
|
if (tracePackedExport)
|
|
{
|
|
Console.Error.WriteLine(
|
|
"[AGC][PACKED-EXPORT] rejected: source is not a VGPR");
|
|
}
|
|
return false;
|
|
}
|
|
|
|
for (var index = _request.Program.Instructions.Count - 1; index >= 0; index--)
|
|
{
|
|
var candidate = _request.Program.Instructions[index];
|
|
if (candidate.Pc >= exportInstruction.Pc)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (exportInstruction.Pc - candidate.Pc > 128)
|
|
{
|
|
break;
|
|
}
|
|
|
|
if (!candidate.Destinations.Any(destination =>
|
|
destination.Kind == Gen5OperandKind.VectorRegister &&
|
|
destination.Value == packedSource.Value))
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (tracePackedExport)
|
|
{
|
|
Console.Error.WriteLine(
|
|
$"[AGC][PACKED-EXPORT] nearest_pc=0x{candidate.Pc:X} " +
|
|
$"opcode={candidate.Opcode} distance=" +
|
|
$"{exportInstruction.Pc - candidate.Pc}");
|
|
}
|
|
|
|
if (candidate.Opcode != "VCvtPkrtzF16F32" ||
|
|
candidate.Sources.Count < 2)
|
|
{
|
|
if (tracePackedExport)
|
|
{
|
|
Console.Error.WriteLine(
|
|
"[AGC][PACKED-EXPORT] rejected: nearest writer is " +
|
|
candidate.Opcode);
|
|
}
|
|
return false;
|
|
}
|
|
|
|
var packedPointer = PackedHalfPointer(packedSource.Value);
|
|
if (Environment.GetEnvironmentVariable(
|
|
"SHARPEMU_FORCE_PACKED_EXPORT_STORE_ONE") == "1" &&
|
|
_request.Program.Address == 0x0000000500781200ul)
|
|
{
|
|
Store(
|
|
packedPointer,
|
|
_module.AddInstruction(
|
|
SpirvOp.CompositeConstruct,
|
|
_vec2Type,
|
|
Float(1f),
|
|
Float(1f)));
|
|
}
|
|
|
|
var packedPair = Load(
|
|
_vec2Type,
|
|
packedPointer);
|
|
value = _module.AddInstruction(
|
|
SpirvOp.CompositeExtract,
|
|
_floatType,
|
|
packedPair,
|
|
(uint)(component & 1));
|
|
if (Environment.GetEnvironmentVariable(
|
|
"SHARPEMU_FORCE_PACKED_EXPORT_ONE") == "1")
|
|
{
|
|
value = Float(1f);
|
|
}
|
|
if (tracePackedExport)
|
|
{
|
|
Console.Error.WriteLine(
|
|
"[AGC][PACKED-EXPORT] selected shadow pair");
|
|
}
|
|
return true;
|
|
}
|
|
|
|
if (tracePackedExport)
|
|
{
|
|
Console.Error.WriteLine(
|
|
"[AGC][PACKED-EXPORT] rejected: no nearby writer");
|
|
}
|
|
return false;
|
|
}
|
|
|
|
private bool TraceShaderAddressMatches(string environmentVariable)
|
|
{
|
|
var filter = Environment.GetEnvironmentVariable(environmentVariable);
|
|
if (string.IsNullOrWhiteSpace(filter))
|
|
{
|
|
return true;
|
|
}
|
|
|
|
var span = filter.AsSpan();
|
|
if (span.StartsWith("0x", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
span = span[2..];
|
|
}
|
|
|
|
return ulong.TryParse(
|
|
span,
|
|
System.Globalization.NumberStyles.HexNumber,
|
|
System.Globalization.CultureInfo.InvariantCulture,
|
|
out var address) &&
|
|
_request.Program.Address == address;
|
|
}
|
|
|
|
private uint GetPixelOutputType(Gen5PixelOutputKind kind) =>
|
|
kind switch
|
|
{
|
|
Gen5PixelOutputKind.Uint => _uvec4Type,
|
|
Gen5PixelOutputKind.Sint => _module.TypeVector(_intType, 4),
|
|
_ => _vec4Type,
|
|
};
|
|
|
|
private uint LoadBufferWord(int binding, uint dwordAddress)
|
|
{
|
|
var inRange = IsBufferWordInRange(binding, dwordAddress);
|
|
var safeAddress = _module.AddInstruction(
|
|
SpirvOp.Select,
|
|
_uintType,
|
|
inRange,
|
|
dwordAddress,
|
|
UInt(0));
|
|
var value = Load(_uintType, BufferWordPointer(binding, safeAddress));
|
|
return _module.AddInstruction(
|
|
SpirvOp.Select,
|
|
_uintType,
|
|
inRange,
|
|
value,
|
|
UInt(0));
|
|
}
|
|
|
|
private uint ApplyGuestBufferByteBias(int binding, uint byteAddress)
|
|
{
|
|
{
|
|
// The packed memory-offset byte of this buffer, loaded at entry.
|
|
return IAdd(byteAddress, Load(_uintType, RuntimeBufferBiasPointer(binding)));
|
|
}
|
|
}
|
|
|
|
private void StoreBufferWord(int binding, uint dwordAddress, uint value)
|
|
{
|
|
EmitConditional(
|
|
IsBufferWordInRange(binding, dwordAddress),
|
|
() => Store(BufferWordPointer(binding, dwordAddress), value));
|
|
}
|
|
|
|
private uint IsBufferWordInRange(int binding, uint dwordAddress)
|
|
{
|
|
var buffer = _module.AddInstruction(
|
|
SpirvOp.AccessChain,
|
|
_storageBlockPointer,
|
|
_globalBuffers,
|
|
UInt((uint)binding));
|
|
var length = _module.AddInstruction(
|
|
SpirvOp.ArrayLength,
|
|
_uintType,
|
|
buffer,
|
|
0);
|
|
return _module.AddInstruction(
|
|
SpirvOp.ULessThan,
|
|
_boolType,
|
|
dwordAddress,
|
|
length);
|
|
}
|
|
|
|
private uint BufferWordPointer(int binding, uint dwordAddress) =>
|
|
_module.AddInstruction(
|
|
SpirvOp.AccessChain,
|
|
_storageUintPointer,
|
|
_globalBuffers,
|
|
UInt((uint)binding),
|
|
UInt(0),
|
|
dwordAddress);
|
|
|
|
// Without subgroup operations each invocation is a one-lane wave, so a V_WRITELANE to
|
|
// lane N > 0 used to be dropped and V_READLANE returned lane 0. Compilers spill SGPRs
|
|
// (EXEC included) to VGPR lanes like this; restoring the wrong EXEC made waterfall
|
|
// loops spin until the dispatcher guard. Only VGPRs that are also read back get slots.
|
|
private List<(uint Register, uint Lane)> FindLaneSpillSlots()
|
|
{
|
|
var slots = new List<(uint Register, uint Lane)>();
|
|
if (_emulateWave64)
|
|
{
|
|
return slots;
|
|
}
|
|
|
|
var ownsLaneZero = !UsesSubgroupOperations();
|
|
var readRegisters = _request.Program.Instructions
|
|
.Where(static instruction => instruction.Opcode == "VReadlaneB32" &&
|
|
instruction.Sources.Count > 0 &&
|
|
instruction.Sources[0].Kind == Gen5OperandKind.VectorRegister)
|
|
.Select(static instruction => instruction.Sources[0].Value)
|
|
.ToHashSet();
|
|
foreach (var instruction in _request.Program.Instructions)
|
|
{
|
|
if (instruction.Opcode == "VWritelaneB32" &&
|
|
TryGetVectorDestination(instruction, out var register) &&
|
|
readRegisters.Contains(register) &&
|
|
TryGetConstantLane(instruction, out var lane) &&
|
|
(lane != 0 || !ownsLaneZero) &&
|
|
!slots.Contains((register, lane)))
|
|
{
|
|
slots.Add((register, lane));
|
|
}
|
|
}
|
|
|
|
return slots;
|
|
}
|
|
|
|
// The lane select operand (source 1) of V_READLANE/V_WRITELANE when it is a constant.
|
|
private bool TryGetConstantLane(Gen5ShaderInstruction instruction, out uint lane)
|
|
{
|
|
lane = 0;
|
|
if (instruction.Sources.Count < 2)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
var operand = instruction.Sources[1];
|
|
uint value;
|
|
if (operand.Kind == Gen5OperandKind.LiteralConstant)
|
|
{
|
|
value = operand.Value;
|
|
}
|
|
else if (operand.Kind != Gen5OperandKind.EncodedConstant ||
|
|
!TryDecodeInlineConstant(operand.Value, out value))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
lane = value & LaneSelectMask;
|
|
return true;
|
|
}
|
|
|
|
private uint LaneSelectMask => _stage == Gen5SpirvStage.Compute ? _waveLaneCount - 1 : 63u;
|
|
|
|
private uint ReadLaneSpillSlot(Gen5ShaderInstruction instruction, uint selectedLane, uint value)
|
|
{
|
|
if (instruction.Sources[0].Kind != Gen5OperandKind.VectorRegister)
|
|
{
|
|
return value;
|
|
}
|
|
|
|
var register = instruction.Sources[0].Value;
|
|
if (TryGetConstantLane(instruction, out var lane))
|
|
{
|
|
return _laneSpillSlots.TryGetValue((register, lane), out var slot) ? Load(_uintType, slot) : value;
|
|
}
|
|
|
|
return SelectLaneSpillSlot(register, selectedLane, value);
|
|
}
|
|
|
|
// Folds the spill slots of register into value: selected lane == slot lane reads the slot.
|
|
private uint SelectLaneSpillSlot(uint register, uint selectedLane, uint value)
|
|
{
|
|
foreach (var ((slotRegister, slotLane), variable) in _laneSpillSlots)
|
|
{
|
|
if (slotRegister != register)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
var isSlotLane = _module.AddInstruction(SpirvOp.IEqual, _boolType, selectedLane, UInt(slotLane));
|
|
value = _module.AddInstruction(SpirvOp.Select, _uintType, isSlotLane, Load(_uintType, variable), value);
|
|
}
|
|
|
|
return value;
|
|
}
|
|
|
|
private uint ScalarPointer(uint register)
|
|
{
|
|
if (register >= ScalarRegisterCount)
|
|
{
|
|
throw new InvalidOperationException($"Scalar register s{register} is outside the register file.");
|
|
}
|
|
|
|
return RegisterVariable(_scalarRegisterVariables, register, "s");
|
|
}
|
|
|
|
private uint RegisterVariable(Dictionary<uint, uint> variables, uint register, string prefix)
|
|
{
|
|
if (!variables.TryGetValue(register, out var variable))
|
|
{
|
|
variable = _module.AddGlobalVariable(_privateUintPointer, SpirvStorageClass.Private, _module.ConstantNull(_uintType));
|
|
_interfaces.Add(variable);
|
|
_module.AddName(variable, $"{prefix}{register}");
|
|
variables.Add(register, variable);
|
|
}
|
|
|
|
return variable;
|
|
}
|
|
|
|
private uint RuntimeBufferBiasPointer(int binding) =>
|
|
_module.AddInstruction(
|
|
SpirvOp.AccessChain,
|
|
_privateUintPointer,
|
|
_runtimeBufferBiases,
|
|
UInt(checked((uint)binding)));
|
|
|
|
private uint VectorPointer(uint register)
|
|
{
|
|
if (_vectorRegisters != 0)
|
|
{
|
|
return _module.AddInstruction(
|
|
SpirvOp.AccessChain,
|
|
_privateUintPointer,
|
|
_vectorRegisters,
|
|
UInt(register));
|
|
}
|
|
|
|
if (register >= VectorRegisterCount)
|
|
{
|
|
throw new InvalidOperationException($"Vector register v{register} is outside the register file.");
|
|
}
|
|
|
|
return RegisterVariable(_vectorRegisterVariables, register, "v");
|
|
}
|
|
|
|
// The V_MOVREL* opcodes address the VGPR file with a register number that
|
|
// is only known at run time (encoded number + M0), so the access chain
|
|
// takes a computed index instead of a constant. The index is masked to
|
|
// the array bounds: SPIR-V leaves an out-of-range Private access chain
|
|
// undefined, and a mask costs nothing next to the surrounding load.
|
|
// Every register a relative access can reach: from the lowest V_MOVREL* base to past the
|
|
// highest register the program names (a multi-dword result may extend beyond its listed
|
|
// first register). An index outside it reads zero and writes nothing.
|
|
private static (uint First, uint Last) DynamicVectorRange(Gen5ShaderProgram program)
|
|
{
|
|
var first = VectorRegisterCount - 1;
|
|
var last = 0u;
|
|
foreach (var instruction in program.Instructions)
|
|
{
|
|
foreach (var operand in instruction.Sources.Concat(instruction.Destinations))
|
|
{
|
|
if (operand.Kind == Gen5OperandKind.VectorRegister)
|
|
{
|
|
last = Math.Max(last, operand.Value);
|
|
if (instruction.Opcode.StartsWith("VMovrel", StringComparison.Ordinal))
|
|
{
|
|
first = Math.Min(first, operand.Value);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
last = Math.Min(last + 8, VectorRegisterCount - 1);
|
|
return (Math.Min(first, last), last);
|
|
}
|
|
|
|
private uint LoadVDynamic(uint registerIndex)
|
|
{
|
|
var result = UInt(0);
|
|
for (var register = _dynamicVectorRange.First; register <= _dynamicVectorRange.Last; register++)
|
|
{
|
|
result = _module.AddInstruction(
|
|
SpirvOp.Select,
|
|
_uintType,
|
|
_module.AddInstruction(SpirvOp.IEqual, _boolType, registerIndex, UInt(register)),
|
|
Load(_uintType, VectorPointer(register)),
|
|
result);
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
private void StoreVDynamic(uint registerIndex, uint value)
|
|
{
|
|
// With EXEC known to be all ones the write needs no EXEC test.
|
|
var exec = _execKnownFull ? 0 : Load(_boolType, _exec);
|
|
for (var register = _dynamicVectorRange.First; register <= _dynamicVectorRange.Last; register++)
|
|
{
|
|
var pointer = VectorPointer(register);
|
|
var selected = _module.AddInstruction(SpirvOp.IEqual, _boolType, registerIndex, UInt(register));
|
|
if (exec != 0)
|
|
{
|
|
selected = _module.AddInstruction(SpirvOp.LogicalAnd, _boolType, exec, selected);
|
|
}
|
|
Store(pointer, _module.AddInstruction(SpirvOp.Select, _uintType, selected, value, Load(_uintType, pointer)));
|
|
}
|
|
}
|
|
|
|
private uint PackedHalfPointer(uint register) =>
|
|
_module.AddInstruction(
|
|
SpirvOp.AccessChain,
|
|
_privateVec2Pointer,
|
|
PackedHalfRegisters(),
|
|
UInt(register));
|
|
|
|
// Declared on first use when Private. AMD's compiler keeps an unused 4 KiB private
|
|
// array as a named .bss global: two stages then fail to link, and the driver copies
|
|
// the NOBITS section as file data and reads past the end of the ELF.
|
|
private uint PackedHalfRegisters()
|
|
{
|
|
if (_packedHalfRegisters != 0)
|
|
{
|
|
return _packedHalfRegisters;
|
|
}
|
|
|
|
var arrayType = _module.TypeArray(_vec2Type, VectorRegisterCount);
|
|
_packedHalfRegisters = _module.AddGlobalVariable(
|
|
_module.TypePointer(SpirvStorageClass.Private, arrayType),
|
|
SpirvStorageClass.Private,
|
|
_module.ConstantNull(arrayType));
|
|
_interfaces.Add(_packedHalfRegisters);
|
|
_module.AddName(_packedHalfRegisters, "vgprPackedHalf");
|
|
return _packedHalfRegisters;
|
|
}
|
|
|
|
private uint LoadS(uint register) => Load(_uintType, ScalarPointer(register));
|
|
|
|
private uint LoadV(uint register) => Load(_uintType, VectorPointer(register));
|
|
|
|
private void StoreS(uint register, uint value)
|
|
{
|
|
Store(ScalarPointer(register), value);
|
|
if (register is 106 or 107)
|
|
{
|
|
if (_waveLaneCount != 32 || register == 106)
|
|
{
|
|
Store(_vcc, IsLaneSetInMaskRegisters(106));
|
|
}
|
|
}
|
|
else if (register is 126 or 127)
|
|
{
|
|
if (_waveLaneCount != 32 || register == 126)
|
|
{
|
|
Store(_exec, IsLaneSetInMaskRegisters(126));
|
|
}
|
|
}
|
|
}
|
|
|
|
// In wave32 a lane mask is its low register alone (VCC_HI and EXEC_HI are ordinary SGPRs),
|
|
// so the lane's bit is tested on one dword instead of a composed 64-bit mask.
|
|
private uint IsLaneSetInMaskRegisters(uint lowRegister)
|
|
{
|
|
if (_waveLaneCount != 32)
|
|
{
|
|
return IsWaveMaskActive(LoadS64(lowRegister));
|
|
}
|
|
|
|
var laneBit = _subgroupInvocationIdInput == 0
|
|
? UInt(1)
|
|
: ShiftLeftLogical(UInt(1), GuestWaveLane());
|
|
return IsNotZero(BitwiseAnd(LoadS(lowRegister), laneBit));
|
|
}
|
|
|
|
// SHARPEMU_EXEC_GUARD_ELISION=0 guards every vector register write with EXEC again.
|
|
private static readonly bool ExecGuardElision = !string.Equals(
|
|
Environment.GetEnvironmentVariable("SHARPEMU_EXEC_GUARD_ELISION"), "0", StringComparison.Ordinal);
|
|
|
|
private IReadOnlySet<uint>? _execFullPcs;
|
|
|
|
// Set while an instruction that starts with every EXEC lane set is emitted: its vector
|
|
// register writes need no EXEC guard, so the old values need not stay live until them.
|
|
private bool _execKnownFull;
|
|
|
|
private bool IsExecKnownFull(uint pc)
|
|
{
|
|
if (!ExecGuardElision || !_request.EnableExecGuardElision)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
_execFullPcs ??= Ir.Gen5ExecFullAnalysis.Analyze(_request.Program, wave32: _waveLaneCount == 32);
|
|
return _execFullPcs.Contains(pc);
|
|
}
|
|
|
|
private void StoreV(uint register, uint value, bool guardWithExec = true)
|
|
{
|
|
if (guardWithExec && !_execKnownFull)
|
|
{
|
|
var active = Load(_boolType, _exec);
|
|
var oldValue = LoadV(register);
|
|
value = _module.AddInstruction(
|
|
SpirvOp.Select,
|
|
_uintType,
|
|
active,
|
|
value,
|
|
oldValue);
|
|
}
|
|
|
|
Store(VectorPointer(register), value);
|
|
}
|
|
|
|
private void StorePackedHalf(uint register, uint value)
|
|
{
|
|
if (_execKnownFull)
|
|
{
|
|
Store(PackedHalfPointer(register), value);
|
|
return;
|
|
}
|
|
|
|
var active = Load(_boolType, _exec);
|
|
if (Environment.GetEnvironmentVariable(
|
|
"SHARPEMU_FORCE_PACKED_STORE_EXEC_VALUES") == "1" &&
|
|
_request.Program.Address == 0x0000000500781200ul)
|
|
{
|
|
var activePair = _module.AddInstruction(
|
|
SpirvOp.CompositeConstruct,
|
|
_vec2Type,
|
|
Float(1f),
|
|
Float(1f));
|
|
var inactivePair = _module.AddInstruction(
|
|
SpirvOp.CompositeConstruct,
|
|
_vec2Type,
|
|
Float(0.5f),
|
|
Float(0.5f));
|
|
value = _module.AddInstruction(
|
|
SpirvOp.Select,
|
|
_vec2Type,
|
|
active,
|
|
activePair,
|
|
inactivePair);
|
|
Store(PackedHalfPointer(register), value);
|
|
return;
|
|
}
|
|
|
|
value = _module.AddInstruction(
|
|
SpirvOp.Select,
|
|
_vec2Type,
|
|
active,
|
|
value,
|
|
Load(_vec2Type, PackedHalfPointer(register)));
|
|
Store(PackedHalfPointer(register), value);
|
|
}
|
|
|
|
private uint Load(uint type, uint pointer)
|
|
{
|
|
if (pointer == 0)
|
|
{
|
|
throw new InvalidOperationException(
|
|
"SPIR-V generator attempted OpLoad from id 0.");
|
|
}
|
|
|
|
if (_flagVariables.Contains(pointer))
|
|
{
|
|
return _module.AddInstruction(
|
|
SpirvOp.INotEqual,
|
|
_boolType,
|
|
_module.AddInstruction(SpirvOp.Load, _uintType, pointer),
|
|
UInt(0));
|
|
}
|
|
|
|
return _module.AddInstruction(SpirvOp.Load, type, pointer);
|
|
}
|
|
|
|
private void Store(uint pointer, uint value)
|
|
{
|
|
if (_flagVariables.Contains(pointer))
|
|
{
|
|
value = _module.AddInstruction(
|
|
SpirvOp.Select,
|
|
_uintType,
|
|
value,
|
|
UInt(1),
|
|
UInt(0));
|
|
}
|
|
|
|
_module.AddStatement(SpirvOp.Store, pointer, value);
|
|
}
|
|
|
|
private uint UInt(uint value) => _module.Constant(_uintType, value);
|
|
|
|
private uint Float(float value) => _module.ConstantFloat(_floatType, value);
|
|
|
|
private uint Bitcast(uint type, uint value) =>
|
|
_module.AddInstruction(SpirvOp.Bitcast, type, value);
|
|
|
|
private uint IAdd(uint left, uint right) =>
|
|
_module.AddInstruction(SpirvOp.IAdd, _uintType, left, right);
|
|
|
|
private uint ShiftLeftLogical(uint left, uint right) =>
|
|
_module.AddInstruction(
|
|
SpirvOp.ShiftLeftLogical,
|
|
_uintType,
|
|
left,
|
|
BitwiseAnd(right, UInt(31)));
|
|
|
|
private uint ShiftRightLogical(uint left, uint right) =>
|
|
_module.AddInstruction(
|
|
SpirvOp.ShiftRightLogical,
|
|
_uintType,
|
|
left,
|
|
BitwiseAnd(right, UInt(31)));
|
|
|
|
private uint ShiftRightArithmetic(uint left, uint right) =>
|
|
Bitcast(
|
|
_uintType,
|
|
_module.AddInstruction(
|
|
SpirvOp.ShiftRightArithmetic,
|
|
_intType,
|
|
Bitcast(_intType, left),
|
|
BitwiseAnd(right, UInt(31))));
|
|
|
|
private uint _uintPairType;
|
|
private uint _intPairType;
|
|
|
|
private (uint Low, uint High) MultiplyExtended(uint left, uint right, bool signed)
|
|
{
|
|
var elementType = signed ? _intType : _uintType;
|
|
if (signed && _intPairType == 0)
|
|
{
|
|
_intPairType = _module.TypeStruct(_intType, _intType);
|
|
}
|
|
else if (!signed && _uintPairType == 0)
|
|
{
|
|
_uintPairType = _module.TypeStruct(_uintType, _uintType);
|
|
}
|
|
|
|
var product = _module.AddInstruction(
|
|
signed ? SpirvOp.SMulExtended : SpirvOp.UMulExtended,
|
|
signed ? _intPairType : _uintPairType,
|
|
left,
|
|
right);
|
|
return (
|
|
_module.AddInstruction(SpirvOp.CompositeExtract, elementType, product, 0),
|
|
_module.AddInstruction(SpirvOp.CompositeExtract, elementType, product, 1));
|
|
}
|
|
|
|
private uint ShiftLeftLogical64(uint left, uint right) =>
|
|
_module.AddInstruction(
|
|
SpirvOp.ShiftLeftLogical,
|
|
_ulongType,
|
|
left,
|
|
BitwiseAnd64(right, _module.Constant64(_ulongType, 63)));
|
|
|
|
private uint ShiftRightLogical64(uint left, uint right) =>
|
|
_module.AddInstruction(
|
|
SpirvOp.ShiftRightLogical,
|
|
_ulongType,
|
|
left,
|
|
BitwiseAnd64(right, _module.Constant64(_ulongType, 63)));
|
|
|
|
private uint ShiftRightArithmetic64(uint left, uint right) =>
|
|
Bitcast(
|
|
_ulongType,
|
|
_module.AddInstruction(
|
|
SpirvOp.ShiftRightArithmetic,
|
|
_longType,
|
|
Bitcast(_longType, left),
|
|
BitwiseAnd64(right, _module.Constant64(_ulongType, 63))));
|
|
|
|
private uint BitwiseAnd(uint left, uint right) =>
|
|
_module.AddInstruction(SpirvOp.BitwiseAnd, _uintType, left, right);
|
|
|
|
private uint BitwiseAnd64(uint left, uint right) =>
|
|
_module.AddInstruction(SpirvOp.BitwiseAnd, _ulongType, left, right);
|
|
|
|
private uint BitwiseOr64(uint left, uint right) =>
|
|
_module.AddInstruction(SpirvOp.BitwiseOr, _ulongType, left, right);
|
|
|
|
private uint BitwiseOr(uint left, uint right) =>
|
|
_module.AddInstruction(SpirvOp.BitwiseOr, _uintType, left, right);
|
|
|
|
private uint BitwiseXor(uint left, uint right) =>
|
|
_module.AddInstruction(SpirvOp.BitwiseXor, _uintType, left, right);
|
|
|
|
private uint LogicalNot(uint value) =>
|
|
_module.AddInstruction(SpirvOp.LogicalNot, _boolType, value);
|
|
|
|
private uint SubgroupAny(uint condition) =>
|
|
_subgroupInvocationIdInput == 0
|
|
? condition
|
|
: _emulateWave64
|
|
? IsNotZero64(BooleanToWaveMask(condition))
|
|
: _module.AddInstruction(
|
|
SpirvOp.GroupNonUniformAny,
|
|
_boolType,
|
|
UInt(3),
|
|
condition);
|
|
|
|
private uint GuestWaveLane()
|
|
{
|
|
if (_waveLaneCount == 64 && _localInvocationIndexInput != 0)
|
|
{
|
|
return BitwiseAnd(
|
|
Load(_uintType, _localInvocationIndexInput),
|
|
UInt(63));
|
|
}
|
|
|
|
if (_subgroupInvocationIdInput != 0)
|
|
{
|
|
return BitwiseAnd(
|
|
Load(_uintType, _subgroupInvocationIdInput),
|
|
UInt(31));
|
|
}
|
|
|
|
// Graphics stages without subgroup support have one logical lane;
|
|
// they must not emit OpLoad for absent SPIR-V input ID zero.
|
|
return UInt(0);
|
|
}
|
|
|
|
// Reads value from another lane of the host subgroup. Without subgroup
|
|
// support the invocation is a one-lane wave, so the only lane is itself;
|
|
// emitting the shuffle there would need a capability the module lacks.
|
|
// 1.0 when "reference <function> texel" holds, else 0.0. The function is the
|
|
// guest sampler's depth compare field, which uses VkCompareOp's order.
|
|
private uint EmulatedDepthCompare(uint reference, uint texel, int function)
|
|
{
|
|
SpirvOp op;
|
|
switch (function)
|
|
{
|
|
case 0: return Float(0f);
|
|
case 7: return Float(1f);
|
|
case 1: op = SpirvOp.FOrdLessThan; break;
|
|
case 2: op = SpirvOp.FOrdEqual; break;
|
|
case 3: op = SpirvOp.FOrdLessThanEqual; break;
|
|
case 4: op = SpirvOp.FOrdGreaterThan; break;
|
|
case 5: op = SpirvOp.FUnordNotEqual; break;
|
|
case 6: op = SpirvOp.FOrdGreaterThanEqual; break;
|
|
default: throw new InvalidOperationException($"invalid depth compare function {function}");
|
|
}
|
|
|
|
var passed = _module.AddInstruction(op, _boolType, reference, texel);
|
|
return _module.AddInstruction(SpirvOp.Select, _floatType, passed, Float(1f), Float(0f));
|
|
}
|
|
|
|
private uint ShuffleLane(uint value, uint lane) =>
|
|
_subgroupInvocationIdInput == 0
|
|
? value
|
|
: _module.AddInstruction(SpirvOp.GroupNonUniformShuffle, _uintType, UInt(3), value, lane);
|
|
|
|
private uint CurrentLaneBit()
|
|
{
|
|
if (_subgroupInvocationIdInput == 0)
|
|
{
|
|
return _module.Constant64(_ulongType, 1);
|
|
}
|
|
|
|
var maskedLane = GuestWaveLane();
|
|
var shifted = ShiftLeftLogical64(
|
|
_module.Constant64(_ulongType, 1),
|
|
_module.AddInstruction(
|
|
SpirvOp.UConvert,
|
|
_ulongType,
|
|
maskedLane));
|
|
return _emulateWave64
|
|
? shifted
|
|
: _module.AddInstruction(
|
|
SpirvOp.Select,
|
|
_ulongType,
|
|
IsCurrentLaneInRdnaWave(),
|
|
shifted,
|
|
_module.Constant64(_ulongType, 0));
|
|
}
|
|
|
|
private uint IsCurrentLaneInRdnaWave() =>
|
|
_module.AddInstruction(
|
|
SpirvOp.ULessThan,
|
|
_boolType,
|
|
Load(_uintType, _subgroupInvocationIdInput),
|
|
UInt(32));
|
|
|
|
private uint BooleanToLaneMask(uint condition) =>
|
|
_module.AddInstruction(
|
|
SpirvOp.Select,
|
|
_ulongType,
|
|
condition,
|
|
CurrentLaneBit(),
|
|
_module.Constant64(_ulongType, 0));
|
|
|
|
private uint BooleanToWaveMask(uint condition)
|
|
{
|
|
if (_subgroupInvocationIdInput == 0)
|
|
{
|
|
return BooleanToLaneMask(condition);
|
|
}
|
|
|
|
if (_emulateWave64)
|
|
{
|
|
var lane = GuestWaveLane();
|
|
var owned = OwnHalfBallot(condition);
|
|
var exchange = BeginWave64Exchange();
|
|
EmitConditional(IsHalfWaveLeader(lane), () =>
|
|
Store(Wave64ExchangePointer(exchange, ShiftRightLogical(lane, UInt(5))), owned));
|
|
EmitWave64Barrier();
|
|
return Pair64(
|
|
Load(_uintType, Wave64ExchangePointer(exchange, UInt(0))),
|
|
Load(_uintType, Wave64ExchangePointer(exchange, UInt(1))));
|
|
}
|
|
|
|
var ballot = _module.AddInstruction(
|
|
SpirvOp.GroupNonUniformBallot,
|
|
_uvec4Type,
|
|
UInt(3),
|
|
condition);
|
|
var low = _module.AddInstruction(
|
|
SpirvOp.CompositeExtract,
|
|
_uintType,
|
|
ballot,
|
|
0);
|
|
var widened = _module.AddInstruction(SpirvOp.UConvert, _ulongType, low);
|
|
if (_waveLaneCount != 64)
|
|
{
|
|
return widened;
|
|
}
|
|
|
|
return _module.AddInstruction(
|
|
SpirvOp.Select,
|
|
_ulongType,
|
|
_module.AddInstruction(
|
|
SpirvOp.UGreaterThanEqual,
|
|
_boolType,
|
|
GuestWaveLane(),
|
|
UInt(32)),
|
|
ShiftLeftLogical64(
|
|
widened,
|
|
_module.Constant64(_ulongType, 32)),
|
|
widened);
|
|
}
|
|
|
|
private uint OwnHalfBallot(uint condition)
|
|
{
|
|
var ballot = _module.AddInstruction(
|
|
SpirvOp.GroupNonUniformBallot,
|
|
_uvec4Type,
|
|
UInt(3),
|
|
condition);
|
|
var component = ShiftRightLogical(Load(_uintType, _subgroupInvocationIdInput), UInt(5));
|
|
// Avoid dynamic extraction of subgroup ballot words: on the NVIDIA
|
|
// device tests it produced zero masks. Select the same word explicitly.
|
|
var result = _module.AddInstruction(SpirvOp.CompositeExtract, _uintType, ballot, 0);
|
|
for (uint index = 1; index < 4; index++)
|
|
{
|
|
var word = _module.AddInstruction(SpirvOp.CompositeExtract, _uintType, ballot, index);
|
|
result = _module.AddInstruction(SpirvOp.Select, _uintType,
|
|
_module.AddInstruction(SpirvOp.IEqual, _boolType, component, UInt(index)), word, result);
|
|
}
|
|
return result;
|
|
}
|
|
|
|
private uint BooleanToHalfWaveMask(uint condition)
|
|
{
|
|
var owned = _module.AddInstruction(SpirvOp.UConvert, _ulongType, OwnHalfBallot(condition));
|
|
return _module.AddInstruction(
|
|
SpirvOp.Select,
|
|
_ulongType,
|
|
_module.AddInstruction(SpirvOp.UGreaterThanEqual, _boolType, GuestWaveLane(), UInt(32)),
|
|
ShiftLeftLogical64(owned, _module.Constant64(_ulongType, 32)),
|
|
owned);
|
|
}
|
|
|
|
private uint BothHalvesOfOwnBallot(uint condition)
|
|
{
|
|
var owned = _module.AddInstruction(SpirvOp.UConvert, _ulongType, OwnHalfBallot(condition));
|
|
return BitwiseOr64(owned, ShiftLeftLogical64(owned, _module.Constant64(_ulongType, 32)));
|
|
}
|
|
|
|
private uint IsHalfWaveLeader(uint lane) =>
|
|
_module.AddInstruction(SpirvOp.IEqual, _boolType, BitwiseAnd(lane, UInt(31)), UInt(0));
|
|
|
|
private uint BeginWave64Exchange()
|
|
{
|
|
var parity = Load(_uintType, _wave64ExchangeParity);
|
|
Store(_wave64ExchangeParity, BitwiseXor(parity, UInt(1)));
|
|
return IAdd(
|
|
UInt(_wave64ExchangeOffset),
|
|
_module.AddInstruction(SpirvOp.IMul, _uintType, parity, UInt(Wave64ExchangeSlotCount)));
|
|
}
|
|
|
|
private uint Wave64ExchangePointer(uint exchange, uint slot) =>
|
|
_module.AddInstruction(
|
|
SpirvOp.AccessChain,
|
|
_wave64ExchangeElementPointer,
|
|
_wave64Exchange,
|
|
IAdd(exchange, slot));
|
|
|
|
private uint ExchangeWave64Value(uint writer, uint value)
|
|
{
|
|
var exchange = BeginWave64Exchange();
|
|
EmitConditional(writer, () => Store(Wave64ExchangePointer(exchange, UInt(0)), value));
|
|
EmitWave64Barrier();
|
|
return Load(_uintType, Wave64ExchangePointer(exchange, UInt(0)));
|
|
}
|
|
|
|
private void EmitExactWaveMasks(IReadOnlyList<uint> pairs)
|
|
{
|
|
var lane = GuestWaveLane();
|
|
var upperHalf = ShiftRightLogical(lane, UInt(5));
|
|
var lowerHalf = _module.AddInstruction(SpirvOp.IEqual, _boolType, upperHalf, UInt(0));
|
|
const int pairsPerExchange = (int)Wave64ExchangeSlotCount / 2;
|
|
for (var start = 0; start < pairs.Count; start += pairsPerExchange)
|
|
{
|
|
var count = Math.Min(pairsPerExchange, pairs.Count - start);
|
|
var owned = new uint[count];
|
|
for (var index = 0; index < count; index++)
|
|
{
|
|
var pair = pairs[start + index];
|
|
owned[index] = _module.AddInstruction(SpirvOp.Select, _uintType, lowerHalf, LoadS(pair), LoadS(pair + 1));
|
|
}
|
|
|
|
var exchange = BeginWave64Exchange();
|
|
EmitConditional(IsHalfWaveLeader(lane), () =>
|
|
{
|
|
for (var index = 0; index < count; index++)
|
|
{
|
|
Store(Wave64ExchangePointer(exchange, IAdd(UInt((uint)index * 2), upperHalf)), owned[index]);
|
|
}
|
|
});
|
|
EmitWave64Barrier();
|
|
for (var index = 0; index < count; index++)
|
|
{
|
|
var pair = pairs[start + index];
|
|
StoreS(pair, Load(_uintType, Wave64ExchangePointer(exchange, UInt((uint)index * 2))));
|
|
StoreS(pair + 1, Load(_uintType, Wave64ExchangePointer(exchange, UInt((uint)index * 2 + 1))));
|
|
}
|
|
}
|
|
}
|
|
|
|
private uint WaveMaskAny(uint register, uint laneActive) =>
|
|
_emulateWave64 && _subgroupInvocationIdInput != 0
|
|
? IsNotZero64(LoadS64(register))
|
|
: SubgroupAny(Load(_boolType, laneActive));
|
|
|
|
private static readonly bool HalfWaveMasks = !string.Equals(
|
|
Environment.GetEnvironmentVariable("SHARPEMU_WAVE64_HALF_MASKS"), "0", StringComparison.Ordinal);
|
|
|
|
private bool _halfMaskPlanBuilt;
|
|
|
|
private Ir.Gen5Wave64HalfMaskPlan? HalfMaskPlan()
|
|
{
|
|
if (_halfMaskPlanBuilt)
|
|
{
|
|
return _halfMaskPlan;
|
|
}
|
|
|
|
_halfMaskPlanBuilt = true;
|
|
if (HalfWaveMasks && UsesWave64Exchange() && _subgroupInvocationIdInput != 0)
|
|
{
|
|
var sharedFlatPcs = _request.Memory.Entries
|
|
.Where(static memory => memory.AddressSpace is FlatAddressSpace.Shared or FlatAddressSpace.SharedOrPrivate)
|
|
.Select(static memory => memory.Pc)
|
|
.ToHashSet();
|
|
_halfMaskPlan = Ir.Gen5Wave64HalfMaskAnalysis.Analyze(_request.Program, sharedFlatPcs);
|
|
}
|
|
|
|
return _halfMaskPlan;
|
|
}
|
|
|
|
private void EmitWave64Barrier()
|
|
{
|
|
var workgroup = UInt(2);
|
|
_module.AddStatement(
|
|
SpirvOp.ControlBarrier,
|
|
workgroup,
|
|
workgroup,
|
|
UInt(0x108));
|
|
}
|
|
|
|
// A wave-mask SGPR (VCC/EXEC) consumed as a per-lane predicate — the
|
|
// condition of VCndmask, a VCC/EXEC branch, or the derived _vcc/_exec
|
|
// bool — must be tested at the CURRENT lane's bit, exactly as the
|
|
// hardware does, not as "the 64-bit value is non-zero". The two coincide
|
|
// for comparison results (only the lane's own bit is ever set), so the
|
|
// single-lane path historically used a cheaper whole-word non-zero test.
|
|
// But bitwise-complement wave-mask idioms (S_NOT/S_ORN2/S_ANDN2/S_NAND/
|
|
// S_NOR on a 64-bit mask) set the unused upper 63 bits; a whole-word test
|
|
// then reports "lane active" even when this lane's bit is clear. Unity's
|
|
// PostProcessing NaN killer does exactly this (`anyNaN | ~allFinite`),
|
|
// which made every valid pixel read as NaN and get replaced with 0 —
|
|
// zeroing the whole scene before tonemap. Extract the lane bit always.
|
|
private uint IsWaveMaskActive(uint mask) =>
|
|
IsCurrentLaneSet(mask);
|
|
|
|
private uint IsCurrentLaneSet(uint mask) =>
|
|
IsNotZero64(
|
|
_module.AddInstruction(
|
|
SpirvOp.BitwiseAnd,
|
|
_ulongType,
|
|
mask,
|
|
CurrentLaneBit()));
|
|
|
|
// In wave32 a lane mask fills its register alone and the next one keeps its value.
|
|
// That includes VCC and EXEC: compilers use VCC_HI (s107) as an ordinary SGPR.
|
|
private void StoreWaveMask(uint register, uint condition)
|
|
{
|
|
if (_waveLaneCount == 32)
|
|
{
|
|
StoreS(register, Narrow(BooleanToWaveMask(condition)));
|
|
return;
|
|
}
|
|
|
|
if (_emulateWave64 &&
|
|
_subgroupInvocationIdInput != 0 &&
|
|
_emittingPc is { } pc &&
|
|
HalfMaskPlan() is { } plan &&
|
|
plan.HalfMaskWrites.TryGetValue(pc, out var halfRegister) &&
|
|
halfRegister == register)
|
|
{
|
|
StoreS64(register, BooleanToHalfWaveMask(condition));
|
|
return;
|
|
}
|
|
|
|
StoreS64(register, BooleanToWaveMask(condition));
|
|
}
|
|
|
|
private void EmitExecConditional(Action emit)
|
|
{
|
|
var active = Load(_boolType, _exec);
|
|
EmitConditional(active, emit);
|
|
}
|
|
|
|
private void EmitConditional(uint condition, Action emit)
|
|
{
|
|
var activeLabel = _module.AllocateId();
|
|
var mergeLabel = _module.AllocateId();
|
|
_module.AddStatement(SpirvOp.SelectionMerge, mergeLabel, 0);
|
|
_module.AddStatement(
|
|
SpirvOp.BranchConditional,
|
|
condition,
|
|
activeLabel,
|
|
mergeLabel);
|
|
_module.AddLabel(activeLabel);
|
|
emit();
|
|
_module.AddStatement(SpirvOp.Branch, mergeLabel);
|
|
_module.AddLabel(mergeLabel);
|
|
}
|
|
|
|
private void EmitConditional(uint condition, Action whenTrue, Action whenFalse)
|
|
{
|
|
var trueLabel = _module.AllocateId();
|
|
var falseLabel = _module.AllocateId();
|
|
var mergeLabel = _module.AllocateId();
|
|
_module.AddStatement(SpirvOp.SelectionMerge, mergeLabel, 0);
|
|
_module.AddStatement(SpirvOp.BranchConditional, condition, trueLabel, falseLabel);
|
|
_module.AddLabel(trueLabel);
|
|
whenTrue();
|
|
_module.AddStatement(SpirvOp.Branch, mergeLabel);
|
|
_module.AddLabel(falseLabel);
|
|
whenFalse();
|
|
_module.AddStatement(SpirvOp.Branch, mergeLabel);
|
|
_module.AddLabel(mergeLabel);
|
|
}
|
|
|
|
// Only instructions that address LDS memory need the array. ds_swizzle/ds_bpermute move data
|
|
// between lanes and GDS instructions use the global data share, so a graphics stage using
|
|
// just those must not get an 8 KiB zero-initialized per-invocation array: Metal pays for it
|
|
// in compile time (seconds for a large pixel shader) and in private memory per pixel.
|
|
private bool UsesLds() =>
|
|
_request.Program.Instructions.Any(static instruction =>
|
|
instruction.Control is Gen5DataShareControl { Gds: false } &&
|
|
instruction.Opcode is not ("DsSwizzleB32" or "DsBpermuteB32")) ||
|
|
_request.Memory.Entries.Any(static memory =>
|
|
memory.AddressSpace is FlatAddressSpace.Shared or FlatAddressSpace.SharedOrPrivate);
|
|
|
|
private bool UsesSubgroupShuffle() =>
|
|
_request.Program.Instructions.Any(instruction =>
|
|
instruction.Control is Gen5DppControl or Gen5Dpp8Control ||
|
|
instruction.Opcode is "VPermlane16B32" or "VPermlanex16B32" or "VReadlaneB32" or
|
|
"DsAppend" or "DsConsume" or "DsSwizzleB32" or "DsBpermuteB32");
|
|
|
|
private bool UsesSubgroupBroadcast() =>
|
|
_request.Program.Instructions.Any(instruction =>
|
|
instruction.Opcode == "VReadfirstlaneB32");
|
|
|
|
private bool UsesWaveControl() =>
|
|
_request.Program.Instructions.Any(instruction =>
|
|
instruction.Opcode.Contains("Saveexec", StringComparison.Ordinal) ||
|
|
instruction.Opcode.StartsWith("SCbranchExec", StringComparison.Ordinal) ||
|
|
instruction.Opcode.StartsWith("SCbranchVcc", StringComparison.Ordinal) ||
|
|
instruction.Opcode.StartsWith("VCmp", StringComparison.Ordinal) ||
|
|
instruction.Sources.Any(IsWaveMaskOperand) ||
|
|
instruction.Destinations.Any(IsWaveMaskOperand));
|
|
|
|
private bool UsesSubgroupOperations() =>
|
|
_enableGraphicsSubgroupOperations &&
|
|
(UsesSubgroupShuffle() ||
|
|
UsesSubgroupBroadcast() ||
|
|
UsesWaveControl() ||
|
|
_request.Program.Instructions.Any(static instruction =>
|
|
instruction.Opcode is "VMbcntLoU32B32" or "VMbcntHiU32B32" or "DsWriteAddtidB32" or "DsReadAddtidB32"));
|
|
|
|
private static bool IsWaveMaskOperand(Gen5Operand operand) =>
|
|
operand.Kind == Gen5OperandKind.ScalarRegister &&
|
|
operand.Value is 106 or 107 or 126 or 127;
|
|
|
|
private static bool TryGetVectorDestination(
|
|
Gen5ShaderInstruction instruction,
|
|
out uint destination)
|
|
{
|
|
if (instruction.Destinations.Count != 0 &&
|
|
instruction.Destinations[0].Kind == Gen5OperandKind.VectorRegister)
|
|
{
|
|
destination = instruction.Destinations[0].Value;
|
|
return true;
|
|
}
|
|
|
|
destination = 0;
|
|
return false;
|
|
}
|
|
|
|
private static bool IsBranch(string opcode) =>
|
|
opcode == "SBranch" ||
|
|
opcode.StartsWith("SCbranch", StringComparison.Ordinal);
|
|
|
|
private static bool TryGetBranchTargetPc(
|
|
Gen5ShaderInstruction instruction,
|
|
out uint targetPc)
|
|
{
|
|
targetPc = 0;
|
|
if (instruction.Encoding != Gen5ShaderEncoding.Sopp ||
|
|
instruction.Words.Count == 0)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
var offset = unchecked((short)(instruction.Words[0] & 0xFFFF));
|
|
var nextPc = (long)instruction.Pc +
|
|
(instruction.Words.Count * sizeof(uint));
|
|
var target = nextPc + (offset * sizeof(uint));
|
|
if (target < 0 || target > uint.MaxValue)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
targetPc = (uint)target;
|
|
return true;
|
|
}
|
|
|
|
private static IReadOnlyList<ShaderBlock> BuildBasicBlocks(
|
|
IReadOnlyList<Gen5ShaderInstruction> instructions)
|
|
{
|
|
if (instructions.Count == 0)
|
|
{
|
|
return [];
|
|
}
|
|
|
|
var leaders = new SortedSet<uint> { instructions[0].Pc };
|
|
for (var index = 0; index < instructions.Count; index++)
|
|
{
|
|
var instruction = instructions[index];
|
|
if (IsBranch(instruction.Opcode) &&
|
|
TryGetBranchTargetPc(instruction, out var targetPc))
|
|
{
|
|
leaders.Add(targetPc);
|
|
}
|
|
|
|
if ((IsBranch(instruction.Opcode) || instruction.Opcode == "SEndpgm") &&
|
|
index + 1 < instructions.Count)
|
|
{
|
|
leaders.Add(instructions[index + 1].Pc);
|
|
}
|
|
}
|
|
|
|
var starts = leaders
|
|
.Where(pc => instructions.Any(instruction => instruction.Pc == pc))
|
|
.ToArray();
|
|
var blocks = new List<ShaderBlock>(starts.Length);
|
|
for (var index = 0; index < starts.Length; index++)
|
|
{
|
|
var startIndex = FindInstructionIndex(instructions, starts[index]);
|
|
var endIndex = index + 1 < starts.Length
|
|
? FindInstructionIndex(instructions, starts[index + 1])
|
|
: instructions.Count;
|
|
if (startIndex >= 0 && endIndex > startIndex)
|
|
{
|
|
blocks.Add(new ShaderBlock(starts[index], startIndex, endIndex));
|
|
}
|
|
}
|
|
|
|
return blocks;
|
|
}
|
|
|
|
private static int FindInstructionIndex(
|
|
IReadOnlyList<Gen5ShaderInstruction> instructions,
|
|
uint pc)
|
|
{
|
|
for (var index = 0; index < instructions.Count; index++)
|
|
{
|
|
if (instructions[index].Pc == pc)
|
|
{
|
|
return index;
|
|
}
|
|
}
|
|
|
|
return -1;
|
|
}
|
|
|
|
private static bool TryFindBlock(
|
|
IReadOnlyList<ShaderBlock> blocks,
|
|
uint pc,
|
|
out int block)
|
|
{
|
|
for (var index = 0; index < blocks.Count; index++)
|
|
{
|
|
if (blocks[index].StartPc == pc)
|
|
{
|
|
block = index;
|
|
return true;
|
|
}
|
|
}
|
|
|
|
block = -1;
|
|
return false;
|
|
}
|
|
|
|
private readonly record struct ShaderBlock(
|
|
uint StartPc,
|
|
int StartIndex,
|
|
int EndIndex);
|
|
}
|
|
}
|