From 36acf4a40ea3e4cd440d302a14f6f75cfbe8778b Mon Sep 17 00:00:00 2001 From: Berk <12572227+par274@users.noreply.github.com> Date: Thu, 1 Oct 2026 21:15:55 +0300 Subject: [PATCH] Des dcc fixes (#1000) * [SpirvTranslator] Take high products from extended multiplies Co-authored-by: TarkusR <90155059+TarkusR@users.noreply.github.com> * [SpirvTranslator] Keep shader flags in uint variables Co-authored-by: TarkusR <90155059+TarkusR@users.noreply.github.com> * [SpirvTranslator] Synchronize shared memory phases in multi block shaders Co-authored-by: TarkusR <90155059+TarkusR@users.noreply.github.com> * [GPU] Merge device address ranges before buffers are bound Co-authored-by: TarkusR <90155059+TarkusR@users.noreply.github.com> * [GPU] Track per-slice guest DCC clear metadata * [GPU] Materialize guest DCC clears in color and sampled images * [GPU] Preserve color targets without pixel shader exports * [ShaderCompiler] Track wave-indexed image descriptor waterfalls * [ShaderCompiler] Analyze wave64 half masks and shared-memory hazards * [ShaderCompiler] Pass guest LDS size to SPIR-V compilation * [SpirvTranslator] Synchronize wave64 half masks through LDS exchange --------- Co-authored-by: TarkusR <90155059+TarkusR@users.noreply.github.com> --- .../Gpu/Images/GuestImageCache.Metadata.cs | 64 ++ .../Gpu/Images/ImageDescription.cs | 18 + .../ImageRequestBuilders.ColorTarget.cs | 20 +- .../Gpu/Pipelines/ShaderPipelineCache.cs | 2 +- .../Gpu/Pipelines/ShaderProgramCache.cs | 1 + .../Gpu/Rendering/RenderExecutor.Targets.cs | 2 +- .../VulkanVideoPresenter.CachedImages.cs | 143 +++-- .../VulkanVideoPresenter.Descriptors.cs | 16 + .../VulkanVideoPresenter.Rendering.cs | 35 +- .../Gen5SpirvTranslator.Alu.cs | 212 ++----- .../Gen5SpirvTranslator.Resources.cs | 5 +- .../Gen5SpirvTranslator.cs | 404 +++++++++---- .../Ir/Gen5Wave64HalfMaskAnalysis.cs | 565 ++++++++++++++++++ .../Resources/ResourceTracker.DenseImages.cs | 147 ++++- .../Resources/ScalarValueGraph.Builder.cs | 15 +- .../Resources/ShaderResourcePlan.cs | 3 +- .../ShaderCompileRequest.cs | 1 + .../Images/GuestImageCacheTests.Metadata.cs | 30 + .../Gpu/Images/ImageDescriptionTests.cs | 22 + .../Gpu/Pipelines/ShaderPipelineCacheTests.cs | 20 + .../Gpu/Rendering/RenderExecutorStateTests.cs | 13 + .../Gpu/Vulkan/MultiplyHighDeviceTests.cs | 69 +++ .../Gpu/Vulkan/Wave64HalfMaskDeviceTests.cs | 269 +++++++++ .../VideoOut/RenderHostDeviceTests.cs | 40 ++ .../Gen5MultiplyHighTests.cs | 68 +++ .../Gen5SharedMemoryBarrierTests.cs | 14 +- .../Gen5Wave64HalfMaskAnalysisTests.cs | 221 +++++++ .../Resources/DirectImageTableTests.cs | 126 +++- 28 files changed, 2138 insertions(+), 407 deletions(-) create mode 100644 src/SharpEmu.ShaderCompiler/Ir/Gen5Wave64HalfMaskAnalysis.cs create mode 100644 tests/SharpEmu.Libs.Tests/Gpu/Vulkan/MultiplyHighDeviceTests.cs create mode 100644 tests/SharpEmu.Libs.Tests/Gpu/Vulkan/Wave64HalfMaskDeviceTests.cs create mode 100644 tests/SharpEmu.ShaderCompiler.Tests/Gen5MultiplyHighTests.cs create mode 100644 tests/SharpEmu.ShaderCompiler.Tests/Gen5Wave64HalfMaskAnalysisTests.cs diff --git a/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.Metadata.cs b/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.Metadata.cs index 1040652..8d97137 100644 --- a/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.Metadata.cs +++ b/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.Metadata.cs @@ -84,6 +84,70 @@ public sealed partial class GuestImageCache return false; } + public static bool IsDccClearCode(byte code) => code is 0x00 or 0x20 or 0x40 or 0x80 or 0xc0; + + public bool TryReadGuestDccClear(ulong metadataAddress, ulong sliceSize, uint slice, out ulong sliceAddress, out byte code) + { + sliceAddress = 0; + code = 0; + if (metadataAddress == 0 || sliceSize == 0 || sliceSize > int.MaxValue || + (ulong)slice > (ulong.MaxValue - metadataAddress) / sliceSize) + { + return false; + } + + var address = metadataAddress + (ulong)slice * sliceSize; + if (!IsValidRange(address, sliceSize) || _bufferCache.HasGpuDirtyBytes(address, sliceSize)) + { + return false; + } + + Span first = stackalloc byte[1]; + if (!_backing.TryReadBacking(address, first) || !IsDccClearCode(first[0])) + { + return false; + } + + var buffer = System.Buffers.ArrayPool.Shared.Rent((int)Math.Min(sliceSize, 64 * 1024)); + try + { + for (ulong position = 0; position < sliceSize;) + { + var span = buffer.AsSpan(0, (int)Math.Min(sliceSize - position, (ulong)buffer.Length)); + if (!_backing.TryReadBacking(address + position, span) || span.IndexOfAnyExcept(first[0]) >= 0) + { + return false; + } + + position += (ulong)span.Length; + } + } + finally + { + System.Buffers.ArrayPool.Shared.Return(buffer); + } + + sliceAddress = address; + code = first[0]; + return true; + } + + public void SynchronizeGuestDccMetadata(ulong metadataAddress, ulong sliceSize, uint baseLayer, uint layerCount) + { + if (metadataAddress == 0 || sliceSize == 0 || layerCount == 0 || + (ulong)baseLayer + layerCount > (ulong.MaxValue - metadataAddress) / sliceSize) + { + return; + } + + var address = metadataAddress + (ulong)baseLayer * sliceSize; + var size = (ulong)layerCount * sliceSize; + if (IsValidRange(address, size) && _bufferCache.HasGpuDirtyBytes(address, size)) + { + _ = _bufferCache.TrySynchronizeCpuRead(address, size); + } + } + public bool SetMetadataSlice(ulong address, uint slice, bool isClear) { using var held = _lock.Hold(); diff --git a/src/SharpEmu.Libs/Gpu/Images/ImageDescription.cs b/src/SharpEmu.Libs/Gpu/Images/ImageDescription.cs index ce58daa..7bcea93 100644 --- a/src/SharpEmu.Libs/Gpu/Images/ImageDescription.cs +++ b/src/SharpEmu.Libs/Gpu/Images/ImageDescription.cs @@ -245,6 +245,24 @@ public struct ImageDescription public readonly bool IsTiledRenderTarget => TileMode == GuestTileMode.RenderTarget || IsSupportedStandard64RenderTarget; + public readonly ulong DccSliceSize + { + get + { + const ulong DccBlockBytes = 4096; + if (Metadata.Kind != MetadataKind.Dcc || TileMode != GuestTileMode.RenderTarget || Resources.Levels != 1 || Samples != 1 || + Extent.Width == 0 || Extent.Height == 0 || BytesPerBlock is 0 or > 16 || !System.Numerics.BitOperations.IsPow2(BytesPerBlock)) + { + return 0; + } + + var coverageBits = 20 - System.Numerics.BitOperations.TrailingZeroCount(BytesPerBlock); + var blockWidth = 1UL << ((coverageBits + 1) / 2); + var blockHeight = 1UL << (coverageBits / 2); + return (Extent.Width + blockWidth - 1) / blockWidth * ((Extent.Height + blockHeight - 1) / blockHeight) * DccBlockBytes; + } + } + public static bool IsSupportedDisplayRenderTargetTileMode(GuestTileMode tileMode) => tileMode == GuestTileMode.RenderTarget; public static bool CanUseDisplayNativeWithoutUpload(DisplayCompression compression, bool renderTarget, bool gpuModified, bool guestModified) => diff --git a/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.ColorTarget.cs b/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.ColorTarget.cs index 1be295f..783bef1 100644 --- a/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.ColorTarget.cs +++ b/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.ColorTarget.cs @@ -46,18 +46,20 @@ public static partial class ImageRequestBuilders { ClearColorValue value = default; var supported = hasDcc && PackedClearValue.TryDecodeColor(format, packedClear, out value); - var fixedSupported = hasDcc && format switch - { - Format.R8Unorm or Format.R8G8Unorm or Format.R8G8B8A8Unorm or Format.R8G8B8A8Srgb or Format.B8G8R8A8Unorm or Format.B8G8R8A8Srgb or - Format.A2B10G10R10UnormPack32 or Format.A2R10G10B10UnormPack32 or Format.R5G6B5UnormPack16 or Format.A1R5G5B5UnormPack16 or - Format.R4G4B4A4UnormPack16 or Format.R16Unorm or Format.R16G16Unorm or Format.R16G16B16A16Unorm or Format.R16Sfloat or - Format.R16G16Sfloat or Format.R16G16B16A16Sfloat or Format.R32Sfloat or Format.R32G32Sfloat or Format.R32G32B32A32Sfloat or - Format.B10G11R11UfloatPack32 => true, - _ => false, - }; + var fixedSupported = hasDcc && SupportsDccFixedClear(format); return (supported, fixedSupported, supported ? value : default); } + public static bool SupportsDccFixedClear(Format format) => format switch + { + Format.R8Unorm or Format.R8G8Unorm or Format.R8G8B8A8Unorm or Format.R8G8B8A8Srgb or Format.B8G8R8A8Unorm or Format.B8G8R8A8Srgb or + Format.A2B10G10R10UnormPack32 or Format.A2R10G10B10UnormPack32 or Format.R5G6B5UnormPack16 or Format.A1R5G5B5UnormPack16 or + Format.R4G4B4A4UnormPack16 or Format.R16Unorm or Format.R16G16Unorm or Format.R16G16B16A16Unorm or Format.R16Sfloat or + Format.R16G16Sfloat or Format.R16G16B16A16Sfloat or Format.R32Sfloat or Format.R32G32Sfloat or Format.R32G32B32A32Sfloat or + Format.B10G11R11UfloatPack32 => true, + _ => false, + }; + // Builds the request for a bound color target. Null when the slot carries no target. public static ColorTargetResolution? ColorTarget(in ColorTargetWords words, uint targetMask, uint drawLayerOffset, bool ignoreTargetMask) { diff --git a/src/SharpEmu.Libs/Gpu/Pipelines/ShaderPipelineCache.cs b/src/SharpEmu.Libs/Gpu/Pipelines/ShaderPipelineCache.cs index 2c7e095..fade42a 100644 --- a/src/SharpEmu.Libs/Gpu/Pipelines/ShaderPipelineCache.cs +++ b/src/SharpEmu.Libs/Gpu/Pipelines/ShaderPipelineCache.cs @@ -455,7 +455,7 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider // A target the pixel program never exports keeps its contents, as on hardware; the // host output would otherwise write an undefined value (e.g. depth-only passes that // leave a color target bound and export only to the null target). - var exported = pixelStage is null || ((pixelStage.PixelColorExportMasks >> (int)(color.Slot * 4)) & 0xFu) != 0; + var exported = pixelStage is not null && ((pixelStage.PixelColorExportMasks >> (int)(color.Slot * 4)) & 0xFu) != 0; var colorMask = exported ? color.Resolution.ExportMapping.ApplyMask(context.RenderTargetMaskForSlot(color.Slot)) : 0; parameters.SetColorMask(index, colorMask); if (RenderTrace.Enabled && RenderTrace.Pipeline()) diff --git a/src/SharpEmu.Libs/Gpu/Pipelines/ShaderProgramCache.cs b/src/SharpEmu.Libs/Gpu/Pipelines/ShaderProgramCache.cs index acbb89f..3492391 100644 --- a/src/SharpEmu.Libs/Gpu/Pipelines/ShaderProgramCache.cs +++ b/src/SharpEmu.Libs/Gpu/Pipelines/ShaderProgramCache.cs @@ -601,6 +601,7 @@ internal sealed class ShaderProgramCache LocalSizeX = Math.Max(info.ThreadsX, 1), LocalSizeY = Math.Max(info.ThreadsY, 1), LocalSizeZ = Math.Max(info.ThreadsZ, 1), + LocalDataShareDwords = info.LocalDataShareDwords, }; } } diff --git a/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Targets.cs b/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Targets.cs index 9e34021..57d0cd2 100644 --- a/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Targets.cs +++ b/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Targets.cs @@ -184,7 +184,7 @@ public sealed partial class RenderExecutor return; } - var exportMasks = pixelProgram?.PixelColorExportMasks ?? uint.MaxValue; + var exportMasks = pixelProgram?.PixelColorExportMasks ?? 0u; var kept = 0u; for (var i = 0; i < state.ColorCount; i++) { diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CachedImages.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CachedImages.cs index d7fe66e..dd2a625 100644 --- a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CachedImages.cs +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CachedImages.cs @@ -7,6 +7,7 @@ using System.Collections.Concurrent; using SharpEmu.Libs.Gpu; using SharpEmu.Libs.Gpu.Buffers; using SharpEmu.Libs.Gpu.Images; +using SharpEmu.Libs.Gpu.Rendering; using SharpEmu.Libs.Gpu.Scheduling; using SharpEmu.Libs.Gpu.Vulkan; using Silk.NET.Vulkan; @@ -287,53 +288,14 @@ internal static unsafe partial class VulkanVideoPresenter var metadataAddress = target.Request.Description.Metadata.Range.Address; if (!_imageCache.IsMetadataCleared(metadataAddress, view.BaseLayer, out var metadataValue)) { - return false; + return ResolveGuestDccAttachmentClear(target, out clearValue); } var resolution = target.Resolution; - switch ((byte)metadataValue) + if (!TryDecodeDccClear((byte)metadataValue, resolution.MetadataClearSupported, resolution.MetadataFixedClearSupported, + resolution.ColorClearValue, out clearValue)) { - case 0x00: - break; - case 0x20: - if (!resolution.MetadataClearSupported) - { - return false; - } - - clearValue = resolution.ColorClearValue; - break; - case 0x40: - if (!resolution.MetadataFixedClearSupported) - { - return false; - } - - clearValue.Float32_3 = 1f; - break; - case 0x80: - if (!resolution.MetadataFixedClearSupported) - { - return false; - } - - clearValue.Float32_0 = 1f; - clearValue.Float32_1 = 1f; - clearValue.Float32_2 = 1f; - break; - case 0xc0: - if (!resolution.MetadataFixedClearSupported) - { - return false; - } - - clearValue.Float32_0 = 1f; - clearValue.Float32_1 = 1f; - clearValue.Float32_2 = 1f; - clearValue.Float32_3 = 1f; - break; - default: - return false; + return false; } for (uint layer = 1; layer < view.LayerCount; layer++) @@ -353,9 +315,104 @@ internal static unsafe partial class VulkanVideoPresenter } } + ConsumeGuestDccClears(target.Request.Description, view.BaseLayer, view.LayerCount); return true; } + private bool ResolveGuestDccAttachmentClear(ColorAttachment target, out ClearColorValue clearValue) + { + clearValue = default; + var description = target.Request.Description; + var sliceSize = description.DccSliceSize; + var view = target.Request.View; + if (sliceSize == 0 || view.LayerCount == 0) + { + return false; + } + + var metadataAddress = description.Metadata.Range.Address; + var slices = new ulong[view.LayerCount]; + byte clearCode = 0; + for (uint layer = 0; layer < view.LayerCount; layer++) + { + if (!_imageCache.TryReadGuestDccClear(metadataAddress, sliceSize, view.BaseLayer + layer, out slices[layer], out var code) || + (layer != 0 && code != clearCode)) + { + return false; + } + + clearCode = code; + } + + var resolution = target.Resolution; + if (!TryDecodeDccClear(clearCode, resolution.MetadataClearSupported, resolution.MetadataFixedClearSupported, + resolution.ColorClearValue, out clearValue)) + { + return false; + } + + foreach (var slice in slices) + { + _bufferCache.FillBuffer(slice, sliceSize, uint.MaxValue, false); + } + + if (RenderTrace.Enabled && RenderTrace.MetadataClear()) + { + RenderTrace.Write( + $"Materialized a guest DCC clear on a color target: metadata=0x{metadataAddress:X16} code=0x{clearCode:X2} " + + $"layers={view.BaseLayer}+{view.LayerCount} slice=0x{sliceSize:X} format={description.PixelFormat}"); + } + + return true; + } + + private void ConsumeGuestDccClears(in ImageDescription description, uint baseLayer, uint layerCount) + { + var sliceSize = description.DccSliceSize; + if (sliceSize == 0) + { + return; + } + + for (uint layer = 0; layer < layerCount; layer++) + { + if (_imageCache.TryReadGuestDccClear(description.Metadata.Range.Address, sliceSize, baseLayer + layer, out var slice, out _)) + { + _bufferCache.FillBuffer(slice, sliceSize, uint.MaxValue, false); + } + } + } + + private static bool TryDecodeDccClear(byte code, bool registerClearSupported, bool fixedClearSupported, ClearColorValue registerClear, + out ClearColorValue clearValue) + { + clearValue = default; + switch (code) + { + case 0x00: + return true; + case 0x20: + clearValue = registerClear; + return registerClearSupported; + case 0x40: + clearValue.Float32_3 = 1f; + return fixedClearSupported; + case 0x80: + clearValue.Float32_0 = 1f; + clearValue.Float32_1 = 1f; + clearValue.Float32_2 = 1f; + return fixedClearSupported; + case 0xc0: + clearValue.Float32_0 = 1f; + clearValue.Float32_1 = 1f; + clearValue.Float32_2 = 1f; + clearValue.Float32_3 = 1f; + return fixedClearSupported; + default: + return false; + } + } + private DepthAttachment? DiscoverDepthTarget(GuestDepthTarget target) { if (target.Registers is not { } words || ImageRequestBuilders.DepthTarget(words, _deviceInfo) is not { } resolution) diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Descriptors.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Descriptors.cs index 01e29c9..1b3d85d 100644 --- a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Descriptors.cs +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Descriptors.cs @@ -285,6 +285,7 @@ internal static unsafe partial class VulkanVideoPresenter descriptors.GlobalDataShare = new BufferView(_bufferCache.GdsBuffer.Handle, 0, Vk.WholeSize); } + FindDeviceAddressBuffers(prepared); FindBuffers(prepared); preparation.Stages.Add(prepared); ValidateDrawImageTypes(prepared); @@ -322,6 +323,21 @@ internal static unsafe partial class VulkanVideoPresenter } } + private void FindDeviceAddressBuffers(PreparedStageBindings prepared) + { + foreach (var range in prepared.Stage.Resources.DeviceAddressRanges) + { + if (!range.Planned || range.Size == 0 || + range.Base >= PageOwnerTable.AddressSpaceSize || range.Size > PageOwnerTable.AddressSpaceSize - range.Base || + (!range.Written && !_guestMemory.CanRead(range.Base, 1))) + { + continue; + } + + _ = _bufferCache.FindBuffer(range.Base, ClampMappedSize(range.Base, range.Size)); + } + } + // The cache buffer each descriptor's range lives in; a null descriptor has none. private void FindBuffers(PreparedStageBindings prepared) { diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Rendering.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Rendering.cs index 8c924f5..7986e6d 100644 --- a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Rendering.cs +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Rendering.cs @@ -236,6 +236,12 @@ internal static unsafe partial class VulkanVideoPresenter public ResourceSlotIdentifier FindImage(ref ImageRequest request, bool exactFormat) { _ = BeginBatchedGuestCommands(); + if (request.Role == ImageRole.ColorTarget && request.Description.DccSliceSize is var sliceSize and not 0) + { + _imageCache.SynchronizeGuestDccMetadata(request.Description.Metadata.Range.Address, sliceSize, + request.View.BaseLayer, request.View.LayerCount); + } + return _imageCache.FindImage(ref request, exactFormat); } @@ -534,8 +540,8 @@ internal static unsafe partial class VulkanVideoPresenter // A DCC fast clear stays pending until the surface binds as a color target. A shader that // samples or writes the surface first must see the cleared contents, and the later bind must // not clear over its writes: Astro Bot's save-slot cards were drawn by compute into a - // fast-cleared target, then wiped black by the deferred clear. Only the zero clear code is - // materialized here; the register clear color is known only when the surface is a target. + // fast-cleared target, then wiped black by the deferred clear. The register clear color is + // known only when the surface is a target. private void RecordSampledColorMetadataClears(TextureResource[] bindings) { const byte DccClearToZero = 0x00; @@ -547,10 +553,17 @@ internal static unsafe partial class VulkanVideoPresenter } var metadataAddress = image.Description.Metadata.Range.Address; + var sliceSize = image.Description.DccSliceSize; + var fixedClearSupported = ImageRequestBuilders.SupportsDccFixedClear(image.Description.PixelFormat); var view = binding.Request.View; for (var layer = view.BaseLayer; layer < view.BaseLayer + view.LayerCount; layer++) { - if (!_imageCache.IsMetadataCleared(metadataAddress, layer, out var metadataValue) || (byte)metadataValue != DccClearToZero) + var tracked = _imageCache.IsMetadataCleared(metadataAddress, layer, out var metadataValue) && (byte)metadataValue == DccClearToZero; + var clearValue = default(ClearColorValue); + ulong guestSlice = 0; + if (!tracked && + (sliceSize == 0 || !_imageCache.TryReadGuestDccClear(metadataAddress, sliceSize, layer, out guestSlice, out var code) || + !TryDecodeDccClear(code, false, fixedClearSupported, default, out clearValue))) { continue; } @@ -560,12 +573,26 @@ internal static unsafe partial class VulkanVideoPresenter var range = new SubresourceRange(0, 1, layer, 1); image.Transition(ImageLayout.TransferDstOptimal, AccessFlags.TransferWriteBit, range, command); var vkRange = new ImageSubresourceRange(ImageAspectFlags.ColorBit, 0, 1, layer, 1); - var clearValue = default(ClearColorValue); _vk.CmdClearColorImage(command, image.Backing.Handle, ImageLayout.TransferDstOptimal, &clearValue, 1, &vkRange); + if (!tracked) + { + _bufferCache.FillBuffer(guestSlice, sliceSize, uint.MaxValue, false); + if (RenderTrace.Enabled && RenderTrace.MetadataClear()) + { + RenderTrace.Write( + $"Materialized a guest DCC clear on a sampled image: metadata=0x{metadataAddress:X16} layer={layer} " + + $"slice=0x{sliceSize:X} format={image.Description.PixelFormat}"); + } + + continue; + } + if (!_imageCache.SetMetadataSlice(metadataAddress, layer, false)) { throw SubmissionScheduler.Fatal($"The DCC clear state could not be consumed: metadata=0x{metadataAddress:X16} layer={layer}."); } + + ConsumeGuestDccClears(image.Description, layer, 1); } } } diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Alu.cs b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Alu.cs index 674f205..763d49b 100644 --- a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Alu.cs +++ b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Alu.cs @@ -840,52 +840,17 @@ public static partial class Gen5SpirvTranslator right = BitwiseAnd(right, UInt(0x00FF_FFFF)); } - var wideLeft = _module.AddInstruction( - SpirvOp.UConvert, - _ulongType, - left); - var wideRight = _module.AddInstruction( - SpirvOp.UConvert, - _ulongType, - right); - var product = _module.AddInstruction( - SpirvOp.IMul, - _ulongType, - wideLeft, - wideRight); - result = _module.AddInstruction( - SpirvOp.UConvert, - _uintType, - ShiftRightLogical64( - product, - _module.Constant64(_ulongType, 32))); + result = MultiplyExtended(left, right, signed: false).High; break; } case "VMulHiI32": { - var wideLeft = _module.AddInstruction( - SpirvOp.SConvert, - _longType, - Bitcast(_intType, GetRawSource(instruction, 0))); - var wideRight = _module.AddInstruction( - SpirvOp.SConvert, - _longType, - Bitcast(_intType, GetRawSource(instruction, 1))); - var product = _module.AddInstruction( - SpirvOp.IMul, - _longType, - wideLeft, - wideRight); result = Bitcast( _uintType, - _module.AddInstruction( - SpirvOp.SConvert, - _intType, - _module.AddInstruction( - SpirvOp.ShiftRightArithmetic, - _longType, - product, - _module.Constant64(_longType, 32)))); + MultiplyExtended( + Bitcast(_intType, GetRawSource(instruction, 0)), + Bitcast(_intType, GetRawSource(instruction, 1)), + signed: true).High); break; } case "VBcntU32B32": @@ -1319,19 +1284,8 @@ public static partial class Gen5SpirvTranslator // sources are 32-bit factors; the third is a 64-bit addend // held in a VGPR or SGPR pair. Its SDST receives the carry // mask for the unsigned 64-bit addition. - var wideLeft = _module.AddInstruction( - SpirvOp.UConvert, - _ulongType, - GetRawSource(instruction, 0)); - var wideRight = _module.AddInstruction( - SpirvOp.UConvert, - _ulongType, - GetRawSource(instruction, 1)); - var product = _module.AddInstruction( - SpirvOp.IMul, - _ulongType, - wideLeft, - wideRight); + var (productLow, productHigh) = MultiplyExtended(GetRawSource(instruction, 0), GetRawSource(instruction, 1), signed: false); + var product = Pair64(productLow, productHigh); var addend = GetRawSource64(instruction, 2); var wideResult = _module.AddInstruction( SpirvOp.IAdd, @@ -3034,47 +2988,13 @@ public static partial class Gen5SpirvTranslator right); break; case "SMulHiI32": - { - var wideLeft = _module.AddInstruction( - SpirvOp.SConvert, - _longType, - Bitcast(_intType, left)); - var wideRight = _module.AddInstruction( - SpirvOp.SConvert, - _longType, - Bitcast(_intType, right)); - var product = _module.AddInstruction( - SpirvOp.IMul, - _longType, - wideLeft, - wideRight); result = Bitcast( _uintType, - _module.AddInstruction( - SpirvOp.SConvert, - _intType, - _module.AddInstruction( - SpirvOp.ShiftRightArithmetic, - _longType, - product, - _module.Constant64(_longType, 32)))); + MultiplyExtended(Bitcast(_intType, left), Bitcast(_intType, right), signed: true).High); break; - } case "SMulHiU32": - { - var product = _module.AddInstruction( - SpirvOp.IMul, - _ulongType, - _module.AddInstruction(SpirvOp.UConvert, _ulongType, left), - _module.AddInstruction(SpirvOp.UConvert, _ulongType, right)); - result = _module.AddInstruction( - SpirvOp.UConvert, - _uintType, - ShiftRightLogical64( - product, - _module.Constant64(_ulongType, 32))); + result = MultiplyExtended(left, right, signed: false).High; break; - } case "SAndB32": result = BitwiseAnd(left, right); Store(_scc, IsNotZero(result)); @@ -3448,7 +3368,9 @@ public static partial class Gen5SpirvTranslator var left = GetRawSource64(instruction, 0); if (instruction.Opcode.EndsWith("SaveexecB64", StringComparison.Ordinal)) { - var oldExec = BooleanToWaveMask(Load(_boolType, _exec)); + var oldExec = _emulateWave64 && _subgroupInvocationIdInput != 0 + ? LoadS64(126) + : BooleanToWaveMask(Load(_boolType, _exec)); var notLeft = _module.AddInstruction(SpirvOp.Not, _ulongType, left); var newExec = instruction.Opcode switch { @@ -3821,14 +3743,14 @@ public static partial class Gen5SpirvTranslator _module.AddInstruction( SpirvOp.Select, _uintType, - LogicalNot(SubgroupAny(Load(_boolType, _vcc))), + LogicalNot(WaveMaskAny(106, _vcc)), UInt(1), UInt(0)), Gen5OperandKind.EncodedConstant when operand.Value == 252 => _module.AddInstruction( SpirvOp.Select, _uintType, - LogicalNot(SubgroupAny(Load(_boolType, _exec))), + LogicalNot(WaveMaskAny(126, _exec)), UInt(1), UInt(0)), Gen5OperandKind.EncodedConstant when operand.Value == 253 => @@ -4905,76 +4827,48 @@ public static partial class Gen5SpirvTranslator // An emulated wave64 spans two host subgroups, so a subgroup broadcast cannot reach // the other half. The selected guest lane publishes through workgroup scratch, like - // BroadcastFirstWave64Active; the trailing barrier keeps the next use from racing. - private uint BroadcastWave64Lane(uint value, uint lane) - { - EmitConditional( - _module.AddInstruction(SpirvOp.IEqual, _boolType, GuestWaveLane(), lane), - () => Store(WaveBroadcastScratchPointer(), value)); - EmitWave64Barrier(); - var result = Load(_uintType, WaveBroadcastScratchPointer()); - EmitWave64Barrier(); - return result; - } + // BroadcastFirstWave64Active. + private uint BroadcastWave64Lane(uint value, uint lane) => + ExchangeWave64Value(_module.AddInstruction(SpirvOp.IEqual, _boolType, GuestWaveLane(), lane), value); private uint BroadcastFirstWave64Active(uint value) { var lane = GuestWaveLane(); - EmitConditional( - _module.AddInstruction( - SpirvOp.IEqual, - _boolType, - lane, - UInt(0)), - () => Store(WaveBroadcastScratchPointer(), UInt(0))); - EmitWave64Barrier(); - - var activeMask = BooleanToWaveMask(Load(_boolType, _exec)); - var lowMask = _module.AddInstruction( - SpirvOp.UConvert, - _uintType, - activeMask); - var highMask = _module.AddInstruction( - SpirvOp.UConvert, - _uintType, - ShiftRightLogical64( - activeMask, - _module.Constant64(_ulongType, 32))); - var hasLow = IsNotZero(lowMask); - var hasHigh = IsNotZero(highMask); - var firstLow = Ext(73, _uintType, lowMask); - var firstHigh = IAdd(UInt(32), Ext(73, _uintType, highMask)); - var firstLane = _module.AddInstruction( + var upperHalf = ShiftRightLogical(lane, UInt(5)); + var activeInHalf = OwnHalfBallot(Load(_boolType, _exec)); + var halfHasActive = IsNotZero(activeInHalf); + var firstInHalf = _module.AddInstruction( SpirvOp.Select, _uintType, - hasLow, - firstLow, - _module.AddInstruction( - SpirvOp.Select, - _uintType, - hasHigh, - firstHigh, - UInt(0))); - var isFirst = _module.AddInstruction( - SpirvOp.IEqual, - _boolType, - lane, - firstLane); - EmitConditional( - _module.AddInstruction( - SpirvOp.LogicalAnd, - _boolType, - isFirst, - _module.AddInstruction( - SpirvOp.LogicalOr, - _boolType, - hasLow, - hasHigh)), - () => Store(WaveBroadcastScratchPointer(), value)); + halfHasActive, + Ext(73, _uintType, activeInHalf), + UInt(0)); + var halfBase = BitwiseAnd(Load(_uintType, _subgroupInvocationIdInput), UInt(~31u)); + var halfValue = _module.AddInstruction( + SpirvOp.GroupNonUniformBroadcast, + _uintType, + UInt(3), + value, + IAdd(halfBase, firstInHalf)); + var exchange = BeginWave64Exchange(); + EmitConditional(IsHalfWaveLeader(lane), () => + { + Store(Wave64ExchangePointer(exchange, upperHalf), halfValue); + Store( + Wave64ExchangePointer(exchange, IAdd(UInt(2), upperHalf)), + _module.AddInstruction(SpirvOp.Select, _uintType, halfHasActive, UInt(1), UInt(0))); + }); EmitWave64Barrier(); - var result = Load(_uintType, WaveBroadcastScratchPointer()); - EmitWave64Barrier(); - return result; + var lowerValue = Load(_uintType, Wave64ExchangePointer(exchange, UInt(0))); + var upperValue = Load(_uintType, Wave64ExchangePointer(exchange, UInt(1))); + var lowerActive = IsNotZero(Load(_uintType, Wave64ExchangePointer(exchange, UInt(2)))); + var upperActive = IsNotZero(Load(_uintType, Wave64ExchangePointer(exchange, UInt(3)))); + return _module.AddInstruction( + SpirvOp.Select, + _uintType, + _module.AddInstruction(SpirvOp.LogicalAnd, _boolType, LogicalNot(lowerActive), upperActive), + upperValue, + lowerValue); } private void StoreCarryOut( @@ -5015,13 +4909,7 @@ public static partial class Gen5SpirvTranslator { // The selected guest lane can belong to another host subgroup. // Read it even when the guest execution mask disables that lane. - var isSelectedLane = _module.AddInstruction( - SpirvOp.IEqual, _boolType, GuestWaveLane(), selectedLane); - EmitConditional(isSelectedLane, () => Store(WaveBroadcastScratchPointer(), sourceValue)); - EmitWave64Barrier(); - var broadcast = Load(_uintType, WaveBroadcastScratchPointer()); - EmitWave64Barrier(); - StoreS(destination, broadcast); + StoreS(destination, BroadcastWave64Lane(sourceValue, selectedLane)); } else if (_subgroupInvocationIdInput != 0) { diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Resources.cs b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Resources.cs index 34df42b..43bae54 100644 --- a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Resources.cs +++ b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Resources.cs @@ -1904,10 +1904,7 @@ public static partial class Gen5SpirvTranslator uint broadcast; if (_emulateWave64) { - EmitConditional(isFirstActive, () => Store(WaveBroadcastScratchPointer(), firstValue)); - EmitWave64Barrier(); - broadcast = Load(_uintType, WaveBroadcastScratchPointer()); - EmitWave64Barrier(); + broadcast = ExchangeWave64Value(isFirstActive, firstValue); } else if (_stage == Gen5SpirvStage.Compute || _enableGraphicsSubgroupOperations) { diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.cs b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.cs index 68a3cad..b57576e 100644 --- a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.cs +++ b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.cs @@ -17,6 +17,8 @@ public static partial class Gen5SpirvTranslator // so a smaller array is safe. private const uint PrivateLdsDwordCount = 2048; private const uint RdnaWaveLaneCount = 32; + private const uint Wave64ExchangeSlotCount = 4; + private const uint Wave64ExchangeDwordCount = Wave64ExchangeSlotCount * 2; internal static SpirvImageFormat DecodeStorageImageFormat( uint dataFormat, @@ -142,7 +144,7 @@ public static partial class Gen5SpirvTranslator private uint _uvec4Type; private uint _privateUintPointer; private uint _privateVec2Pointer; - private uint _privateBoolPointer; + private readonly HashSet _flagVariables = []; private uint _runtimeBufferBiases; private uint _scalarRegisters; private uint _vectorRegisters; @@ -185,12 +187,13 @@ public static partial class Gen5SpirvTranslator private uint _localInvocationIndexInput; private uint _workGroupIdInput; private uint _pushConstantUintPointer; - private uint _subgroupSizeInput; private uint _subgroupInvocationIdInput; - private uint _waveMaskScratch; - private uint _waveMaskScratchElementPointer; - private uint _waveBroadcastScratch; - private bool _waveScratchInLds; + private uint _wave64Exchange; + private uint _wave64ExchangeElementPointer; + private uint _wave64ExchangeOffset; + private uint _wave64ExchangeParity; + private Ir.Gen5Wave64HalfMaskPlan? _halfMaskPlan; + private uint? _emittingPc; private uint _glsl; private enum ImageComponentKind @@ -564,8 +567,6 @@ public static partial class Gen5SpirvTranslator _module.TypePointer(SpirvStorageClass.Private, _uintType); _privateVec2Pointer = _module.TypePointer(SpirvStorageClass.Private, _vec2Type); - _privateBoolPointer = - _module.TypePointer(SpirvStorageClass.Private, _boolType); var scalarArrayType = _module.TypeArray(_uintType, ScalarRegisterCount); var vectorArrayType = _module.TypeArray(_uintType, VectorRegisterCount); @@ -622,7 +623,7 @@ public static partial class Gen5SpirvTranslator private void DeclareWave64Scratch() { - if (!_emulateWave64 || !UsesSubgroupOperations()) + if (!UsesWave64Exchange()) { return; } @@ -630,36 +631,39 @@ public static partial class Gen5SpirvTranslator // Metal exposes 32 KiB of threadgroup memory on the Apple GPUs we // target. Some PS5 compute shaders legitimately request all of it, // so allocating another workgroup variable for the wave64 bridge - // makes pipeline creation fail. Reuse the final three dwords of the + // makes pipeline creation fail. Reuse the final dwords of the // existing LDS allocation in that case. The translator already // bounds guest LDS accesses to this fixed allocation; keeping the // bridge inside it preserves the host limit and still provides the // cross-subgroup rendezvous needed to model one 64-lane guest wave. if (_lds != 0) { - _waveScratchInLds = true; - _waveMaskScratchElementPointer = _ldsElementPointer; + _wave64Exchange = _lds; + _wave64ExchangeElementPointer = _ldsElementPointer; + _wave64ExchangeOffset = _ldsDwordMask + 1 < LdsDwordCount + ? _ldsDwordMask + 1 + : LdsDwordCount - Wave64ExchangeDwordCount; return; } - var maskArrayType = _module.TypeArray(_uintType, 2); - var maskArrayPointer = - _module.TypePointer(SpirvStorageClass.Workgroup, maskArrayType); - _waveMaskScratchElementPointer = - _module.TypePointer(SpirvStorageClass.Workgroup, _uintType); - _waveMaskScratch = _module.AddGlobalVariable( - maskArrayPointer, + var exchangeArrayType = _module.TypeArray(_uintType, Wave64ExchangeDwordCount); + _wave64ExchangeElementPointer = _module.TypePointer(SpirvStorageClass.Workgroup, _uintType); + _wave64Exchange = _module.AddGlobalVariable( + _module.TypePointer(SpirvStorageClass.Workgroup, exchangeArrayType), SpirvStorageClass.Workgroup); - _module.AddName(_waveMaskScratch, "wave64MaskScratch"); - _interfaces.Add(_waveMaskScratch); + _wave64ExchangeOffset = 0; + _module.AddName(_wave64Exchange, "wave64Exchange"); + _interfaces.Add(_wave64Exchange); + } - var uintPointer = - _module.TypePointer(SpirvStorageClass.Workgroup, _uintType); - _waveBroadcastScratch = _module.AddGlobalVariable( - uintPointer, - SpirvStorageClass.Workgroup); - _module.AddName(_waveBroadcastScratch, "wave64BroadcastScratch"); - _interfaces.Add(_waveBroadcastScratch); + private bool UsesWave64Exchange() => _emulateWave64 && UsesSubgroupOperations(); + + private uint ComputeLdsGuestDwordCount() + { + var declared = _request.LocalDataShareDwords; + return declared == 0 || declared > LdsDwordCount + ? LdsDwordCount + : System.Numerics.BitOperations.RoundUpToPowerOf2(declared); } private void DeclareLds() @@ -681,11 +685,14 @@ public static partial class Gen5SpirvTranslator ? SpirvStorageClass.Workgroup : SpirvStorageClass.Private; var dwordCount = _stage == Gen5SpirvStage.Compute - ? LdsDwordCount + ? ComputeLdsGuestDwordCount() : PrivateLdsDwordCount; _ldsDwordMask = dwordCount - 1; + var arrayDwordCount = UsesWave64Exchange() && dwordCount < LdsDwordCount + ? dwordCount + Wave64ExchangeDwordCount + : dwordCount; - var ldsArrayType = _module.TypeArray(_uintType, dwordCount); + var ldsArrayType = _module.TypeArray(_uintType, arrayDwordCount); var ldsPointer = _module.TypePointer(storageClass, ldsArrayType); _ldsElementPointer = _module.TypePointer(storageClass, _uintType); _lds64ElementPointer = _module.TypePointer(storageClass, _ulongType); @@ -769,24 +776,6 @@ public static partial class Gen5SpirvTranslator } _interfaces.Add(_subgroupInvocationIdInput); - if (_emulateWave64) - { - _subgroupSizeInput = _module.AddGlobalVariable( - subgroupPointer, - SpirvStorageClass.Input); - _module.AddDecoration( - _subgroupSizeInput, - SpirvDecoration.BuiltIn, - (uint)SpirvBuiltIn.SubgroupSize); - if (_stage == Gen5SpirvStage.Pixel) - { - _module.AddDecoration( - _subgroupSizeInput, - SpirvDecoration.Flat); - } - _interfaces.Add(_subgroupSizeInput); - } - // LocalInvocationIndex is a compute-stage built-in. Graphics // stages can still use native subgroup operations, but they // cannot combine two subgroup32 halves through a workgroup. @@ -1103,7 +1092,12 @@ public static partial class Gen5SpirvTranslator { Store(_pixelValidMaskActive, _module.ConstantBool(true)); } - if (_subgroupInvocationIdInput != 0) + if (_subgroupInvocationIdInput != 0 && _emulateWave64) + { + StoreS64(106, BothHalvesOfOwnBallot(_module.ConstantBool(false))); + StoreS64(126, BothHalvesOfOwnBallot(_module.ConstantBool(true))); + } + else if (_subgroupInvocationIdInput != 0) { StoreWaveMask(106, _module.ConstantBool(false)); StoreWaveMask(126, _module.ConstantBool(true)); @@ -1362,13 +1356,28 @@ public static partial class Gen5SpirvTranslator { error = string.Empty; var block = blocks[blockIndex]; + var halfMaskPlan = HalfMaskPlan(); // One guest wave can span two host subgroups. Keep its shared-memory phases ordered. - // Restrict added barriers to a single block, where all invocations follow the same path. - var synchronizeSharedMemory = _emulateWave64 && blocks.Count == 1; + // The half-mask plan inserts its own barriers; other wave64 programs also + // need LDS phase ordering when reads and writes span basic blocks. + var synchronizeSharedMemory = _emulateWave64 && halfMaskPlan is null; var sharedMemoryPhase = SharedMemoryPhase.None; for (var index = block.StartIndex; index < block.EndIndex; index++) { var instruction = _request.Program.Instructions[index]; + if (halfMaskPlan is not null) + { + if (halfMaskPlan.ExactPairsBefore.TryGetValue(instruction.Pc, out var exactPairs)) + { + EmitExactWaveMasks(exactPairs); + } + + if (halfMaskPlan.SharedMemoryBarriersBefore.Contains(instruction.Pc)) + { + EmitWave64Barrier(); + } + } + if (IsBranch(instruction.Opcode) || instruction.Opcode == "SEndpgm") { continue; @@ -1389,7 +1398,9 @@ public static partial class Gen5SpirvTranslator } _execKnownFull = IsExecKnownFull(instruction.Pc); + _emittingPc = instruction.Pc; var emitted = TryEmitInstruction(instruction, out error); + _emittingPc = null; _execKnownFull = false; if (!emitted) { @@ -1653,10 +1664,10 @@ public static partial class Gen5SpirvTranslator { "SCbranchScc0" => LogicalNot(Load(_boolType, _scc)), "SCbranchScc1" => Load(_boolType, _scc), - "SCbranchVccz" => LogicalNot(SubgroupAny(Load(_boolType, _vcc))), - "SCbranchVccnz" => SubgroupAny(Load(_boolType, _vcc)), - "SCbranchExecz" => LogicalNot(SubgroupAny(Load(_boolType, _exec))), - "SCbranchExecnz" => SubgroupAny(Load(_boolType, _exec)), + "SCbranchVccz" => LogicalNot(WaveMaskAny(106, _vcc)), + "SCbranchVccnz" => WaveMaskAny(106, _vcc), + "SCbranchExecz" => LogicalNot(WaveMaskAny(126, _exec)), + "SCbranchExecnz" => WaveMaskAny(126, _exec), // The emulator does not expose a shader debug session. "SCbranchCdbgsys" or "SCbranchCdbguser" or @@ -2191,11 +2202,7 @@ public static partial class Gen5SpirvTranslator uint broadcast; if (_emulateWave64) { - EmitConditional(isFirstActive, () => - Store(WaveBroadcastScratchPointer(), firstValue)); - EmitWave64Barrier(); - broadcast = Load(_uintType, WaveBroadcastScratchPointer()); - EmitWave64Barrier(); + broadcast = ExchangeWave64Value(isFirstActive, firstValue); } else { @@ -7006,6 +7013,13 @@ public static partial class Gen5SpirvTranslator return variable; } + uint Flag(bool initialValue) + { + var variable = Variable(_uintType, UInt(initialValue ? 1u : 0u)); + _flagVariables.Add(variable); + return variable; + } + _scalarRegisters = Variable(_scalarArrayType, _module.ConstantNull(_scalarArrayType)); _vectorRegisters = Variable(_vectorArrayType, _module.ConstantNull(_vectorArrayType)); if (_functionScopeState) @@ -7016,18 +7030,23 @@ public static partial class Gen5SpirvTranslator _module.AddName(_packedHalfRegisters, "vgprPackedHalf"); } - _scc = Variable(_boolType, _module.ConstantBool(false)); - _vcc = Variable(_boolType, _module.ConstantBool(false)); - _exec = Variable(_boolType, _module.ConstantBool(true)); - _reachedPixelExport = Variable(_boolType, _module.ConstantBool(false)); + _scc = Flag(false); + _vcc = Flag(false); + _exec = Flag(true); + _reachedPixelExport = Flag(false); if (_usesPixelValidMask) { - _pixelValidMaskActive = Variable(_boolType, _module.ConstantBool(true)); + _pixelValidMaskActive = Flag(true); _module.AddName(_pixelValidMaskActive, "pixelValidMaskActive"); } _programCounter = Variable(_uintType, _module.Constant(_uintType, 0)); - _programActive = Variable(_boolType, _module.ConstantBool(true)); + _programActive = Flag(true); + if (UsesWave64Exchange()) + { + _wave64ExchangeParity = Variable(_uintType, _module.Constant(_uintType, 0)); + _module.AddName(_wave64ExchangeParity, "wave64ExchangeParity"); + } if (_maxDispatcherSteps > 0) { _iterationGuard = Variable(_uintType, _module.Constant(_uintType, 0)); @@ -7307,11 +7326,32 @@ public static partial class Gen5SpirvTranslator "SPIR-V generator attempted OpLoad from id 0."); } + if (_flagVariables.Contains(pointer)) + { + return _module.AddInstruction( + SpirvOp.INotEqual, + _boolType, + _module.AddInstruction(SpirvOp.Load, _uintType, pointer), + UInt(0)); + } + return _module.AddInstruction(SpirvOp.Load, type, pointer); } - private void Store(uint pointer, uint value) => + private void Store(uint pointer, uint value) + { + if (_flagVariables.Contains(pointer)) + { + value = _module.AddInstruction( + SpirvOp.Select, + _uintType, + value, + UInt(1), + UInt(0)); + } + _module.AddStatement(SpirvOp.Store, pointer, value); + } private uint UInt(uint value) => _module.Constant(_uintType, value); @@ -7346,6 +7386,31 @@ public static partial class Gen5SpirvTranslator Bitcast(_intType, left), BitwiseAnd(right, UInt(31)))); + private uint _uintPairType; + private uint _intPairType; + + private (uint Low, uint High) MultiplyExtended(uint left, uint right, bool signed) + { + var elementType = signed ? _intType : _uintType; + if (signed && _intPairType == 0) + { + _intPairType = _module.TypeStruct(_intType, _intType); + } + else if (!signed && _uintPairType == 0) + { + _uintPairType = _module.TypeStruct(_uintType, _uintType); + } + + var product = _module.AddInstruction( + signed ? SpirvOp.SMulExtended : SpirvOp.UMulExtended, + signed ? _intPairType : _uintPairType, + left, + right); + return ( + _module.AddInstruction(SpirvOp.CompositeExtract, elementType, product, 0), + _module.AddInstruction(SpirvOp.CompositeExtract, elementType, product, 1)); + } + private uint ShiftLeftLogical64(uint left, uint right) => _module.AddInstruction( SpirvOp.ShiftLeftLogical, @@ -7495,6 +7560,19 @@ public static partial class Gen5SpirvTranslator return BooleanToLaneMask(condition); } + if (_emulateWave64) + { + var lane = GuestWaveLane(); + var owned = OwnHalfBallot(condition); + var exchange = BeginWave64Exchange(); + EmitConditional(IsHalfWaveLeader(lane), () => + Store(Wave64ExchangePointer(exchange, ShiftRightLogical(lane, UInt(5))), owned)); + EmitWave64Barrier(); + return Pair64( + Load(_uintType, Wave64ExchangePointer(exchange, UInt(0))), + Load(_uintType, Wave64ExchangePointer(exchange, UInt(1)))); + } + var ballot = _module.AddInstruction( SpirvOp.GroupNonUniformBallot, _uvec4Type, @@ -7505,57 +7583,6 @@ public static partial class Gen5SpirvTranslator _uintType, ballot, 0); - if (_emulateWave64) - { - var high = _module.AddInstruction( - SpirvOp.CompositeExtract, - _uintType, - ballot, - 1); - var subgroupLane = - Load(_uintType, _subgroupInvocationIdInput); - var firstLane = _module.AddInstruction( - SpirvOp.IEqual, - _boolType, - subgroupLane, - UInt(0)); - var half = ShiftRightLogical(GuestWaveLane(), UInt(5)); - EmitConditional(firstLane, () => - { - Store(WaveMaskScratchPointer(half), low); - - var nativeWave64 = _module.AddInstruction( - SpirvOp.UGreaterThanEqual, - _boolType, - Load(_uintType, _subgroupSizeInput), - UInt(64)); - EmitConditional(nativeWave64, () => - { - Store(WaveMaskScratchPointer(UInt(1)), high); - }); - }); - EmitWave64Barrier(); - var lowMask = Load( - _uintType, - WaveMaskScratchPointer(UInt(0))); - var highMask = Load( - _uintType, - WaveMaskScratchPointer(UInt(1))); - var combined = BitwiseOr64( - _module.AddInstruction( - SpirvOp.UConvert, - _ulongType, - lowMask), - ShiftLeftLogical64( - _module.AddInstruction( - SpirvOp.UConvert, - _ulongType, - highMask), - _module.Constant64(_ulongType, 32))); - EmitWave64Barrier(); - return combined; - } - var widened = _module.AddInstruction(SpirvOp.UConvert, _ulongType, low); if (_waveLaneCount != 64) { @@ -7576,21 +7603,127 @@ public static partial class Gen5SpirvTranslator widened); } - private uint WaveMaskScratchPointer(uint index) => + private uint OwnHalfBallot(uint condition) + { + var ballot = _module.AddInstruction( + SpirvOp.GroupNonUniformBallot, + _uvec4Type, + UInt(3), + condition); + return _module.AddInstruction( + SpirvOp.VectorExtractDynamic, + _uintType, + ballot, + ShiftRightLogical(Load(_uintType, _subgroupInvocationIdInput), UInt(5))); + } + + private uint BooleanToHalfWaveMask(uint condition) + { + var owned = _module.AddInstruction(SpirvOp.UConvert, _ulongType, OwnHalfBallot(condition)); + return _module.AddInstruction( + SpirvOp.Select, + _ulongType, + _module.AddInstruction(SpirvOp.UGreaterThanEqual, _boolType, GuestWaveLane(), UInt(32)), + ShiftLeftLogical64(owned, _module.Constant64(_ulongType, 32)), + owned); + } + + private uint BothHalvesOfOwnBallot(uint condition) + { + var owned = _module.AddInstruction(SpirvOp.UConvert, _ulongType, OwnHalfBallot(condition)); + return BitwiseOr64(owned, ShiftLeftLogical64(owned, _module.Constant64(_ulongType, 32))); + } + + private uint IsHalfWaveLeader(uint lane) => + _module.AddInstruction(SpirvOp.IEqual, _boolType, BitwiseAnd(lane, UInt(31)), UInt(0)); + + private uint BeginWave64Exchange() + { + var parity = Load(_uintType, _wave64ExchangeParity); + Store(_wave64ExchangeParity, BitwiseXor(parity, UInt(1))); + return IAdd( + UInt(_wave64ExchangeOffset), + _module.AddInstruction(SpirvOp.IMul, _uintType, parity, UInt(Wave64ExchangeSlotCount))); + } + + private uint Wave64ExchangePointer(uint exchange, uint slot) => _module.AddInstruction( SpirvOp.AccessChain, - _waveMaskScratchElementPointer, - _waveScratchInLds ? _lds : _waveMaskScratch, - _waveScratchInLds ? IAdd(UInt(LdsDwordCount - 3), index) : index); + _wave64ExchangeElementPointer, + _wave64Exchange, + IAdd(exchange, slot)); - private uint WaveBroadcastScratchPointer() => - _waveScratchInLds - ? _module.AddInstruction( - SpirvOp.AccessChain, - _ldsElementPointer, - _lds, - UInt(LdsDwordCount - 1)) - : _waveBroadcastScratch; + private uint ExchangeWave64Value(uint writer, uint value) + { + var exchange = BeginWave64Exchange(); + EmitConditional(writer, () => Store(Wave64ExchangePointer(exchange, UInt(0)), value)); + EmitWave64Barrier(); + return Load(_uintType, Wave64ExchangePointer(exchange, UInt(0))); + } + + private void EmitExactWaveMasks(IReadOnlyList pairs) + { + var lane = GuestWaveLane(); + var upperHalf = ShiftRightLogical(lane, UInt(5)); + var lowerHalf = _module.AddInstruction(SpirvOp.IEqual, _boolType, upperHalf, UInt(0)); + const int pairsPerExchange = (int)Wave64ExchangeSlotCount / 2; + for (var start = 0; start < pairs.Count; start += pairsPerExchange) + { + var count = Math.Min(pairsPerExchange, pairs.Count - start); + var owned = new uint[count]; + for (var index = 0; index < count; index++) + { + var pair = pairs[start + index]; + owned[index] = _module.AddInstruction(SpirvOp.Select, _uintType, lowerHalf, LoadS(pair), LoadS(pair + 1)); + } + + var exchange = BeginWave64Exchange(); + EmitConditional(IsHalfWaveLeader(lane), () => + { + for (var index = 0; index < count; index++) + { + Store(Wave64ExchangePointer(exchange, IAdd(UInt((uint)index * 2), upperHalf)), owned[index]); + } + }); + EmitWave64Barrier(); + for (var index = 0; index < count; index++) + { + var pair = pairs[start + index]; + StoreS(pair, Load(_uintType, Wave64ExchangePointer(exchange, UInt((uint)index * 2)))); + StoreS(pair + 1, Load(_uintType, Wave64ExchangePointer(exchange, UInt((uint)index * 2 + 1)))); + } + } + } + + private uint WaveMaskAny(uint register, uint laneActive) => + _emulateWave64 && _subgroupInvocationIdInput != 0 + ? IsNotZero64(LoadS64(register)) + : SubgroupAny(Load(_boolType, laneActive)); + + private static readonly bool HalfWaveMasks = !string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_WAVE64_HALF_MASKS"), "0", StringComparison.Ordinal); + + private bool _halfMaskPlanBuilt; + + private Ir.Gen5Wave64HalfMaskPlan? HalfMaskPlan() + { + if (_halfMaskPlanBuilt) + { + return _halfMaskPlan; + } + + _halfMaskPlanBuilt = true; + if (HalfWaveMasks && UsesWave64Exchange() && _subgroupInvocationIdInput != 0) + { + var sharedFlatPcs = _request.Memory.Entries + .Where(static memory => memory.AddressSpace is FlatAddressSpace.Shared or FlatAddressSpace.SharedOrPrivate) + .Select(static memory => memory.Pc) + .ToHashSet(); + _halfMaskPlan = Ir.Gen5Wave64HalfMaskAnalysis.Analyze(_request.Program, sharedFlatPcs); + } + + return _halfMaskPlan; + } private void EmitWave64Barrier() { @@ -7635,6 +7768,17 @@ public static partial class Gen5SpirvTranslator return; } + if (_emulateWave64 && + _subgroupInvocationIdInput != 0 && + _emittingPc is { } pc && + HalfMaskPlan() is { } plan && + plan.HalfMaskWrites.TryGetValue(pc, out var halfRegister) && + halfRegister == register) + { + StoreS64(register, BooleanToHalfWaveMask(condition)); + return; + } + StoreS64(register, BooleanToWaveMask(condition)); } diff --git a/src/SharpEmu.ShaderCompiler/Ir/Gen5Wave64HalfMaskAnalysis.cs b/src/SharpEmu.ShaderCompiler/Ir/Gen5Wave64HalfMaskAnalysis.cs new file mode 100644 index 0000000..f238002 --- /dev/null +++ b/src/SharpEmu.ShaderCompiler/Ir/Gen5Wave64HalfMaskAnalysis.cs @@ -0,0 +1,565 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.ShaderCompiler.Ir; + +public sealed class Gen5Wave64HalfMaskPlan +{ + public required IReadOnlyDictionary ExactPairsBefore { get; init; } + + public required IReadOnlyDictionary HalfMaskWrites { get; init; } + + public required IReadOnlySet SharedMemoryBarriersBefore { get; init; } +} + +public static class Gen5Wave64HalfMaskAnalysis +{ + private const uint Vcc = 106; + private const uint Exec = 126; + private const uint VcczSource = 251; + private const uint ExeczSource = 252; + private const uint SccSource = 253; + + private static readonly UInt128 EvenRegisters = new(0x5555_5555_5555_5555UL, 0x5555_5555_5555_5555UL); + private static readonly UInt128 OddRegisters = EvenRegisters << 1; + + private static readonly HashSet CarryIn = new(StringComparer.Ordinal) + { + "VAddcU32", "VAddCoCiU32", "VSubbU32", "VSubCoCiU32", "VSubbrevU32", "VSubrevCoCiU32", + }; + + private static readonly HashSet CarryOut = new(CarryIn, StringComparer.Ordinal) + { + "VAddCoU32", "VSubCoU32", "VSubrevCoU32", "VMadU64U32", "VMadI64I32", + }; + + private static readonly HashSet BitwiseWithScc = new(StringComparer.Ordinal) + { + "SNotB64", "SAndB64", "SOrB64", "SXorB64", "SAndn1B64", "SAndn2B64", "SOrn1B64", "SOrn2B64", + "SNandB64", "SNorB64", "SXnorB64", + "SAndSaveexecB64", "SOrSaveexecB64", "SXorSaveexecB64", "SAndn1SaveexecB64", "SAndn2SaveexecB64", + "SOrn1SaveexecB64", "SOrn2SaveexecB64", "SNandSaveexecB64", "SNorSaveexecB64", "SXnorSaveexecB64", + }; + + private static readonly HashSet BitwiseWithoutScc = new(StringComparer.Ordinal) + { + "SMovB64", "SWqmB64", "SCselectB64", + }; + + private static readonly HashSet SccReaders = new(StringComparer.Ordinal) + { + "SCbranchScc0", "SCbranchScc1", "SAddcU32", "SSubbU32", "SCselectB32", "SCselectB64", + }; + + private static readonly HashSet SccWriters = new(StringComparer.Ordinal) + { + "SNotB32", "SAbsI32", "SWqmB32", "SBrevB32", "SBcnt1I32B32", "SBcnt1I32B64", + "SAddU32", "SSubU32", "SAddI32", "SSubI32", "SAddcU32", "SSubbU32", + "SAndB32", "SOrB32", "SXorB32", "SAndn2B32", "SOrn2B32", "SNandB32", "SNorB32", "SXnorB32", + "SLshlB32", "SLshrB32", "SAshrI32", "SBfeU32", "SBfeI32", "SMinU32", "SMinI32", "SMaxU32", "SMaxI32", + "SLshlB64", "SLshrB64", "SAshrI64", "SBfeU64", "SBfeI64", "SBfmB64", + }; + + private static readonly HashSet DynamicControlFlow = new(StringComparer.Ordinal) + { + "SSetpcB64", "SSwappcB64", "SRfeB64", "SCbranchJoin", "SCbranchGFork", "SCbranchIFork", + }; + + private readonly record struct State(bool Reached, UInt128 Half, bool PendingRead, bool PendingWrite) + { + public static State Meet(State left, State right) => + !left.Reached ? right + : !right.Reached ? left + : new State(true, left.Half | right.Half, left.PendingRead | right.PendingRead, left.PendingWrite | right.PendingWrite); + } + + public static Gen5Wave64HalfMaskPlan? Analyze(Gen5ShaderProgram program, IReadOnlySet sharedFlatPcs) + { + var instructions = program.Instructions; + if (instructions.Count == 0) + { + return null; + } + + var successors = BuildSuccessors(instructions); + if (successors is null) + { + return null; + } + + var sccLiveAfter = ComputeSccLiveAfter(instructions, successors); + var states = new State[instructions.Count]; + states[0] = new State(true, 0, false, false); + var pending = new Queue(); + var queued = new bool[instructions.Count]; + pending.Enqueue(0); + queued[0] = true; + while (pending.TryDequeue(out var index)) + { + queued[index] = false; + var output = Step(instructions[index], states[index], sccLiveAfter[index], sharedFlatPcs, out _, out _, out _); + foreach (var successor in successors[index]) + { + var merged = State.Meet(states[successor], output); + if (merged != states[successor]) + { + states[successor] = merged; + if (!queued[successor]) + { + queued[successor] = true; + pending.Enqueue(successor); + } + } + } + } + + var exactPairs = new Dictionary(); + var halfWrites = new Dictionary(); + var barriers = new HashSet(); + for (var index = 0; index < instructions.Count; index++) + { + if (!states[index].Reached) + { + continue; + } + + var instruction = instructions[index]; + Step(instruction, states[index], sccLiveAfter[index], sharedFlatPcs, out var exact, out var barrier, out var halfWrite); + if (exact != 0) + { + exactPairs[instruction.Pc] = PairBases(exact); + } + + if (barrier) + { + barriers.Add(instruction.Pc); + } + + if (halfWrite && TryGetHalfMaskDestination(instruction, out var maskDestination)) + { + halfWrites[instruction.Pc] = maskDestination; + } + } + + return new Gen5Wave64HalfMaskPlan + { + ExactPairsBefore = exactPairs, + HalfMaskWrites = halfWrites, + SharedMemoryBarriersBefore = barriers, + }; + } + + private static State Step( + Gen5ShaderInstruction instruction, + State input, + bool sccLiveAfter, + IReadOnlySet sharedFlatPcs, + out UInt128 exact, + out bool sharedBarrier, + out bool halfWrite) + { + var opcode = instruction.Opcode; + var half = input.Half; + var bitwise = IsBitwiseMaskOperation(instruction); + UInt128 needed = 0; + UInt128 bitwiseInputs = 0; + for (var index = 0; index < instruction.Sources.Count; index++) + { + var operand = instruction.Sources[index]; + if (operand.Kind == Gen5OperandKind.EncodedConstant) + { + if (operand.Value == VcczSource) + { + needed |= Pair(Vcc); + } + else if (operand.Value == ExeczSource) + { + needed |= Pair(Exec); + } + + continue; + } + + if (operand.Kind != Gen5OperandKind.ScalarRegister) + { + continue; + } + + if (bitwise) + { + bitwiseInputs |= Pair(operand.Value); + continue; + } + + if (index == 2 && (opcode == "VCndmaskB32" || CarryIn.Contains(opcode))) + { + continue; + } + + needed |= ReadRegisters(instruction, index, operand.Value); + } + + if (opcode is "SCbranchExecz" or "SCbranchExecnz") + { + needed |= Pair(Exec); + } + else if (opcode is "SCbranchVccz" or "SCbranchVccnz") + { + needed |= Pair(Vcc); + } + + if ((instruction.Encoding == Gen5ShaderEncoding.Sopk || opcode.StartsWith("SBitset", StringComparison.Ordinal)) && + instruction.Destinations is [{ Kind: Gen5OperandKind.ScalarRegister } readDestination, ..]) + { + needed |= Range(readDestination.Value, opcode.Contains("64", StringComparison.Ordinal) ? 2u : 1u); + } + + if (opcode.StartsWith("SMovrel", StringComparison.Ordinal)) + { + needed |= half; + } + + if (bitwise) + { + if (opcode.EndsWith("SaveexecB64", StringComparison.Ordinal)) + { + bitwiseInputs |= Pair(Exec); + } + + if (sccLiveAfter && BitwiseWithScc.Contains(opcode)) + { + needed |= bitwiseInputs; + } + } + + exact = Expand(needed & half); + half &= ~exact; + + var pendingRead = input.PendingRead; + var pendingWrite = input.PendingWrite; + if (exact != 0) + { + pendingRead = false; + pendingWrite = false; + } + + var (reads, writes) = SharedMemoryAccess(instruction, sharedFlatPcs); + sharedBarrier = (writes && (pendingRead || pendingWrite)) || (reads && pendingWrite); + if (sharedBarrier) + { + pendingRead = false; + pendingWrite = false; + } + + pendingRead |= reads; + pendingWrite |= writes; + if (opcode == "SBarrier") + { + pendingRead = false; + pendingWrite = false; + } + + halfWrite = false; + if (TryGetHalfMaskDestination(instruction, out var maskDestination)) + { + half |= Pair(maskDestination); + halfWrite = true; + } + else if (bitwise) + { + var destination = instruction.Destinations[0].Value; + var outputHalf = (bitwiseInputs & half) != 0; + half = outputHalf ? half | Pair(destination) : half & ~Pair(destination); + if (opcode.EndsWith("SaveexecB64", StringComparison.Ordinal)) + { + half = outputHalf ? half | Pair(Exec) : half & ~Pair(Exec); + } + } + else + { + var wide = IsWideScalar(opcode); + foreach (var destination in instruction.Destinations) + { + if (destination.Kind != Gen5OperandKind.ScalarRegister) + { + continue; + } + + half &= ~Bit(destination.Value); + if (wide) + { + half &= ~Bit(destination.Value + 1); + } + } + + if (opcode.EndsWith("SaveexecB32", StringComparison.Ordinal)) + { + half &= ~Pair(Exec); + } + } + + return new State(true, half, pendingRead, pendingWrite); + } + + private static bool TryGetHalfMaskDestination(Gen5ShaderInstruction instruction, out uint destination) + { + var opcode = instruction.Opcode; + if (opcode.StartsWith("VCmpx", StringComparison.Ordinal)) + { + destination = Exec; + } + else if (opcode.StartsWith("VCmp", StringComparison.Ordinal)) + { + destination = instruction.Control switch + { + Gen5SdwaControl { ScalarDestination: { } sdwa } => sdwa, + Gen5Vop3Control { ScalarDestination: { } vop3 } => vop3, + _ => Vcc, + }; + } + else if (CarryOut.Contains(opcode)) + { + destination = instruction.Control is Gen5Vop3Control { ScalarDestination: { } carry } ? carry : Vcc; + } + else + { + destination = 0; + return false; + } + + return (destination & 1) == 0 && (destination < 124 || destination == Exec); + } + + private static bool IsBitwiseMaskOperation(Gen5ShaderInstruction instruction) + { + if (!BitwiseWithScc.Contains(instruction.Opcode) && !BitwiseWithoutScc.Contains(instruction.Opcode)) + { + return false; + } + + if (instruction.Destinations is not [{ Kind: Gen5OperandKind.ScalarRegister } destination] || + (destination.Value & 1) != 0 || + (destination.Value >= 124 && destination.Value != Exec)) + { + return false; + } + + foreach (var source in instruction.Sources) + { + if (source.Kind == Gen5OperandKind.VectorRegister) + { + return false; + } + + if (source.Kind == Gen5OperandKind.ScalarRegister && + ((source.Value & 1) != 0 || (source.Value >= 124 && source.Value != Exec))) + { + return false; + } + } + + return true; + } + + private static (bool Reads, bool Writes) SharedMemoryAccess(Gen5ShaderInstruction instruction, IReadOnlySet sharedFlatPcs) + { + if (instruction.Control is Gen5DataShareControl { Gds: false }) + { + var opcode = instruction.Opcode; + if (opcode is "DsSwizzleB32" or "DsBpermuteB32" or "DsPermuteB32" or "DsNop") + { + return (false, false); + } + + if (opcode.StartsWith("DsRead", StringComparison.Ordinal)) + { + return (true, false); + } + + if (opcode.StartsWith("DsWrite", StringComparison.Ordinal)) + { + return (false, true); + } + + return (true, true); + } + + return sharedFlatPcs.Contains(instruction.Pc) ? (true, true) : (false, false); + } + + private static int[][]? BuildSuccessors(IReadOnlyList instructions) + { + var indexByPc = new Dictionary(instructions.Count); + for (var index = 0; index < instructions.Count; index++) + { + if (DynamicControlFlow.Contains(instructions[index].Opcode)) + { + return null; + } + + indexByPc[instructions[index].Pc] = index; + } + + var last = instructions[^1]; + var endPc = last.Pc + (uint)(last.Words.Count * sizeof(uint)); + var successors = new int[instructions.Count][]; + for (var index = 0; index < instructions.Count; index++) + { + var instruction = instructions[index]; + var next = index + 1 < instructions.Count ? index + 1 : -1; + if (Gen5IrBranchResolver.IsTerminator(instruction)) + { + successors[index] = []; + continue; + } + + var unconditional = Gen5IrBranchResolver.IsUnconditionalBranch(instruction); + if (!unconditional && !Gen5IrBranchResolver.Instance.IsConditional(instruction)) + { + successors[index] = next < 0 ? [] : [next]; + continue; + } + + if (!Gen5IrBranchResolver.Instance.TryGetBranchTarget(instruction, out var targetPc)) + { + return null; + } + + var targets = new List(2); + if (indexByPc.TryGetValue(targetPc, out var target)) + { + targets.Add(target); + } + else if (targetPc < endPc) + { + return null; + } + + if (!unconditional && next >= 0) + { + targets.Add(next); + } + + successors[index] = [.. targets]; + } + + return successors; + } + + private static bool[] ComputeSccLiveAfter(IReadOnlyList instructions, int[][] successors) + { + var liveBefore = new bool[instructions.Count]; + var liveAfter = new bool[instructions.Count]; + bool changed; + do + { + changed = false; + for (var index = instructions.Count - 1; index >= 0; index--) + { + var after = false; + foreach (var successor in successors[index]) + { + after |= liveBefore[successor]; + } + + var instruction = instructions[index]; + var before = ReadsScc(instruction) || (!WritesScc(instruction) && after); + if (after != liveAfter[index] || before != liveBefore[index]) + { + liveAfter[index] = after; + liveBefore[index] = before; + changed = true; + } + } + } + while (changed); + + return liveAfter; + } + + private static bool ReadsScc(Gen5ShaderInstruction instruction) + { + if (SccReaders.Contains(instruction.Opcode)) + { + return true; + } + + foreach (var source in instruction.Sources) + { + if (source.Kind == Gen5OperandKind.EncodedConstant && source.Value == SccSource) + { + return true; + } + } + + return false; + } + + private static bool WritesScc(Gen5ShaderInstruction instruction) => + instruction.Encoding == Gen5ShaderEncoding.Sopc || + instruction.Opcode.StartsWith("SCmpk", StringComparison.Ordinal) || + instruction.Opcode.EndsWith("SaveexecB32", StringComparison.Ordinal) || + SccWriters.Contains(instruction.Opcode) || + BitwiseWithScc.Contains(instruction.Opcode); + + private static bool IsWideScalar(string opcode) => + opcode.StartsWith('S') && + (opcode.EndsWith("B64", StringComparison.Ordinal) || + opcode.EndsWith("U64", StringComparison.Ordinal) || + opcode.EndsWith("I64", StringComparison.Ordinal)); + + private static UInt128 ReadRegisters(Gen5ShaderInstruction instruction, int sourceIndex, uint register) + { + var scalarOrdinal = 0; + for (var index = 0; index < sourceIndex; index++) + { + if (instruction.Sources[index].Kind == Gen5OperandKind.ScalarRegister) + { + scalarOrdinal++; + } + } + + var count = instruction.Encoding switch + { + Gen5ShaderEncoding.Smem or Gen5ShaderEncoding.Smrd => scalarOrdinal != 0 ? 1u + : instruction.Opcode.StartsWith("SLoad", StringComparison.Ordinal) ? 2u : 4u, + Gen5ShaderEncoding.Mubuf or Gen5ShaderEncoding.Mtbuf => scalarOrdinal == 0 ? 4u : 1u, + Gen5ShaderEncoding.Mimg => scalarOrdinal == 0 ? 8u : 4u, + Gen5ShaderEncoding.Flat => 2u, + _ => instruction.Opcode.Contains("64", StringComparison.Ordinal) ? 2u : 1u, + }; + return Range(register, count); + } + + private static UInt128 Range(uint register, uint count) + { + UInt128 registers = 0; + for (var offset = 0u; offset < count; offset++) + { + registers |= Bit(register + offset); + } + + return registers; + } + + private static UInt128 Expand(UInt128 registers) => + registers | ((registers & EvenRegisters) << 1) | ((registers & OddRegisters) >> 1); + + private static uint[] PairBases(UInt128 registers) + { + var bases = new List(); + for (uint register = 0; register < 128; register += 2) + { + if ((registers & Pair(register)) != 0) + { + bases.Add(register); + } + } + + return [.. bases]; + } + + private static UInt128 Pair(uint register) => Expand(Bit(register)); + + private static UInt128 Bit(uint register) => register < 128 ? UInt128.One << (int)register : 0; +} diff --git a/src/SharpEmu.ShaderCompiler/Resources/ResourceTracker.DenseImages.cs b/src/SharpEmu.ShaderCompiler/Resources/ResourceTracker.DenseImages.cs index fab01fe..2545faa 100644 --- a/src/SharpEmu.ShaderCompiler/Resources/ResourceTracker.DenseImages.cs +++ b/src/SharpEmu.ShaderCompiler/Resources/ResourceTracker.DenseImages.cs @@ -1,6 +1,8 @@ // Copyright (C) 2026 SharpEmu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +using SharpEmu.ShaderCompiler.Ir; + namespace SharpEmu.ShaderCompiler.Resources; public sealed partial class ResourceTracker @@ -195,9 +197,11 @@ public sealed partial class ResourceTracker var instructions = _graph.Program.Instructions; var firstLaneIndex = FindInstructionIndex(instructions, (uint)key.Payload); - if (firstLaneIndex < 0 || instructions[firstLaneIndex] is not { Opcode: "VReadfirstlaneB32", Sources.Count: 1, Destinations.Count: 1 } firstLane || + if (firstLaneIndex < 0 || instructions[firstLaneIndex] is not { Destinations.Count: 1 } firstLane || + firstLane.Sources.Count == 0 || firstLane.Sources[0] is not { Kind: Gen5OperandKind.VectorRegister } vectorSource || - firstLane.Destinations[0] is not { Kind: Gen5OperandKind.ScalarRegister } scalarDestination) + firstLane.Destinations[0] is not { Kind: Gen5OperandKind.ScalarRegister } || + !(firstLane is { Opcode: "VReadfirstlaneB32", Sources.Count: 1 } || ReadsActiveLane(instructions, firstLaneIndex))) { return null; } @@ -222,12 +226,8 @@ public sealed partial class ResourceTracker var descriptorLoadIndex = FindInstructionIndex(instructions, _plan.Memory[descriptorMemoryIndex].Pc); if (descriptorLoadIndex < 0 || instructions[descriptorLoadIndex] is not { - Control: Gen5ScalarMemoryControl - { - DestinationCount: >= 8, - DynamicOffsetRegister: { } dynamicOffsetRegister, - }, - } descriptorLoad || dynamicOffsetRegister != scalarDestination.Value || + Control: Gen5ScalarMemoryControl { DynamicOffsetRegister: not null }, + } descriptorLoad || descriptorLoad.Sources.Count == 0 || descriptorLoad.Sources[0] != Gen5Operand.Scalar(global.ScalarAddress)) { return null; @@ -244,8 +244,7 @@ public sealed partial class ResourceTracker if (TryGetSelfAddedConstant(instructions[addressDefinitionIndex], address, out _) && TryGetWaveIndexedStride(instructions, addressDefinitionIndex, address, heapAddress, out var maskRegister, out var bitRegister, out var maskOffset, out var indexStride) && - indexDataOffset != 0 && indexStride != 0 && instructions.Any(instruction => instruction.Opcode == "SBitset0B32" && - instruction.Destinations.Contains(maskRegister) && instruction.Sources.Contains(bitRegister))) + indexDataOffset != 0 && indexStride != 0 && ClearsMaskBit(instructions, 0, instructions.Count, maskRegister, bitRegister)) { return new(maskOffset, indexDataOffset, indexStride); } @@ -283,9 +282,7 @@ public sealed partial class ResourceTracker { if (instructions[scanIndex] is not { Opcode: "SFF1I32B32", Sources.Count: 1, Destinations.Count: 1 } scan || scan.Sources[0] != maskRegister || scan.Destinations[0] is not { Kind: Gen5OperandKind.ScalarRegister } bitRegister || - !instructions.Skip(scanIndex + 1).Take(before - scanIndex - 1) - .Any(instruction => instruction.Opcode == "SBitset0B32" && instruction.Destinations.Contains(maskRegister) && - instruction.Sources.Contains(bitRegister))) + !ClearsMaskBit(instructions, scanIndex + 1, before, maskRegister, bitRegister)) continue; for (var multiplyIndex = scanIndex + 1; multiplyIndex < before; multiplyIndex++) @@ -358,6 +355,130 @@ public sealed partial class ResourceTracker return true; } + private static bool ClearsMaskBit( + IReadOnlyList instructions, + int start, + int end, + Gen5Operand mask, + Gen5Operand bit) + { + end = Math.Min(end, instructions.Count); + for (var index = Math.Max(0, start); index < end; index++) + { + var instruction = instructions[index]; + if (instruction.Opcode == "SBitset0B32" && instruction.Destinations.Contains(mask) && instruction.Sources.Contains(bit)) + return true; + + if (instruction is not { Opcode: "SLshlB32", Sources.Count: 2, Destinations.Count: 1 } shift || + !TryGetConstant(shift.Sources[0], out var one) || one != 1 || shift.Sources[1] != bit || + shift.Destinations[0] is not { Kind: Gen5OperandKind.ScalarRegister } single) + continue; + + for (var clearIndex = index + 1; clearIndex < end; clearIndex++) + { + var clear = instructions[clearIndex]; + if (clear.Destinations.Contains(mask) && clear.Sources.Count == 2 && + ((clear.Opcode == "SXorB32" && clear.Sources.Contains(mask) && clear.Sources.Contains(single)) || + (clear.Opcode == "SAndn2B32" && clear.Sources[0] == mask && clear.Sources[1] == single))) + return true; + + if (clear.Destinations.Contains(single) || clear.Destinations.Contains(mask) || clear.Destinations.Contains(bit) || + (single.Value is 106 or 107 && clear.Opcode.StartsWith('V'))) + break; + } + } + + return false; + } + + private static bool ReadsActiveLane(IReadOnlyList instructions, int readLaneIndex) + { + var exec = Gen5Operand.Scalar(126); + var readLane = instructions[readLaneIndex]; + if (readLane is not { Opcode: "VReadlaneB32" } || readLane.Sources.Count < 2 || + readLane.Sources[1] is not { Kind: Gen5OperandKind.ScalarRegister } lane) + return false; + + var scanIndex = FindLastDefinition(instructions, readLaneIndex, lane); + if (scanIndex < 0 || instructions[scanIndex] is not { Opcode: "SFF1I32B64", Sources.Count: 1 } scan || + scan.Sources[0] is not { Kind: Gen5OperandKind.ScalarRegister } candidates) + return false; + + var copyIndex = FindLastDefinition(instructions, scanIndex, candidates); + if (copyIndex < 0 || instructions[copyIndex] is not { Opcode: "SMovB64", Sources.Count: 1 } copy || copy.Sources[0] != exec) + return false; + + for (var index = copyIndex + 1; index < readLaneIndex; index++) + { + if (WritesScalarPair(instructions[index], exec) || WritesScalarPair(instructions[index], candidates) || + Gen5IrBranchResolver.Instance.TryGetBranchTarget(instructions[index], out _)) + return false; + } + + var backIndex = -1; + for (var index = readLaneIndex + 1; index < instructions.Count; index++) + { + if (!Gen5IrBranchResolver.Instance.TryGetBranchTarget(instructions[index], out var target)) + continue; + if (target > instructions[copyIndex].Pc && target <= instructions[scanIndex].Pc) + { + backIndex = index; + break; + } + } + + if (backIndex < 0) + return false; + + var restoreIndex = -1; + for (var index = backIndex - 1; index > readLaneIndex; index--) + { + if (WritesScalarPair(instructions[index], exec)) + { + restoreIndex = index; + break; + } + } + + if (restoreIndex < 0 || instructions[restoreIndex] is not { Opcode: "SMovB64", Sources.Count: 1 } restore || + restore.Sources[0] is not { Kind: Gen5OperandKind.ScalarRegister } saved) + return false; + + var saveIndex = FindLastDefinition(instructions, restoreIndex, saved); + if (saveIndex <= readLaneIndex || instructions[saveIndex].Opcode != "SAndSaveexecB64") + return false; + + for (var index = readLaneIndex + 1; index < backIndex; index++) + { + var instruction = instructions[index]; + if (index > saveIndex && index < restoreIndex && WritesScalarPair(instruction, saved)) + return false; + if (WritesScalarPair(instruction, candidates) && + (instruction is not { Opcode: "SAndn2B64", Sources.Count: 2 } || instruction.Sources[0] != candidates)) + return false; + if (Gen5IrBranchResolver.Instance.TryGetBranchTarget(instruction, out var target) && + (target <= instruction.Pc || target > instructions[restoreIndex].Pc)) + return false; + } + + return true; + } + + private static bool WritesScalarPair(Gen5ShaderInstruction instruction, Gen5Operand register) + { + if (register.Value is 106 or 107 && instruction.Opcode.StartsWith('V')) + return true; + if (instruction.Control is Gen5Vop3Control { ScalarDestination: { } scalarDestination } && + scalarDestination + 1 >= register.Value && scalarDestination <= register.Value + 1) + return true; + if (register.Value == 126 && (instruction.Opcode.Contains("Saveexec", StringComparison.Ordinal) || + instruction.Opcode.StartsWith("VCmpx", StringComparison.Ordinal))) + return true; + var width = instruction.Opcode.Contains("64", StringComparison.Ordinal) ? 2u : 1u; + return instruction.Destinations.Any(destination => destination.Kind == Gen5OperandKind.ScalarRegister && + destination.Value + width > register.Value && destination.Value <= register.Value + 1); + } + private static int FindLastDefinition(IReadOnlyList instructions, int before, Gen5Operand destination) { for (var index = before - 1; index >= 0; index--) diff --git a/src/SharpEmu.ShaderCompiler/Resources/ScalarValueGraph.Builder.cs b/src/SharpEmu.ShaderCompiler/Resources/ScalarValueGraph.Builder.cs index e5d0c4c..ff1490c 100644 --- a/src/SharpEmu.ShaderCompiler/Resources/ScalarValueGraph.Builder.cs +++ b/src/SharpEmu.ShaderCompiler/Resources/ScalarValueGraph.Builder.cs @@ -1558,15 +1558,22 @@ public sealed partial class ScalarValueGraph var lane = instruction.Sources.Count > 1 ? Read(instruction.Sources[1], state) : _graph.Undefined(ScalarValueType.U32); if (instruction.Sources.Count < 2 || - instruction.Sources[0] is not { Kind: Gen5OperandKind.VectorRegister } source || - !lane.IsConstant || - !state.Lanes.TryGetValue((source.Value, lane.ConstantU32 & 63), out var value)) + instruction.Sources[0] is not { Kind: Gen5OperandKind.VectorRegister } source) { state.WriteScalar(destination.Value, _graph.Undefined(ScalarValueType.U32)); return; } - state.WriteScalar(destination.Value, value); + if (lane.IsConstant && state.Lanes.TryGetValue((source.Value, lane.ConstantU32 & 63), out var value)) + { + state.WriteScalar(destination.Value, value); + return; + } + + var sourceValue = state.ReadVector(source.Value); + state.WriteScalar(destination.Value, !lane.IsConstant && sourceValue.IsUndefined + ? _graph.FirstLane(sourceValue, state.Exec, instruction.Pc) + : _graph.Undefined(ScalarValueType.U32)); } // ---- memory instructions ---- diff --git a/src/SharpEmu.ShaderCompiler/Resources/ShaderResourcePlan.cs b/src/SharpEmu.ShaderCompiler/Resources/ShaderResourcePlan.cs index 8089357..f13d2d3 100644 --- a/src/SharpEmu.ShaderCompiler/Resources/ShaderResourcePlan.cs +++ b/src/SharpEmu.ShaderCompiler/Resources/ShaderResourcePlan.cs @@ -181,7 +181,8 @@ public sealed class ShaderResourcePlan var value = TableReads[slot].Value; if (value.Kind == ScalarValueKind.ScalarAddressWord && value.Operands.Length != 0 && value.Operands[0].Kind == ScalarValueKind.AddressHandle && - value.Operands[0].Operands.SequenceEqual(heap.Dwords)) + value.Operands[0].Operands.Length == heap.Dwords.Length && + value.Operands[0].Operands.Zip(heap.Dwords).All(pair => ReferenceEquals(pair.First, pair.Second) || Graph.Equivalent(pair.First, pair.Second))) { slots[slot] = 1; } diff --git a/src/SharpEmu.ShaderCompiler/ShaderCompileRequest.cs b/src/SharpEmu.ShaderCompiler/ShaderCompileRequest.cs index 5f17dc6..428b82e 100644 --- a/src/SharpEmu.ShaderCompiler/ShaderCompileRequest.cs +++ b/src/SharpEmu.ShaderCompiler/ShaderCompileRequest.cs @@ -186,6 +186,7 @@ public sealed class ShaderCompileRequest public uint LocalSizeX { get; init; } = 1; public uint LocalSizeY { get; init; } = 1; public uint LocalSizeZ { get; init; } = 1; + public uint LocalDataShareDwords { get; init; } // Fixed bounds for standalone modules; runtime-limit layouts read the dispatch data. public uint ThreadCountX { get; init; } = UnboundedThreadCount; diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Images/GuestImageCacheTests.Metadata.cs b/tests/SharpEmu.Libs.Tests/Gpu/Images/GuestImageCacheTests.Metadata.cs index b572546..125a6a8 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Images/GuestImageCacheTests.Metadata.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Images/GuestImageCacheTests.Metadata.cs @@ -156,6 +156,36 @@ public sealed partial class GuestImageCacheTests harness.Shutdown(); } + [Fact] + public void GuestDccClear_ReadsOnlyUniformClearSlices() + { + if (!GatePrerequisites.Ready(_vulkan)) return; + using var harness = new CacheHarness(_vulkan); + const ulong SliceSize = 0x2000; + var metadata = harness.MapBacked(0x10000, ReadWrite); + harness.Write(metadata, Enumerable.Repeat((byte)0x80, (int)SliceSize).ToArray()); + harness.Write(metadata + SliceSize, Enumerable.Repeat((byte)0x00, (int)SliceSize).ToArray()); + var mixed = Enumerable.Repeat((byte)0x40, (int)SliceSize).ToArray(); + mixed[^1] = 0xff; + harness.Write(metadata + 2 * SliceSize, mixed); + harness.Write(metadata + 3 * SliceSize, Enumerable.Repeat((byte)0xff, (int)SliceSize).ToArray()); + + Assert.True(harness.Images.TryReadGuestDccClear(metadata, SliceSize, 0, out var slice, out var code)); + Assert.Equal(metadata, slice); + Assert.Equal(0x80, code); + Assert.True(harness.Images.TryReadGuestDccClear(metadata, SliceSize, 1, out slice, out code)); + Assert.Equal(metadata + SliceSize, slice); + Assert.Equal(0x00, code); + Assert.False(harness.Images.TryReadGuestDccClear(metadata, SliceSize, 2, out _, out _)); + Assert.False(harness.Images.TryReadGuestDccClear(metadata, SliceSize, 3, out _, out _)); + Assert.False(harness.Images.TryReadGuestDccClear(metadata, 0, 0, out _, out _)); + + harness.Worker.Run(() => harness.Cache.FillBuffer(metadata, SliceSize, uint.MaxValue, isGds: false)); + Assert.False(harness.Images.TryReadGuestDccClear(metadata, SliceSize, 0, out _, out _)); + Assert.True(harness.Images.TryReadGuestDccClear(metadata, SliceSize, 1, out _, out _)); + harness.Shutdown(); + } + [Fact] public void Unregister_DropsMetadataInTheRange() { diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Images/ImageDescriptionTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Images/ImageDescriptionTests.cs index ac84d0c..619a640 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Images/ImageDescriptionTests.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Images/ImageDescriptionTests.cs @@ -259,6 +259,28 @@ public sealed class ImageDescriptionTests Reject(association, "stencil-association image", fatal); } + [Theory] + [InlineData(2560u, 1440u, 8u, 1u, 1u, 0x1E000ul)] + [InlineData(1920u, 1080u, 4u, 1u, 1u, 0xC000ul)] + [InlineData(3840u, 2160u, 4u, 1u, 1u, 0x28000ul)] + [InlineData(640u, 360u, 16u, 1u, 1u, 0x6000ul)] + [InlineData(1920u, 1080u, 12u, 1u, 1u, 0ul)] + [InlineData(1920u, 1080u, 4u, 2u, 1u, 0ul)] + [InlineData(1920u, 1080u, 4u, 1u, 2u, 0ul)] + public void DccSliceSize_CoversOneMetadataBytePerColorBlock(uint width, uint height, uint bytesPerElement, uint levels, uint samples, ulong expected) + { + var description = Color(Base, 0x100, width, height, levels); + description.BytesPerBlock = bytesPerElement; + description.Samples = samples; + description.TileMode = GuestTileMode.RenderTarget; + description.Metadata.Kind = MetadataKind.Dcc; + description.Metadata.Range = new GuestSpan(0x200000, 0); + Assert.Equal(expected, description.DccSliceSize); + + description.TileMode = GuestTileMode.Linear; + Assert.Equal(0ul, description.DccSliceSize); + } + [Fact] public void Predicates_ReadTheDescription() { diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Pipelines/ShaderPipelineCacheTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Pipelines/ShaderPipelineCacheTests.cs index b6aec6f..4b3558d 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Pipelines/ShaderPipelineCacheTests.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Pipelines/ShaderPipelineCacheTests.cs @@ -303,6 +303,26 @@ public sealed class ShaderPipelineCacheTests : IDisposable Assert.Equal(expectedMask, description.StaticParameters.GetColorMask(0)); } + [Fact] + public void ColorTarget_WithoutAPixelStage_IsNotWritten() + { + var banks = Banks(); + banks.Context.RenderTargetMask = 0xF; + var programs = Programs(); + var resolution = new ColorTargetResolution( + default, 0x1000, 0x10000, new Extent2D(64, 64), 0, 0, 1, ColorComponentMap.Identity, false, false, default); + ColorTargetState[] colors = [new(in resolution, 0, new SharpEmu.Libs.Gpu.Buffers.ResourceSlotIdentifier(1, 1))]; + var rendering = new RenderingState { Samples = 1, ColorAttachmentCount = 1 }; + rendering.ColorAttachments[0] = new RenderingAttachment( + default, ImageLayout.ColorAttachmentOptimal, Format.R8G8B8A8Unorm, 0, 0, 0, 0, false, false, false, false, false); + + var description = ShaderPipelineCache.BuildGraphicsDescription( + colors, default, programs.VertexInput, null, banks.Context, in rendering, PrimitiveTopology.TriangleList, false, false, + programs.Vertex, programs.Pixel, SampleCountFlags.Count1Bit); + + Assert.Equal(0u, description.StaticParameters.GetColorMask(0)); + } + [Fact] public void NoAttachments_AtAnUnsupportedSampleCount_AreFatal() { diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorStateTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorStateTests.cs index 174018e..a046fdc 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorStateTests.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorStateTests.cs @@ -338,6 +338,19 @@ public sealed class RenderExecutorStateTests : IDisposable Assert.Equal((expectedColors, expectedExtent, expectedExtent), (rendering.ColorAttachmentCount, rendering.Width, rendering.Height)); } + [Fact] + public void Attachments_AColorTargetWithoutAnActivePixelShaderIsDroppedFromTheDepthPass() + { + var banks = Banks(withDepth: true); + banks.Context.ColorTargets[0] = RegisterWords.Color(ColorBase, 32, 32); + banks.Context.ShaderInterface.ColorShaderMask = 0; + _executor.DrawIndexed(1, banks, Indexed(3)); + + var rendering = _host.BegunRenderings[0]; + Assert.Equal((0u, 64u, 64u), (rendering.ColorAttachmentCount, rendering.Width, rendering.Height)); + Assert.Contains("create_graphics_pipeline colors=0 depth=True topology=TriangleList restart=False", _pipelines.Calls); + } + [Fact] public void Attachments_StencilFormatIsReportedOnlyWithAStencilAspect() { diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Vulkan/MultiplyHighDeviceTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Vulkan/MultiplyHighDeviceTests.cs new file mode 100644 index 0000000..7f7c968 --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Gpu/Vulkan/MultiplyHighDeviceTests.cs @@ -0,0 +1,69 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Buffers.Binary; +using SharpEmu.Libs.Gpu.Buffers; +using SharpEmu.Libs.Tests.Gpu.Images; +using SharpEmu.ShaderCompiler; +using SharpEmu.ShaderCompiler.Resources; +using SharpEmu.ShaderCompiler.Vulkan; +using Xunit; +using static SharpEmu.ShaderCompiler.Tests.Resources.ResourceTestProgram; + +namespace SharpEmu.Libs.Tests.Gpu.Vulkan; + +public sealed class MultiplyHighDeviceTests(HeadlessVulkanFixture fixture) : IClassFixture +{ + private const uint Factor = 0x9E37_79B9; + private const uint SignedFactor = 0x8000_0005; + private const ulong Addend = 0x1_FFFF_FFF0; + private const uint LaneBase = 0x7FFF_FFF0; + private const uint ThreadCount = 32; + + [Fact] + public void WideProductsMatchTheHost() + { + var vulkan = fixture.Vulkan; + if (!GatePrerequisites.Ready(vulkan, shaderInt64: true)) return; + var program = Program( + Vop2(0, "VAddI32", 10, Operand(LaneBase), Gen5Operand.Vector(0)), + MoveScalar(8, 8, Factor), + MoveScalar(16, 9, SignedFactor), + MoveScalar(24, 10, unchecked((uint)Addend)), + MoveScalar(32, 11, (uint)(Addend >> 32)), + Vop3(40, "VMadU64U32", 2, Gen5Operand.Scalar(8), Gen5Operand.Vector(10), Gen5Operand.Scalar(10)), + Vop3(48, "VMulHiU32", 4, Gen5Operand.Vector(10), Gen5Operand.Scalar(9)), + Vop3(56, "VMulHiI32", 5, Gen5Operand.Vector(10), Gen5Operand.Scalar(9)), + Sop2(64, "SMulHiU32", 12, Gen5Operand.Scalar(8), Gen5Operand.Scalar(9)), + MoveVectorFromScalar(68, 6, 12), + Vop2(72, "VLshlrevB32", 7, Operand(5), Gen5Operand.Vector(0)), + BufferAccess(76, "BufferStoreDwordx4", 4, vectorData: 2, dwords: 4, offsetEnabled: true, vectorAddress: 7), + BufferAccess(84, "BufferStoreDword", 4, offset: 16, vectorData: 6, offsetEnabled: true, vectorAddress: 7), + EndProgram(92)); + var (plan, resources, layout) = Prepare(program); + var request = new ShaderCompileRequest(plan, resources, layout) + { + LocalSizeX = ThreadCount, ThreadCountX = ThreadCount, WaveSize = 32, + }; + Assert.True(Gen5SpirvTranslator.TryCompileProgram(request, out var shader, out var error), error); + using var harness = new ImageTestHarness(vulkan); + using var runner = new LayoutComputeRunner(harness, request, shader.Spirv); + var result = runner.CreateBuffer(ThreadCount * 32); + var registers = new uint[256]; + registers[6] = ThreadCount * 32; + harness.Run(() => runner.Dispatch(registers, + new Dictionary { [DescriptorBindingKind.Buffers] = [result] }, 1)); + var actual = runner.ReadBack(result, 0, ThreadCount * 32); + for (var lane = 0u; lane < ThreadCount; lane++) + { + var value = LaneBase + lane; + var mad = (ulong)Factor * value + Addend; + var row = actual.AsSpan((int)(lane * 32)); + Assert.Equal((uint)mad, BinaryPrimitives.ReadUInt32LittleEndian(row)); + Assert.Equal((uint)(mad >> 32), BinaryPrimitives.ReadUInt32LittleEndian(row[4..])); + Assert.Equal((uint)(((ulong)value * SignedFactor) >> 32), BinaryPrimitives.ReadUInt32LittleEndian(row[8..])); + Assert.Equal((uint)(((long)(int)value * unchecked((int)SignedFactor)) >> 32), BinaryPrimitives.ReadUInt32LittleEndian(row[12..])); + Assert.Equal((uint)(((ulong)Factor * SignedFactor) >> 32), BinaryPrimitives.ReadUInt32LittleEndian(row[16..])); + } + } +} diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Vulkan/Wave64HalfMaskDeviceTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Vulkan/Wave64HalfMaskDeviceTests.cs new file mode 100644 index 0000000..5bda8e7 --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Gpu/Vulkan/Wave64HalfMaskDeviceTests.cs @@ -0,0 +1,269 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Buffers.Binary; +using SharpEmu.Libs.Gpu.Buffers; +using SharpEmu.Libs.Tests.Gpu.Images; +using SharpEmu.ShaderCompiler; +using SharpEmu.ShaderCompiler.Resources; +using SharpEmu.ShaderCompiler.Vulkan; +using Xunit; +using static SharpEmu.ShaderCompiler.Tests.Resources.ResourceTestProgram; + +namespace SharpEmu.Libs.Tests.Gpu.Vulkan; + +public sealed class Wave64HalfMaskDeviceTests(HeadlessVulkanFixture fixture) : IClassFixture +{ + private const uint Vcc = 106; + private const uint Exec = 126; + private static readonly Gen5Operand AllLanes = Gen5Operand.Source(193); + + private readonly List _instructions = []; + private uint _pc; + + private uint Add(Gen5ShaderInstruction instruction) + { + var pc = _pc; + _instructions.Add(instruction with { Pc = pc }); + _pc += 8; + return pc; + } + + private void Point(uint branchPc, uint targetPc) + { + var index = _instructions.FindIndex(instruction => instruction.Pc == branchPc); + var offset = (short)(((int)targetPc - (int)branchPc - 4) / 4); + _instructions[index] = _instructions[index] with { Words = [unchecked((uint)(ushort)offset)] }; + } + + private void Lane() + { + Add(Vop2(0, "VLshlrevB32", 2, Operand(3), Gen5Operand.Vector(1))); + Add(Vop2(0, "VAddI32", 2, Gen5Operand.Vector(0), Gen5Operand.Vector(2))); + } + + private uint CompareTo(string opcode, uint destination, Gen5Operand left, Gen5Operand right) => + Add(new Gen5ShaderInstruction(0, Gen5ShaderEncoding.Vop3, opcode, [0u, 0u], [left, right], [Gen5Operand.Scalar(destination)], + new Gen5Vop3Control(0, 0, 0, false, 0, destination))); + + private uint[] Run(uint localDataShareDwords = 0) + { + Add(Vop2(0, "VLshlrevB32", 5, Operand(2), Gen5Operand.Vector(2))); + Add(BufferAccess(0, "BufferStoreDword", 4, vectorData: 4, offsetEnabled: true, vectorAddress: 5)); + Add(EndProgram(0)); + var (plan, resources, layout) = Prepare(Program([.. _instructions])); + var request = new ShaderCompileRequest(plan, resources, layout) + { + LocalSizeX = 8, LocalSizeY = 8, ThreadCountX = 8, ThreadCountY = 8, WaveSize = 64, + LocalDataShareDwords = localDataShareDwords, + }; + Assert.True(Gen5SpirvTranslator.TryCompileProgram(request, out var shader, out var error), error); + Assert.NotNull(fixture.Vulkan); + using var harness = new ImageTestHarness(fixture.Vulkan); + using var runner = new LayoutComputeRunner(harness, request, shader.Spirv); + var output = runner.CreateBuffer(256); + var registers = new uint[256]; + registers[6] = 256; + harness.Run(() => runner.Dispatch(registers, + new Dictionary { [DescriptorBindingKind.Buffers] = [output] }, 1)); + var bytes = runner.ReadBack(output, 0, 256); + harness.AssertNoValidationMessages(); + var lanes = new uint[64]; + for (var lane = 0; lane < lanes.Length; lane++) + { + lanes[lane] = BinaryPrimitives.ReadUInt32LittleEndian(bytes.AsSpan(lane * sizeof(uint))); + } + + return lanes; + } + + private bool Ready() => GatePrerequisites.Ready(fixture.Vulkan, shaderInt64: true); + + [Fact] + public void MasksAcrossTheHalfBoundary_StayExactPerLaneAndWhenCounted() + { + if (!Ready()) return; + Lane(); + CompareTo("VCmpLtU32", 10, Gen5Operand.Vector(2), Operand(40)); + Add(Vop2(0, "VAndB32", 6, Operand(1), Gen5Operand.Vector(2))); + Add(Vopc(0, "VCmpEqU32", Operand(1), 6)); + Add(Sop2(0, "SAndB64", 12, Gen5Operand.Scalar(10), Gen5Operand.Scalar(Vcc))); + Add(Vop3(0, "VCndmaskB32", 4, Operand(0), Operand(1000), Gen5Operand.Scalar(12))); + Add(Sop1(0, "SBcnt1I32B64", 14, Gen5Operand.Scalar(12))); + Add(Vop2(0, "VAddU32", 4, Gen5Operand.Scalar(14), Gen5Operand.Vector(4))); + + var lanes = Run(); + for (uint lane = 0; lane < 64; lane++) + { + Assert.Equal((lane & 1) != 0 && lane < 40 ? 1020u : 20u, lanes[lane]); + } + } + + [Theory] + [InlineData(48u, 77u)] + [InlineData(8u, 77u)] + [InlineData(64u, 5u)] + public void AnExecBranch_IsDecidedForTheWholeWave(uint firstActiveLane, uint expected) + { + if (!Ready()) return; + Lane(); + Add(MoveScalar(0, 20, 5)); + Add(Vopc(0, "VCmpxLeU32", Operand(firstActiveLane), 2)); + var skip = Add(Branch(0, "SCbranchExecz", 0)); + Add(MoveScalar(0, 20, 77)); + var join = Add(Sop1(0, "SMovB64", Exec, AllLanes)); + Point(skip, join); + Add(MoveVectorFromScalar(0, 4, 20)); + + Assert.All(Run(), value => Assert.Equal(expected, value)); + } + + [Theory] + [InlineData(50u, 9u)] + [InlineData(10u, 9u)] + [InlineData(64u, 3u)] + public void AMaskWhoseSccIsRead_CountsBothHalves(uint threshold, uint expected) + { + if (!Ready()) return; + Lane(); + Add(MoveScalar(0, 20, 3)); + CompareTo("VCmpGeU32", 10, Gen5Operand.Vector(2), Operand(threshold)); + Add(Sop2(0, "SAndB64", 12, Gen5Operand.Scalar(10), Gen5Operand.Scalar(Exec))); + var skip = Add(Branch(0, "SCbranchScc0", 0)); + Add(MoveScalar(0, 20, 9)); + var join = Add(MoveVectorFromScalar(0, 4, 20)); + Point(skip, join); + + Assert.All(Run(), value => Assert.Equal(expected, value)); + } + + [Theory] + [InlineData(0u)] + [InlineData(64u)] + public void SharedMemoryWrittenByOneHalf_IsReadByTheOther(uint localDataShareDwords) + { + if (!Ready()) return; + Lane(); + Add(Vop2(0, "VLshlrevB32", 3, Operand(1), Gen5Operand.Vector(2))); + Add(Vop2(0, "VAddU32", 3, Gen5Operand.Vector(2), Gen5Operand.Vector(3))); + Add(Vop2(0, "VLshlrevB32", 6, Operand(2), Gen5Operand.Vector(2))); + Add(Vopc(0, "VCmpxGtU32", Operand(32), 2)); + Add(BufferAccess(0, "BufferLoadDword", 4, vectorData: 8, offsetEnabled: true, vectorAddress: 6)); + Add(Vop2(0, "VAddU32", 3, Gen5Operand.Vector(8), Gen5Operand.Vector(3))); + Add(DataShare(0, "DsWriteB32", false, [Gen5Operand.Vector(6), Gen5Operand.Vector(3)], [])); + Add(Sop1(0, "SMovB64", Exec, AllLanes)); + Add(Vop2(0, "VAndB32", 7, Operand(31), Gen5Operand.Vector(2))); + Add(Vop2(0, "VLshlrevB32", 7, Operand(2), Gen5Operand.Vector(7))); + Add(DataShare(0, "DsReadB32", false, [Gen5Operand.Vector(7)], [4])); + + var lanes = Run(localDataShareDwords); + for (uint lane = 0; lane < 64; lane++) + { + Assert.Equal((lane & 31) * 3, lanes[lane]); + } + } + + [Fact] + public void AnExecLoop_RunsEveryLaneItsOwnCount() + { + if (!Ready()) return; + Lane(); + Add(Vop2(0, "VAndB32", 3, Operand(7), Gen5Operand.Vector(2))); + Add(Vop2(0, "VAddU32", 3, Operand(1), Gen5Operand.Vector(3))); + Add(MoveVector(0, 4, 0)); + Add(Sop1(0, "SMovB64", 20, Gen5Operand.Scalar(Exec))); + var header = Add(Vopc(0, "VCmpxGtU32", Gen5Operand.Vector(3), 4)); + var exit = Add(Branch(0, "SCbranchExecz", 0)); + Add(Vop2(0, "VAddU32", 4, Operand(1), Gen5Operand.Vector(4))); + var back = Add(Branch(0, "SBranch", 0)); + var done = Add(Sop1(0, "SMovB64", Exec, Gen5Operand.Scalar(20))); + Point(exit, done); + Point(back, header); + + var lanes = Run(); + for (uint lane = 0; lane < 64; lane++) + { + Assert.Equal((lane & 7) + 1, lanes[lane]); + } + } + + [Theory] + [InlineData(40u, 140u)] + [InlineData(3u, 103u)] + [InlineData(64u, 100u)] + public void ReadFirstLane_TakesTheFirstActiveLaneOfTheWholeWave(uint firstActiveLane, uint expected) + { + if (!Ready()) return; + Lane(); + Add(Vop2(0, "VAddU32", 3, Operand(100), Gen5Operand.Vector(2))); + Add(Vopc(0, "VCmpxLeU32", Operand(firstActiveLane), 2)); + Add(ReadFirstLane(0, 20, 3)); + Add(Sop1(0, "SMovB64", Exec, AllLanes)); + Add(MoveVectorFromScalar(0, 4, 20)); + + Assert.All(Run(), value => Assert.Equal(expected, value)); + } + + [Fact] + public void ADemonsSoulsIndexRemap_CompilesOnTheDevice() + { + if (!Ready()) return; + uint[] words = + [ + 0xBEFC03FFu, 0xA65CD91Eu, 0xBF960000u, 0xBFA00003u, 0xD7460002u, 0x04010C0Cu, 0xF4201A84u, 0xFA000000u, + 0xBF8CC07Fu, 0x7DA8046Au, 0xBF88002Au, 0xF4200304u, 0xFA000004u, 0xBF8CC07Fu, 0x7E000C0Cu, 0xBF070C80u, + 0x8588807Eu, 0x7E005700u, 0x100000FFu, 0x4F800000u, 0x7E060F00u, 0xD5766A00u, 0x0202060Cu, 0x7D8A0280u, + 0x4C020080u, 0x02000101u, 0xD56A0001u, 0x00020700u, 0x4C000303u, 0x4A020303u, 0x02000101u, 0xD56A0000u, + 0x00020500u, 0xD5690001u, 0x0002000Cu, 0x4C060302u, 0x7D8C02F9u, 0x06068A02u, 0x7D86060Cu, 0x87EA6A0Au, + 0x50000080u, 0xD5286A00u, 0x002A00C1u, 0xD5010000u, 0x002200C1u, 0xD5690000u, 0x0002000Cu, 0x4C000102u, + 0xE0002000u, 0x80000000u, 0xBF8C3F70u, 0xE0102000u, 0x80010002u, 0xBF810000u, + ]; + var bytes = new byte[words.Length * sizeof(uint)]; + for (var index = 0; index < words.Length; index++) + BinaryPrimitives.WriteUInt32LittleEndian(bytes.AsSpan(index * sizeof(uint)), words[index]); + Assert.True(Gen5ShaderTranslator.TryDecodeProgram(new SharpEmu.HLE.CpuContext(new WordMemory(bytes), SharpEmu.HLE.Generation.Gen5), + 0x1000, out var program, out var decodeError), decodeError); + var (plan, resources, layout) = Prepare(program, userDataCount: 12); + var request = new ShaderCompileRequest(plan, resources, layout) + { + LocalSizeX = 64, WaveSize = 64, EnableExecGuardElision = false, + ComputeSystemRegisters = new Gen5ComputeSystemRegisters(12, null, null, null), + }; + Assert.True(Gen5SpirvTranslator.TryCompileProgram(request, out var shader, out var error), error); + Assert.NotNull(fixture.Vulkan); + using var harness = new ImageTestHarness(fixture.Vulkan); + using var runner = new LayoutComputeRunner(harness, request, shader.Spirv); + harness.AssertNoValidationMessages(); + } + + private sealed class WordMemory(byte[] bytes) : SharpEmu.HLE.ICpuMemory + { + public bool TryRead(ulong address, Span destination) + { + if (address < 0x1000 || address - 0x1000 > (ulong)(bytes.Length - destination.Length)) return false; + bytes.AsSpan((int)(address - 0x1000), destination.Length).CopyTo(destination); + return true; + } + + public bool TryWrite(ulong address, ReadOnlySpan source) => false; + } + + [Fact] + public void ASavedExec_RestoresAfterAHalfMaskedBody() + { + if (!Ready()) return; + Lane(); + Add(MoveVector(0, 4, 1)); + CompareTo("VCmpGeU32", 10, Gen5Operand.Vector(2), Operand(28)); + Add(Sop1(0, "SAndSaveexecB64", 12, Gen5Operand.Scalar(10))); + Add(MoveVector(0, 4, 2)); + Add(Sop2(0, "SOrB64", Exec, Gen5Operand.Scalar(Exec), Gen5Operand.Scalar(12))); + Add(Vop2(0, "VAddU32", 4, Operand(10), Gen5Operand.Vector(4))); + + var lanes = Run(); + for (uint lane = 0; lane < 64; lane++) + { + Assert.Equal(lane >= 28 ? 12u : 11u, lanes[lane]); + } + } +} diff --git a/tests/SharpEmu.Libs.Tests/VideoOut/RenderHostDeviceTests.cs b/tests/SharpEmu.Libs.Tests/VideoOut/RenderHostDeviceTests.cs index 6d87aff..136ea03 100644 --- a/tests/SharpEmu.Libs.Tests/VideoOut/RenderHostDeviceTests.cs +++ b/tests/SharpEmu.Libs.Tests/VideoOut/RenderHostDeviceTests.cs @@ -524,6 +524,46 @@ public sealed unsafe partial class RenderHostDeviceTests : IClassFixture> 32), 32, 0]], + DeviceAddressRanges = [new DeviceAddressRange(0, address, 2 * GuestBufferCache.CachingPageSize, true, false)], + }; + GuestGpuMemoryHook.Attach(harness.Gpu); + try + { + presenter.Run(() => + { + using var preparation = presenter.RenderHost.BeginPreparation(); + var prepared = presenter.RenderHost.PrepareBindings(new ShaderStageResources(program, snapshot)); + presenter.RenderHost.PrepareDeviceAddresses(); + var owner = harness.Cache.GetBuffer(harness.Cache.FindBuffer(written, 32)); + presenter.RenderHost.BindResources(prepared); + Assert.Same(owner, harness.Cache.GetBuffer(harness.Cache.FindBuffer(written, 32))); + Assert.Equal(1, harness.Cache.BufferCount); + }); + } + finally + { + GuestGpuMemoryHook.Attach(null); + } + harness.Shutdown(); + } + [Fact] public void PrepareBindings_PrivateBufferUploadsCurrentBytesAndStillRejectsGpuWrites() { diff --git a/tests/SharpEmu.ShaderCompiler.Tests/Gen5MultiplyHighTests.cs b/tests/SharpEmu.ShaderCompiler.Tests/Gen5MultiplyHighTests.cs new file mode 100644 index 0000000..3bed45c --- /dev/null +++ b/tests/SharpEmu.ShaderCompiler.Tests/Gen5MultiplyHighTests.cs @@ -0,0 +1,68 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Runtime.InteropServices; +using SharpEmu.ShaderCompiler.Tests.Resources; +using SharpEmu.ShaderCompiler.Vulkan; +using Xunit; +using static SharpEmu.ShaderCompiler.Tests.Resources.ResourceTestProgram; + +namespace SharpEmu.ShaderCompiler.Tests; + +public sealed class Gen5MultiplyHighTests +{ + [Theory] + [InlineData("VMulHiU32", false)] + [InlineData("VMulHiU32U24", false)] + [InlineData("VMulHiI32", true)] + [InlineData("VMadU64U32", false)] + public void VectorProducts_UseExtendedMultiplies(string opcode, bool signed) + { + var program = Program( + Vop3(0, opcode, 2, Gen5Operand.Vector(4), Gen5Operand.Scalar(8), Gen5Operand.Scalar(10)), + EndProgram(8)); + AssertExtendedMultiply(program, signed); + } + + [Theory] + [InlineData("SMulHiU32", false)] + [InlineData("SMulHiI32", true)] + public void ScalarProducts_UseExtendedMultiplies(string opcode, bool signed) + { + var program = Program( + Sop2(0, opcode, 12, Gen5Operand.Scalar(8), Gen5Operand.Scalar(9)), + EndProgram(4)); + AssertExtendedMultiply(program, signed); + } + + private static void AssertExtendedMultiply(Gen5ShaderProgram program, bool signed) + { + var (plan, resources, layout) = Prepare(program); + var request = new ShaderCompileRequest(plan, resources, layout) { LocalSizeX = 1, ThreadCountX = 1 }; + Assert.True(Gen5SpirvTranslator.TryCompileProgram(request, out var shader, out var error), error); + var inspector = new SpirvModuleInspector(shader.Spirv); + Assert.Contains((ushort)(signed ? SpirvOp.SMulExtended : SpirvOp.UMulExtended), inspector.Opcodes); + Assert.Empty(WideMultiplies(shader.Spirv)); + } + + private static List WideMultiplies(byte[] spirv) + { + var words = MemoryMarshal.Cast(spirv); + var wideTypes = new HashSet(); + var multiplies = new List(); + for (var index = 5; index < words.Length; index += (int)(words[index] >> 16)) + { + var opcode = (ushort)(words[index] & 0xFFFF); + if (opcode == (ushort)SpirvOp.TypeInt && words[index + 2] == 64) + { + wideTypes.Add(words[index + 1]); + } + else if (opcode == (ushort)SpirvOp.IMul && wideTypes.Contains(words[index + 1])) + { + multiplies.Add(words[index + 2]); + } + } + + return multiplies; + } +} diff --git a/tests/SharpEmu.ShaderCompiler.Tests/Gen5SharedMemoryBarrierTests.cs b/tests/SharpEmu.ShaderCompiler.Tests/Gen5SharedMemoryBarrierTests.cs index ae087fd..ffb5eda 100644 --- a/tests/SharpEmu.ShaderCompiler.Tests/Gen5SharedMemoryBarrierTests.cs +++ b/tests/SharpEmu.ShaderCompiler.Tests/Gen5SharedMemoryBarrierTests.cs @@ -11,12 +11,14 @@ namespace SharpEmu.ShaderCompiler.Tests; public sealed class Gen5SharedMemoryBarrierTests { [Theory] - [InlineData(64u, 64u, false, 2)] - [InlineData(64u, 64u, true, 2)] - [InlineData(32u, 64u, false, 0)] - [InlineData(64u, 128u, false, 0)] + [InlineData(64u, 64u, false, false, 2)] + [InlineData(64u, 64u, true, false, 2)] + [InlineData(32u, 64u, false, false, 0)] + [InlineData(64u, 128u, false, false, 0)] + [InlineData(64u, 64u, false, true, 2)] + [InlineData(32u, 64u, false, true, 0)] public void SharedMemoryPhases_SynchronizeOnlySingleWaveGroups( - uint waveSize, uint threadCount, bool explicitBarrier, int expectedBarriers) + uint waveSize, uint threadCount, bool explicitBarrier, bool splitBlocks, int expectedBarriers) { var instructions = new List { @@ -25,6 +27,8 @@ public sealed class Gen5SharedMemoryBarrierTests }; if (explicitBarrier) instructions.Add(new(8, Gen5ShaderEncoding.Sopp, "SBarrier", [0u], [], [], null)); + if (splitBlocks) + instructions.Add(new(8, Gen5ShaderEncoding.Sopp, "SBranch", [0u], [], [], null)); instructions.Add(new(12, Gen5ShaderEncoding.Ds, "DsReadB32", [0u, 0u], [Gen5Operand.Vector(0)], [Gen5Operand.Vector(2)], new Gen5DataShareControl(0, 0, false))); instructions.Add(new(20, Gen5ShaderEncoding.Sopp, "SEndpgm", [0u], [], [], null)); diff --git a/tests/SharpEmu.ShaderCompiler.Tests/Gen5Wave64HalfMaskAnalysisTests.cs b/tests/SharpEmu.ShaderCompiler.Tests/Gen5Wave64HalfMaskAnalysisTests.cs new file mode 100644 index 0000000..d1d8bd9 --- /dev/null +++ b/tests/SharpEmu.ShaderCompiler.Tests/Gen5Wave64HalfMaskAnalysisTests.cs @@ -0,0 +1,221 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.ShaderCompiler; +using SharpEmu.ShaderCompiler.Ir; +using Xunit; + +namespace SharpEmu.ShaderCompiler.Tests; + +public sealed class Gen5Wave64HalfMaskAnalysisTests +{ + private const uint Vcc = 106; + private const uint Exec = 126; + + private readonly List _program = []; + private uint _pc; + + private uint Add(string opcode, Gen5ShaderEncoding encoding, Gen5Operand[] destinations, Gen5Operand[] sources, + Gen5InstructionControl? control = null, uint word = 0) + { + var pc = _pc; + _program.Add(new Gen5ShaderInstruction(pc, encoding, opcode, [word], sources, destinations, control)); + _pc += 4; + return pc; + } + + private uint CompareToVcc() => + Add("VCmpLtU32", Gen5ShaderEncoding.Vopc, [], [Gen5Operand.Vector(0), Gen5Operand.Vector(1)]); + + private uint CompareTo(uint destination) => + Add("VCmpLtU32", Gen5ShaderEncoding.Vop3, [Gen5Operand.Scalar(destination)], [Gen5Operand.Vector(0), Gen5Operand.Vector(1)], + new Gen5Vop3Control(0, 0, 0, false, 0, destination)); + + private uint CompareToExec() => + Add("VCmpxLtU32", Gen5ShaderEncoding.Vopc, [], [Gen5Operand.Vector(0), Gen5Operand.Vector(1)]); + + private uint Select(uint mask) => + Add("VCndmaskB32", Gen5ShaderEncoding.Vop3, [Gen5Operand.Vector(2)], [Gen5Operand.Vector(3), Gen5Operand.Vector(4), Gen5Operand.Scalar(mask)], + new Gen5Vop3Control(0, 0, 0, false, 0, null)); + + private uint Scalar1(string opcode, uint destination, params Gen5Operand[] sources) => + Add(opcode, Gen5ShaderEncoding.Sop1, [Gen5Operand.Scalar(destination)], sources); + + private uint Scalar2(string opcode, uint destination, params Gen5Operand[] sources) => + Add(opcode, Gen5ShaderEncoding.Sop2, [Gen5Operand.Scalar(destination)], sources); + + private uint Branch(string opcode) => Add(opcode, Gen5ShaderEncoding.Sopp, [], [], word: 0xBF840000u); + + private uint DataShare(string opcode) => + Add(opcode, Gen5ShaderEncoding.Ds, opcode.StartsWith("DsRead", StringComparison.Ordinal) ? [Gen5Operand.Vector(5)] : [], + [Gen5Operand.Vector(0), Gen5Operand.Vector(1)], new Gen5DataShareControl(0, 0, false)); + + private uint End() => Add("SEndpgm", Gen5ShaderEncoding.Sopp, [], [], word: 0xBF810000u); + + private void PointBranch(uint branchPc, uint targetPc) + { + var index = _program.FindIndex(instruction => instruction.Pc == branchPc); + var original = _program[index]; + var offset = (ushort)(short)((int)(targetPc - branchPc - 4) / 4); + _program[index] = original with { Words = [(original.Words[0] & 0xFFFF0000u) | offset] }; + } + + private Gen5Wave64HalfMaskPlan Analyze() => + Gen5Wave64HalfMaskAnalysis.Analyze(new Gen5ShaderProgram(0x1000, _program), new HashSet()) + ?? throw new InvalidOperationException("analysis gave up"); + + [Fact] + public void ACompareReadPerLane_NeedsNoWholeWaveMask() + { + var compare = CompareTo(20); + var select = Select(20); + End(); + + var plan = Analyze(); + Assert.Equal(20u, plan.HalfMaskWrites[compare]); + Assert.False(plan.ExactPairsBefore.ContainsKey(select)); + } + + [Fact] + public void ABranchOnVcc_CombinesVccFirst() + { + CompareToVcc(); + var branch = Branch("SCbranchVccz"); + End(); + PointBranch(branch, _pc - 4); + + Assert.Equal(new[] { Vcc }, Analyze().ExactPairsBefore[branch]); + } + + [Fact] + public void ABranchOnExec_CombinesExecFirst() + { + CompareToExec(); + var branch = Branch("SCbranchExecz"); + End(); + PointBranch(branch, _pc - 4); + + Assert.Equal(new[] { Exec }, Analyze().ExactPairsBefore[branch]); + } + + [Fact] + public void ABitwiseMaskOperationWithADeadScc_StaysHalf() + { + CompareTo(20); + CompareToVcc(); + var and = Scalar2("SAndB64", 22, Gen5Operand.Scalar(20), Gen5Operand.Scalar(Vcc)); + var select = Select(22); + End(); + + var plan = Analyze(); + Assert.False(plan.ExactPairsBefore.ContainsKey(and)); + Assert.False(plan.ExactPairsBefore.ContainsKey(select)); + } + + [Fact] + public void ABitwiseMaskOperationWhoseSccIsRead_CombinesItsInputs() + { + CompareTo(20); + CompareToVcc(); + var and = Scalar2("SAndB64", 22, Gen5Operand.Scalar(20), Gen5Operand.Scalar(Vcc)); + var branch = Branch("SCbranchScc0"); + End(); + PointBranch(branch, _pc - 4); + + var plan = Analyze(); + Assert.Equal(new[] { 20u, Vcc }, plan.ExactPairsBefore[and]); + Assert.False(plan.ExactPairsBefore.ContainsKey(branch)); + } + + [Fact] + public void AScalarReadOfAMaskHalf_CombinesThePair() + { + CompareTo(20); + var move = Scalar1("SMovB32", 30, Gen5Operand.Scalar(21)); + End(); + + Assert.Equal(new[] { 20u }, Analyze().ExactPairsBefore[move]); + } + + [Fact] + public void ACombinedMask_IsNotCombinedAgain() + { + CompareTo(20); + var first = Scalar1("SMovB32", 30, Gen5Operand.Scalar(20)); + var second = Scalar1("SMovB32", 31, Gen5Operand.Scalar(21)); + End(); + + var plan = Analyze(); + Assert.Equal(new[] { 20u }, plan.ExactPairsBefore[first]); + Assert.False(plan.ExactPairsBefore.ContainsKey(second)); + } + + [Fact] + public void ASaveexecCopiesAHalfExec_SoTheRestoreStaysHalf() + { + CompareTo(20); + var save = Scalar1("SAndSaveexecB64", 24, Gen5Operand.Scalar(20)); + Select(Exec); + var restore = Scalar2("SOrB64", Exec, Gen5Operand.Scalar(Exec), Gen5Operand.Scalar(24)); + var read = Scalar1("SMovB32", 30, Gen5Operand.Scalar(24)); + End(); + + var plan = Analyze(); + Assert.False(plan.ExactPairsBefore.ContainsKey(save)); + Assert.False(plan.ExactPairsBefore.ContainsKey(restore)); + Assert.Equal(new[] { 24u }, plan.ExactPairsBefore[read]); + } + + [Fact] + public void ALoopExitOnExec_CombinesExecEveryIteration() + { + var header = CompareToExec(); + var exit = Branch("SCbranchExecz"); + Select(Exec); + var back = Branch("SBranch"); + var end = End(); + PointBranch(exit, end); + PointBranch(back, header); + + var plan = Analyze(); + Assert.Equal(new[] { Exec }, plan.ExactPairsBefore[exit]); + Assert.Contains(header, plan.HalfMaskWrites.Keys); + } + + [Fact] + public void ASharedReadAfterAWrite_WaitsForTheOtherHalf() + { + var write = DataShare("DsWriteB32"); + var read = DataShare("DsReadB32"); + var again = DataShare("DsReadB32"); + End(); + + var plan = Analyze(); + Assert.DoesNotContain(write, plan.SharedMemoryBarriersBefore); + Assert.Contains(read, plan.SharedMemoryBarriersBefore); + Assert.DoesNotContain(again, plan.SharedMemoryBarriersBefore); + } + + [Fact] + public void ACombineBarrier_AlsoOrdersSharedMemory() + { + DataShare("DsWriteB32"); + CompareTo(20); + var move = Scalar1("SMovB32", 30, Gen5Operand.Scalar(20)); + var read = DataShare("DsReadB32"); + End(); + + var plan = Analyze(); + Assert.True(plan.ExactPairsBefore.ContainsKey(move)); + Assert.DoesNotContain(read, plan.SharedMemoryBarriersBefore); + } + + [Fact] + public void ADynamicJump_GivesUp() + { + Scalar1("SSetpcB64", 0, Gen5Operand.Scalar(4)); + End(); + + Assert.Null(Gen5Wave64HalfMaskAnalysis.Analyze(new Gen5ShaderProgram(0x1000, _program), new HashSet())); + } +} diff --git a/tests/SharpEmu.ShaderCompiler.Tests/Resources/DirectImageTableTests.cs b/tests/SharpEmu.ShaderCompiler.Tests/Resources/DirectImageTableTests.cs index c7ef505..0be40ae 100644 --- a/tests/SharpEmu.ShaderCompiler.Tests/Resources/DirectImageTableTests.cs +++ b/tests/SharpEmu.ShaderCompiler.Tests/Resources/DirectImageTableTests.cs @@ -30,6 +30,41 @@ public sealed class DirectImageTableTests EndProgram(72)); } + internal static Gen5ShaderProgram CreateWaveIndexedReadLaneProgram(bool selfAddressed, bool restoreExec = true) + { + var instructions = new List + { + ScalarLoad(0, 0, 16, immediateOffset: 0x80), + Sop1(8, "SFF1I32B32", 18, Gen5Operand.Scalar(16)), + Sop2(12, "SMulI32", 19, Gen5Operand.Scalar(18), new Gen5Operand(Gen5OperandKind.LiteralConstant, 0x90)), + MoveVectorFromScalar(16, 34, 18), + Vop2(20, "VLshlrevB32", 15, Operand(4), Gen5Operand.Vector(34)), + Vop3(24, "VLshlAddU32", selfAddressed ? 15u : 16u, Gen5Operand.Vector(15), Operand(3), Gen5Operand.Vector(15)), + Sop2(32, "SLshlB32", 20, Operand(1), Gen5Operand.Scalar(18)), + Sop2(36, "SXorB32", 16, Gen5Operand.Scalar(20), Gen5Operand.Scalar(16)), + Vop2(40, "VAddI32", 15, new Gen5Operand(Gen5OperandKind.LiteralConstant, 0x40), Gen5Operand.Vector(selfAddressed ? 15u : 16u)), + GlobalMemory(44, "GlobalLoadDword", 0, 15, 22, 0), + Sop1(52, "SMovB64", 12, Gen5Operand.Scalar(126)), + Sop1(56, "SFF1I32B64", 24, Gen5Operand.Scalar(12)), + new(60, Gen5ShaderEncoding.Vop3, "VReadlaneB32", [0u, 0u], + [Gen5Operand.Vector(22), Gen5Operand.Scalar(24), Gen5Operand.Scalar(24)], [Gen5Operand.Scalar(106)], null), + new(68, Gen5ShaderEncoding.Vop3, "VCmpEqU32", [0u, 0u], + [Gen5Operand.Scalar(106), Gen5Operand.Vector(22)], [Gen5Operand.Scalar(14)], new Gen5Vop3Control(0, 0, 0, false, 0, 14)), + Sop1(76, "SAndSaveexecB64", 28, Gen5Operand.Scalar(14)), + Branch(80, "SCbranchExecz", 8), + Sop2(84, "SLshlB32", 106, Gen5Operand.Scalar(106), Operand(5)), + Sop2(88, "SAddI32", 107, Gen5Operand.Scalar(106), Operand(16)), + ScalarLoad(92, 0, 4, 4, immediateOffset: 0x100, dynamicOffsetRegister: 106), + ScalarLoad(100, 0, 8, 4, immediateOffset: 0x100, dynamicOffsetRegister: 107), + Image(108, "ImageLoad", 4, dmask: 1, vectorAddress: 1), + Sop2(116, "SAndn2B64", 12, Gen5Operand.Scalar(12), Gen5Operand.Scalar(14)), + restoreExec ? Sop1(120, "SMovB64", 126, Gen5Operand.Scalar(28)) : Nop(120), + Branch(124, "SCbranchScc1", -18), + EndProgram(128), + }; + return Program([.. instructions]); + } + public static Gen5ShaderProgram CreateProgram(uint mask = 1, bool split = true, bool bitScan = true) { var instructions = new List @@ -107,41 +142,72 @@ public sealed class DirectImageTableTests Assert.Equal(0u, selector.KeyBound); Assert.Equal(new WaveIndexedImageSelector(0x80, 0x40, 0x90), selector.WaveIndexed); - bool Read(ulong address, out uint word) - { - word = 0; - if (address == 0x1000 + 0x80) - { - word = (1u << 1) | (1u << 4); - return true; - } + var snapshot = new ResourceSnapshot(); + var specialization = new ResourceSpecialization(); + Assert.True(ResourceMaterializer.Materialize(plan, Inputs([0x1000, 0], readCleanMemory: ReadWaveIndexedMemory), ref snapshot, ref specialization)); + Assert.Equal(2, snapshot.Images.Length); + } - if (address == 0x1000 + 0x40 + 0x90 || address == 0x1000 + 0x40 + 4 * 0x90) - { - word = address == 0x1000 + 0x40 + 0x90 ? 2u : 5u; - return true; - } - - if (address < 0x1000 + 0x100 || address >= 0x1000 + 0x100 + 6 * 32) - return false; - var relative = address - 0x1000 - 0x100; - var record = relative / 32; - if (record is not (2 or 5)) - return false; - word = (relative % 32 / 4) switch - { - 0 => (record & 1) == 0 ? 0x2000u : 0x1000u, - 1 => 20u << 20, - 3 => 0xFACu | (9u << 28), - _ => 0, - }; - return true; - } + [Theory] + [InlineData(false)] + [InlineData(true)] + public void ReadLaneWaterfallSelectsWaveIndexedDescriptors(bool selfAddressed) + { + var program = CreateWaveIndexedReadLaneProgram(selfAddressed); + var plan = ShaderResourcePlan.Extract(program, ShaderStage.Compute, Hash, 0, 2); + var selector = plan.DescriptorSources[(int)plan.Info.Images[0].Source].IndirectImage!; + Assert.True(selector.Dense, "dense"); + Assert.Equal(0x100u, selector.TableOffset); + Assert.Equal(new WaveIndexedImageSelector(0x80, 0x40, 0x90), selector.WaveIndexed); + Assert.True(Assert.Single(plan.IndirectImages).KeyIsAddressOffset, "address-offset key"); var snapshot = new ResourceSnapshot(); var specialization = new ResourceSpecialization(); - Assert.True(ResourceMaterializer.Materialize(plan, Inputs([0x1000, 0], readCleanMemory: Read), ref snapshot, ref specialization)); + Assert.True(ResourceMaterializer.Materialize(plan, Inputs([0x1000, 0], readCleanMemory: ReadWaveIndexedMemory), ref snapshot, ref specialization, out var failure), $"materialize {failure}"); Assert.Equal(2, snapshot.Images.Length); + var resources = ResourceMaterializer.ApplyTo(plan, specialization); + var layout = BindingLayout.Allocate(resources.Info, BindingLayout.CollectUserDataRegisters(program, 0, 2), + false, ShaderCompileRequest.RequiresFlattenedTable(plan, resources), false); + var request = new ShaderCompileRequest(plan, resources, layout) { LocalSizeX = 64, ThreadCountX = 64 }; + Assert.True(Gen5SpirvTranslator.TryCompileProgram(request, out _, out var error), error); + } + + [Fact] + public void ReadLaneWithoutRestoredExecutionIsNotWaveIndexed() + { + Assert.Throws(() => + ShaderResourcePlan.Extract(CreateWaveIndexedReadLaneProgram(selfAddressed: true, restoreExec: false), ShaderStage.Compute, Hash, 0, 2)); + } + + private static bool ReadWaveIndexedMemory(ulong address, out uint word) + { + word = 0; + if (address == 0x1000 + 0x80) + { + word = (1u << 1) | (1u << 4); + return true; + } + + if (address == 0x1000 + 0x40 + 0x90 || address == 0x1000 + 0x40 + 4 * 0x90) + { + word = address == 0x1000 + 0x40 + 0x90 ? 2u : 5u; + return true; + } + + if (address < 0x1000 + 0x100 || address >= 0x1000 + 0x100 + 6 * 32) + return false; + var relative = address - 0x1000 - 0x100; + var record = relative / 32; + if (record is not (2 or 5)) + return false; + word = (relative % 32 / 4) switch + { + 0 => (record & 1) == 0 ? 0x2000u : 0x1000u, + 1 => 20u << 20, + 3 => 0xFACu | (9u << 28), + _ => 0, + }; + return true; } public static (ResourceSnapshot Snapshot, ShaderCompileRequest Request) PrepareMixedDimensions(uint mask, bool arrayFirst)