Des dcc fixes (#1000)

* [SpirvTranslator] Take high products from extended multiplies

Co-authored-by: TarkusR <90155059+TarkusR@users.noreply.github.com>

* [SpirvTranslator] Keep shader flags in uint variables

Co-authored-by: TarkusR <90155059+TarkusR@users.noreply.github.com>

* [SpirvTranslator] Synchronize shared memory phases in multi block shaders

Co-authored-by: TarkusR <90155059+TarkusR@users.noreply.github.com>

* [GPU] Merge device address ranges before buffers are bound

Co-authored-by: TarkusR <90155059+TarkusR@users.noreply.github.com>

* [GPU] Track per-slice guest DCC clear metadata

* [GPU] Materialize guest DCC clears in color and sampled images

* [GPU] Preserve color targets without pixel shader exports

* [ShaderCompiler] Track wave-indexed image descriptor waterfalls

* [ShaderCompiler] Analyze wave64 half masks and shared-memory hazards

* [ShaderCompiler] Pass guest LDS size to SPIR-V compilation

* [SpirvTranslator] Synchronize wave64 half masks through LDS exchange

---------

Co-authored-by: TarkusR <90155059+TarkusR@users.noreply.github.com>

src/SharpEmu.Libs/Gpu/Images/GuestImageCache.Metadata.cs

@@ -84,6 +84,70 @@ public sealed partial class GuestImageCache
         return false;
     }
 
+    public static bool IsDccClearCode(byte code) => code is 0x00 or 0x20 or 0x40 or 0x80 or 0xc0;
+
+    public bool TryReadGuestDccClear(ulong metadataAddress, ulong sliceSize, uint slice, out ulong sliceAddress, out byte code)
+    {
+        sliceAddress = 0;
+        code = 0;
+        if (metadataAddress == 0 || sliceSize == 0 || sliceSize > int.MaxValue ||
+            (ulong)slice > (ulong.MaxValue - metadataAddress) / sliceSize)
+        {
+            return false;
+        }
+
+        var address = metadataAddress + (ulong)slice * sliceSize;
+        if (!IsValidRange(address, sliceSize) || _bufferCache.HasGpuDirtyBytes(address, sliceSize))
+        {
+            return false;
+        }
+
+        Span<byte> first = stackalloc byte[1];
+        if (!_backing.TryReadBacking(address, first) || !IsDccClearCode(first[0]))
+        {
+            return false;
+        }
+
+        var buffer = System.Buffers.ArrayPool<byte>.Shared.Rent((int)Math.Min(sliceSize, 64 * 1024));
+        try
+        {
+            for (ulong position = 0; position < sliceSize;)
+            {
+                var span = buffer.AsSpan(0, (int)Math.Min(sliceSize - position, (ulong)buffer.Length));
+                if (!_backing.TryReadBacking(address + position, span) || span.IndexOfAnyExcept(first[0]) >= 0)
+                {
+                    return false;
+                }
+
+                position += (ulong)span.Length;
+            }
+        }
+        finally
+        {
+            System.Buffers.ArrayPool<byte>.Shared.Return(buffer);
+        }
+
+        sliceAddress = address;
+        code = first[0];
+        return true;
+    }
+
+    public void SynchronizeGuestDccMetadata(ulong metadataAddress, ulong sliceSize, uint baseLayer, uint layerCount)
+    {
+        if (metadataAddress == 0 || sliceSize == 0 || layerCount == 0 ||
+            (ulong)baseLayer + layerCount > (ulong.MaxValue - metadataAddress) / sliceSize)
+        {
+            return;
+        }
+
+        var address = metadataAddress + (ulong)baseLayer * sliceSize;
+        var size = (ulong)layerCount * sliceSize;
+        if (IsValidRange(address, size) && _bufferCache.HasGpuDirtyBytes(address, size))
+        {
+            _ = _bufferCache.TrySynchronizeCpuRead(address, size);
+        }
+    }
+
     public bool SetMetadataSlice(ulong address, uint slice, bool isClear)
     {
         using var held = _lock.Hold();

src/SharpEmu.Libs/Gpu/Images/ImageDescription.cs

@@ -245,6 +245,24 @@ public struct ImageDescription
 
     public readonly bool IsTiledRenderTarget => TileMode == GuestTileMode.RenderTarget || IsSupportedStandard64RenderTarget;
 
+    public readonly ulong DccSliceSize
+    {
+        get
+        {
+            const ulong DccBlockBytes = 4096;
+            if (Metadata.Kind != MetadataKind.Dcc || TileMode != GuestTileMode.RenderTarget || Resources.Levels != 1 || Samples != 1 ||
+                Extent.Width == 0 || Extent.Height == 0 || BytesPerBlock is 0 or > 16 || !System.Numerics.BitOperations.IsPow2(BytesPerBlock))
+            {
+                return 0;
+            }
+
+            var coverageBits = 20 - System.Numerics.BitOperations.TrailingZeroCount(BytesPerBlock);
+            var blockWidth = 1UL << ((coverageBits + 1) / 2);
+            var blockHeight = 1UL << (coverageBits / 2);
+            return (Extent.Width + blockWidth - 1) / blockWidth * ((Extent.Height + blockHeight - 1) / blockHeight) * DccBlockBytes;
+        }
+    }
+
     public static bool IsSupportedDisplayRenderTargetTileMode(GuestTileMode tileMode) => tileMode == GuestTileMode.RenderTarget;
 
     public static bool CanUseDisplayNativeWithoutUpload(DisplayCompression compression, bool renderTarget, bool gpuModified, bool guestModified) =>

src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.ColorTarget.cs

@@ -46,18 +46,20 @@ public static partial class ImageRequestBuilders
     {
         ClearColorValue value = default;
         var supported = hasDcc && PackedClearValue.TryDecodeColor(format, packedClear, out value);
-        var fixedSupported = hasDcc && format switch
-        {
-            Format.R8Unorm or Format.R8G8Unorm or Format.R8G8B8A8Unorm or Format.R8G8B8A8Srgb or Format.B8G8R8A8Unorm or Format.B8G8R8A8Srgb or
-            Format.A2B10G10R10UnormPack32 or Format.A2R10G10B10UnormPack32 or Format.R5G6B5UnormPack16 or Format.A1R5G5B5UnormPack16 or
-            Format.R4G4B4A4UnormPack16 or Format.R16Unorm or Format.R16G16Unorm or Format.R16G16B16A16Unorm or Format.R16Sfloat or
-            Format.R16G16Sfloat or Format.R16G16B16A16Sfloat or Format.R32Sfloat or Format.R32G32Sfloat or Format.R32G32B32A32Sfloat or
-            Format.B10G11R11UfloatPack32 => true,
-            _ => false,
-        };
+        var fixedSupported = hasDcc && SupportsDccFixedClear(format);
         return (supported, fixedSupported, supported ? value : default);
     }
 
+    public static bool SupportsDccFixedClear(Format format) => format switch
+    {
+        Format.R8Unorm or Format.R8G8Unorm or Format.R8G8B8A8Unorm or Format.R8G8B8A8Srgb or Format.B8G8R8A8Unorm or Format.B8G8R8A8Srgb or
+        Format.A2B10G10R10UnormPack32 or Format.A2R10G10B10UnormPack32 or Format.R5G6B5UnormPack16 or Format.A1R5G5B5UnormPack16 or
+        Format.R4G4B4A4UnormPack16 or Format.R16Unorm or Format.R16G16Unorm or Format.R16G16B16A16Unorm or Format.R16Sfloat or
+        Format.R16G16Sfloat or Format.R16G16B16A16Sfloat or Format.R32Sfloat or Format.R32G32Sfloat or Format.R32G32B32A32Sfloat or
+        Format.B10G11R11UfloatPack32 => true,
+        _ => false,
+    };
+
     // Builds the request for a bound color target. Null when the slot carries no target.
     public static ColorTargetResolution? ColorTarget(in ColorTargetWords words, uint targetMask, uint drawLayerOffset, bool ignoreTargetMask)
     {

src/SharpEmu.Libs/Gpu/Pipelines/ShaderPipelineCache.cs

@@ -455,7 +455,7 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider
             // A target the pixel program never exports keeps its contents, as on hardware; the
             // host output would otherwise write an undefined value (e.g. depth-only passes that
             // leave a color target bound and export only to the null target).
-            var exported = pixelStage is null || ((pixelStage.PixelColorExportMasks >> (int)(color.Slot * 4)) & 0xFu) != 0;
+            var exported = pixelStage is not null && ((pixelStage.PixelColorExportMasks >> (int)(color.Slot * 4)) & 0xFu) != 0;
             var colorMask = exported ? color.Resolution.ExportMapping.ApplyMask(context.RenderTargetMaskForSlot(color.Slot)) : 0;
             parameters.SetColorMask(index, colorMask);
             if (RenderTrace.Enabled && RenderTrace.Pipeline())

src/SharpEmu.Libs/Gpu/Pipelines/ShaderProgramCache.cs

@@ -601,6 +601,7 @@ internal sealed class ShaderProgramCache
                     LocalSizeX = Math.Max(info.ThreadsX, 1),
                     LocalSizeY = Math.Max(info.ThreadsY, 1),
                     LocalSizeZ = Math.Max(info.ThreadsZ, 1),
+                    LocalDataShareDwords = info.LocalDataShareDwords,
                 };
             }
         }

src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Targets.cs

@@ -184,7 +184,7 @@ public sealed partial class RenderExecutor
             return;
         }
 
-        var exportMasks = pixelProgram?.PixelColorExportMasks ?? uint.MaxValue;
+        var exportMasks = pixelProgram?.PixelColorExportMasks ?? 0u;
         var kept = 0u;
         for (var i = 0; i < state.ColorCount; i++)
         {

src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CachedImages.cs

@@ -7,6 +7,7 @@ using System.Collections.Concurrent;
 using SharpEmu.Libs.Gpu;
 using SharpEmu.Libs.Gpu.Buffers;
 using SharpEmu.Libs.Gpu.Images;
+using SharpEmu.Libs.Gpu.Rendering;
 using SharpEmu.Libs.Gpu.Scheduling;
 using SharpEmu.Libs.Gpu.Vulkan;
 using Silk.NET.Vulkan;
@@ -287,53 +288,14 @@ internal static unsafe partial class VulkanVideoPresenter
             var metadataAddress = target.Request.Description.Metadata.Range.Address;
             if (!_imageCache.IsMetadataCleared(metadataAddress, view.BaseLayer, out var metadataValue))
             {
-                return false;
+                return ResolveGuestDccAttachmentClear(target, out clearValue);
             }
 
             var resolution = target.Resolution;
-            switch ((byte)metadataValue)
+            if (!TryDecodeDccClear((byte)metadataValue, resolution.MetadataClearSupported, resolution.MetadataFixedClearSupported,
+                resolution.ColorClearValue, out clearValue))
             {
-                case 0x00:
-                    break;
-                case 0x20:
-                    if (!resolution.MetadataClearSupported)
-                    {
-                        return false;
-                    }
-
-                    clearValue = resolution.ColorClearValue;
-                    break;
-                case 0x40:
-                    if (!resolution.MetadataFixedClearSupported)
-                    {
-                        return false;
-                    }
-
-                    clearValue.Float32_3 = 1f;
-                    break;
-                case 0x80:
-                    if (!resolution.MetadataFixedClearSupported)
-                    {
-                        return false;
-                    }
-
-                    clearValue.Float32_0 = 1f;
-                    clearValue.Float32_1 = 1f;
-                    clearValue.Float32_2 = 1f;
-                    break;
-                case 0xc0:
-                    if (!resolution.MetadataFixedClearSupported)
-                    {
-                        return false;
-                    }
-
-                    clearValue.Float32_0 = 1f;
-                    clearValue.Float32_1 = 1f;
-                    clearValue.Float32_2 = 1f;
-                    clearValue.Float32_3 = 1f;
-                    break;
-                default:
-                    return false;
+                return false;
             }
 
             for (uint layer = 1; layer < view.LayerCount; layer++)
@@ -353,9 +315,104 @@ internal static unsafe partial class VulkanVideoPresenter
                 }
             }
 
+            ConsumeGuestDccClears(target.Request.Description, view.BaseLayer, view.LayerCount);
             return true;
         }
 
+        private bool ResolveGuestDccAttachmentClear(ColorAttachment target, out ClearColorValue clearValue)
+        {
+            clearValue = default;
+            var description = target.Request.Description;
+            var sliceSize = description.DccSliceSize;
+            var view = target.Request.View;
+            if (sliceSize == 0 || view.LayerCount == 0)
+            {
+                return false;
+            }
+
+            var metadataAddress = description.Metadata.Range.Address;
+            var slices = new ulong[view.LayerCount];
+            byte clearCode = 0;
+            for (uint layer = 0; layer < view.LayerCount; layer++)
+            {
+                if (!_imageCache.TryReadGuestDccClear(metadataAddress, sliceSize, view.BaseLayer + layer, out slices[layer], out var code) ||
+                    (layer != 0 && code != clearCode))
+                {
+                    return false;
+                }
+
+                clearCode = code;
+            }
+
+            var resolution = target.Resolution;
+            if (!TryDecodeDccClear(clearCode, resolution.MetadataClearSupported, resolution.MetadataFixedClearSupported,
+                resolution.ColorClearValue, out clearValue))
+            {
+                return false;
+            }
+
+            foreach (var slice in slices)
+            {
+                _bufferCache.FillBuffer(slice, sliceSize, uint.MaxValue, false);
+            }
+
+            if (RenderTrace.Enabled && RenderTrace.MetadataClear())
+            {
+                RenderTrace.Write(
+                    $"Materialized a guest DCC clear on a color target: metadata=0x{metadataAddress:X16} code=0x{clearCode:X2} " +
+                    $"layers={view.BaseLayer}+{view.LayerCount} slice=0x{sliceSize:X} format={description.PixelFormat}");
+            }
+
+            return true;
+        }
+
+        private void ConsumeGuestDccClears(in ImageDescription description, uint baseLayer, uint layerCount)
+        {
+            var sliceSize = description.DccSliceSize;
+            if (sliceSize == 0)
+            {
+                return;
+            }
+
+            for (uint layer = 0; layer < layerCount; layer++)
+            {
+                if (_imageCache.TryReadGuestDccClear(description.Metadata.Range.Address, sliceSize, baseLayer + layer, out var slice, out _))
+                {
+                    _bufferCache.FillBuffer(slice, sliceSize, uint.MaxValue, false);
+                }
+            }
+        }
+
+        private static bool TryDecodeDccClear(byte code, bool registerClearSupported, bool fixedClearSupported, ClearColorValue registerClear,
+            out ClearColorValue clearValue)
+        {
+            clearValue = default;
+            switch (code)
+            {
+                case 0x00:
+                    return true;
+                case 0x20:
+                    clearValue = registerClear;
+                    return registerClearSupported;
+                case 0x40:
+                    clearValue.Float32_3 = 1f;
+                    return fixedClearSupported;
+                case 0x80:
+                    clearValue.Float32_0 = 1f;
+                    clearValue.Float32_1 = 1f;
+                    clearValue.Float32_2 = 1f;
+                    return fixedClearSupported;
+                case 0xc0:
+                    clearValue.Float32_0 = 1f;
+                    clearValue.Float32_1 = 1f;
+                    clearValue.Float32_2 = 1f;
+                    clearValue.Float32_3 = 1f;
+                    return fixedClearSupported;
+                default:
+                    return false;
+            }
+        }
+
         private DepthAttachment? DiscoverDepthTarget(GuestDepthTarget target)
         {
             if (target.Registers is not { } words || ImageRequestBuilders.DepthTarget(words, _deviceInfo) is not { } resolution)

src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Descriptors.cs

@@ -285,6 +285,7 @@ internal static unsafe partial class VulkanVideoPresenter
                 descriptors.GlobalDataShare = new BufferView(_bufferCache.GdsBuffer.Handle, 0, Vk.WholeSize);
             }
 
+            FindDeviceAddressBuffers(prepared);
             FindBuffers(prepared);
             preparation.Stages.Add(prepared);
             ValidateDrawImageTypes(prepared);
@@ -322,6 +323,21 @@ internal static unsafe partial class VulkanVideoPresenter
             }
         }
 
+        private void FindDeviceAddressBuffers(PreparedStageBindings prepared)
+        {
+            foreach (var range in prepared.Stage.Resources.DeviceAddressRanges)
+            {
+                if (!range.Planned || range.Size == 0 ||
+                    range.Base >= PageOwnerTable.AddressSpaceSize || range.Size > PageOwnerTable.AddressSpaceSize - range.Base ||
+                    (!range.Written && !_guestMemory.CanRead(range.Base, 1)))
+                {
+                    continue;
+                }
+
+                _ = _bufferCache.FindBuffer(range.Base, ClampMappedSize(range.Base, range.Size));
+            }
+        }
+
         // The cache buffer each descriptor's range lives in; a null descriptor has none.
         private void FindBuffers(PreparedStageBindings prepared)
         {

src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Rendering.cs

@@ -236,6 +236,12 @@ internal static unsafe partial class VulkanVideoPresenter
         public ResourceSlotIdentifier FindImage(ref ImageRequest request, bool exactFormat)
         {
             _ = BeginBatchedGuestCommands();
+            if (request.Role == ImageRole.ColorTarget && request.Description.DccSliceSize is var sliceSize and not 0)
+            {
+                _imageCache.SynchronizeGuestDccMetadata(request.Description.Metadata.Range.Address, sliceSize,
+                    request.View.BaseLayer, request.View.LayerCount);
+            }
+
             return _imageCache.FindImage(ref request, exactFormat);
         }
 
@@ -534,8 +540,8 @@ internal static unsafe partial class VulkanVideoPresenter
         // A DCC fast clear stays pending until the surface binds as a color target. A shader that
         // samples or writes the surface first must see the cleared contents, and the later bind must
         // not clear over its writes: Astro Bot's save-slot cards were drawn by compute into a
-        // fast-cleared target, then wiped black by the deferred clear. Only the zero clear code is
-        // materialized here; the register clear color is known only when the surface is a target.
+        // fast-cleared target, then wiped black by the deferred clear. The register clear color is
+        // known only when the surface is a target.
         private void RecordSampledColorMetadataClears(TextureResource[] bindings)
         {
             const byte DccClearToZero = 0x00;
@@ -547,10 +553,17 @@ internal static unsafe partial class VulkanVideoPresenter
                 }
 
                 var metadataAddress = image.Description.Metadata.Range.Address;
+                var sliceSize = image.Description.DccSliceSize;
+                var fixedClearSupported = ImageRequestBuilders.SupportsDccFixedClear(image.Description.PixelFormat);
                 var view = binding.Request.View;
                 for (var layer = view.BaseLayer; layer < view.BaseLayer + view.LayerCount; layer++)
                 {
-                    if (!_imageCache.IsMetadataCleared(metadataAddress, layer, out var metadataValue) || (byte)metadataValue != DccClearToZero)
+                    var tracked = _imageCache.IsMetadataCleared(metadataAddress, layer, out var metadataValue) && (byte)metadataValue == DccClearToZero;
+                    var clearValue = default(ClearColorValue);
+                    ulong guestSlice = 0;
+                    if (!tracked &&
+                        (sliceSize == 0 || !_imageCache.TryReadGuestDccClear(metadataAddress, sliceSize, layer, out guestSlice, out var code) ||
+                         !TryDecodeDccClear(code, false, fixedClearSupported, default, out clearValue)))
                     {
                         continue;
                     }
@@ -560,12 +573,26 @@ internal static unsafe partial class VulkanVideoPresenter
                     var range = new SubresourceRange(0, 1, layer, 1);
                     image.Transition(ImageLayout.TransferDstOptimal, AccessFlags.TransferWriteBit, range, command);
                     var vkRange = new ImageSubresourceRange(ImageAspectFlags.ColorBit, 0, 1, layer, 1);
-                    var clearValue = default(ClearColorValue);
                     _vk.CmdClearColorImage(command, image.Backing.Handle, ImageLayout.TransferDstOptimal, &clearValue, 1, &vkRange);
+                    if (!tracked)
+                    {
+                        _bufferCache.FillBuffer(guestSlice, sliceSize, uint.MaxValue, false);
+                        if (RenderTrace.Enabled && RenderTrace.MetadataClear())
+                        {
+                            RenderTrace.Write(
+                                $"Materialized a guest DCC clear on a sampled image: metadata=0x{metadataAddress:X16} layer={layer} " +
+                                $"slice=0x{sliceSize:X} format={image.Description.PixelFormat}");
+                        }
+
+                        continue;
+                    }
+
                     if (!_imageCache.SetMetadataSlice(metadataAddress, layer, false))
                     {
                         throw SubmissionScheduler.Fatal($"The DCC clear state could not be consumed: metadata=0x{metadataAddress:X16} layer={layer}.");
                     }
+
+                    ConsumeGuestDccClears(image.Description, layer, 1);
                 }
             }
         }

src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Alu.cs

@@ -840,52 +840,17 @@ public static partial class Gen5SpirvTranslator
                         right = BitwiseAnd(right, UInt(0x00FF_FFFF));
                     }
 
-                    var wideLeft = _module.AddInstruction(
-                        SpirvOp.UConvert,
-                        _ulongType,
-                        left);
-                    var wideRight = _module.AddInstruction(
-                        SpirvOp.UConvert,
-                        _ulongType,
-                        right);
-                    var product = _module.AddInstruction(
-                        SpirvOp.IMul,
-                        _ulongType,
-                        wideLeft,
-                        wideRight);
-                    result = _module.AddInstruction(
-                        SpirvOp.UConvert,
-                        _uintType,
-                        ShiftRightLogical64(
-                            product,
-                            _module.Constant64(_ulongType, 32)));
+                    result = MultiplyExtended(left, right, signed: false).High;
                     break;
                 }
                 case "VMulHiI32":
                 {
-                    var wideLeft = _module.AddInstruction(
-                        SpirvOp.SConvert,
-                        _longType,
-                        Bitcast(_intType, GetRawSource(instruction, 0)));
-                    var wideRight = _module.AddInstruction(
-                        SpirvOp.SConvert,
-                        _longType,
-                        Bitcast(_intType, GetRawSource(instruction, 1)));
-                    var product = _module.AddInstruction(
-                        SpirvOp.IMul,
-                        _longType,
-                        wideLeft,
-                        wideRight);
                     result = Bitcast(
                         _uintType,
-                        _module.AddInstruction(
-                            SpirvOp.SConvert,
-                            _intType,
-                            _module.AddInstruction(
-                                SpirvOp.ShiftRightArithmetic,
-                                _longType,
-                                product,
-                                _module.Constant64(_longType, 32))));
+                        MultiplyExtended(
+                            Bitcast(_intType, GetRawSource(instruction, 0)),
+                            Bitcast(_intType, GetRawSource(instruction, 1)),
+                            signed: true).High);
                     break;
                 }
                 case "VBcntU32B32":
@@ -1319,19 +1284,8 @@ public static partial class Gen5SpirvTranslator
                     // sources are 32-bit factors; the third is a 64-bit addend
                     // held in a VGPR or SGPR pair. Its SDST receives the carry
                     // mask for the unsigned 64-bit addition.
-                    var wideLeft = _module.AddInstruction(
-                        SpirvOp.UConvert,
-                        _ulongType,
-                        GetRawSource(instruction, 0));
-                    var wideRight = _module.AddInstruction(
-                        SpirvOp.UConvert,
-                        _ulongType,
-                        GetRawSource(instruction, 1));
-                    var product = _module.AddInstruction(
-                        SpirvOp.IMul,
-                        _ulongType,
-                        wideLeft,
-                        wideRight);
+                    var (productLow, productHigh) = MultiplyExtended(GetRawSource(instruction, 0), GetRawSource(instruction, 1), signed: false);
+                    var product = Pair64(productLow, productHigh);
                     var addend = GetRawSource64(instruction, 2);
                     var wideResult = _module.AddInstruction(
                         SpirvOp.IAdd,
@@ -3034,47 +2988,13 @@ public static partial class Gen5SpirvTranslator
                                 right);
                             break;
                         case "SMulHiI32":
-                        {
-                            var wideLeft = _module.AddInstruction(
-                                SpirvOp.SConvert,
-                                _longType,
-                                Bitcast(_intType, left));
-                            var wideRight = _module.AddInstruction(
-                                SpirvOp.SConvert,
-                                _longType,
-                                Bitcast(_intType, right));
-                            var product = _module.AddInstruction(
-                                SpirvOp.IMul,
-                                _longType,
-                                wideLeft,
-                                wideRight);
                             result = Bitcast(
                                 _uintType,
-                                _module.AddInstruction(
-                                    SpirvOp.SConvert,
-                                    _intType,
-                                    _module.AddInstruction(
-                                        SpirvOp.ShiftRightArithmetic,
-                                        _longType,
-                                        product,
-                                        _module.Constant64(_longType, 32))));
+                                MultiplyExtended(Bitcast(_intType, left), Bitcast(_intType, right), signed: true).High);
                             break;
-                        }
                         case "SMulHiU32":
-                        {
-                            var product = _module.AddInstruction(
-                                SpirvOp.IMul,
-                                _ulongType,
-                                _module.AddInstruction(SpirvOp.UConvert, _ulongType, left),
-                                _module.AddInstruction(SpirvOp.UConvert, _ulongType, right));
-                            result = _module.AddInstruction(
-                                SpirvOp.UConvert,
-                                _uintType,
-                                ShiftRightLogical64(
-                                    product,
-                                    _module.Constant64(_ulongType, 32)));
+                            result = MultiplyExtended(left, right, signed: false).High;
                             break;
-                        }
                         case "SAndB32":
                             result = BitwiseAnd(left, right);
                             Store(_scc, IsNotZero(result));
@@ -3448,7 +3368,9 @@ public static partial class Gen5SpirvTranslator
             var left = GetRawSource64(instruction, 0);
             if (instruction.Opcode.EndsWith("SaveexecB64", StringComparison.Ordinal))
             {
-                var oldExec = BooleanToWaveMask(Load(_boolType, _exec));
+                var oldExec = _emulateWave64 && _subgroupInvocationIdInput != 0
+                    ? LoadS64(126)
+                    : BooleanToWaveMask(Load(_boolType, _exec));
                 var notLeft = _module.AddInstruction(SpirvOp.Not, _ulongType, left);
                 var newExec = instruction.Opcode switch
                 {
@@ -3821,14 +3743,14 @@ public static partial class Gen5SpirvTranslator
                     _module.AddInstruction(
                         SpirvOp.Select,
                         _uintType,
-                        LogicalNot(SubgroupAny(Load(_boolType, _vcc))),
+                        LogicalNot(WaveMaskAny(106, _vcc)),
                         UInt(1),
                         UInt(0)),
                 Gen5OperandKind.EncodedConstant when operand.Value == 252 =>
                     _module.AddInstruction(
                         SpirvOp.Select,
                         _uintType,
-                        LogicalNot(SubgroupAny(Load(_boolType, _exec))),
+                        LogicalNot(WaveMaskAny(126, _exec)),
                         UInt(1),
                         UInt(0)),
                 Gen5OperandKind.EncodedConstant when operand.Value == 253 =>
@@ -4905,76 +4827,48 @@ public static partial class Gen5SpirvTranslator
 
         // An emulated wave64 spans two host subgroups, so a subgroup broadcast cannot reach
         // the other half. The selected guest lane publishes through workgroup scratch, like
-        // BroadcastFirstWave64Active; the trailing barrier keeps the next use from racing.
-        private uint BroadcastWave64Lane(uint value, uint lane)
-        {
-            EmitConditional(
-                _module.AddInstruction(SpirvOp.IEqual, _boolType, GuestWaveLane(), lane),
-                () => Store(WaveBroadcastScratchPointer(), value));
-            EmitWave64Barrier();
-            var result = Load(_uintType, WaveBroadcastScratchPointer());
-            EmitWave64Barrier();
-            return result;
-        }
+        // BroadcastFirstWave64Active.
+        private uint BroadcastWave64Lane(uint value, uint lane) =>
+            ExchangeWave64Value(_module.AddInstruction(SpirvOp.IEqual, _boolType, GuestWaveLane(), lane), value);
 
         private uint BroadcastFirstWave64Active(uint value)
         {
             var lane = GuestWaveLane();
-            EmitConditional(
-                _module.AddInstruction(
-                    SpirvOp.IEqual,
-                    _boolType,
-                    lane,
-                    UInt(0)),
-                () => Store(WaveBroadcastScratchPointer(), UInt(0)));
-            EmitWave64Barrier();
-
-            var activeMask = BooleanToWaveMask(Load(_boolType, _exec));
-            var lowMask = _module.AddInstruction(
-                SpirvOp.UConvert,
-                _uintType,
-                activeMask);
-            var highMask = _module.AddInstruction(
-                SpirvOp.UConvert,
-                _uintType,
-                ShiftRightLogical64(
-                    activeMask,
-                    _module.Constant64(_ulongType, 32)));
-            var hasLow = IsNotZero(lowMask);
-            var hasHigh = IsNotZero(highMask);
-            var firstLow = Ext(73, _uintType, lowMask);
-            var firstHigh = IAdd(UInt(32), Ext(73, _uintType, highMask));
-            var firstLane = _module.AddInstruction(
+            var upperHalf = ShiftRightLogical(lane, UInt(5));
+            var activeInHalf = OwnHalfBallot(Load(_boolType, _exec));
+            var halfHasActive = IsNotZero(activeInHalf);
+            var firstInHalf = _module.AddInstruction(
                 SpirvOp.Select,
                 _uintType,
-                hasLow,
-                firstLow,
-                _module.AddInstruction(
-                    SpirvOp.Select,
-                    _uintType,
-                    hasHigh,
-                    firstHigh,
-                    UInt(0)));
-            var isFirst = _module.AddInstruction(
-                SpirvOp.IEqual,
-                _boolType,
-                lane,
-                firstLane);
-            EmitConditional(
-                _module.AddInstruction(
-                    SpirvOp.LogicalAnd,
-                    _boolType,
-                    isFirst,
-                    _module.AddInstruction(
-                        SpirvOp.LogicalOr,
-                        _boolType,
-                        hasLow,
-                        hasHigh)),
-                () => Store(WaveBroadcastScratchPointer(), value));
+                halfHasActive,
+                Ext(73, _uintType, activeInHalf),
+                UInt(0));
+            var halfBase = BitwiseAnd(Load(_uintType, _subgroupInvocationIdInput), UInt(~31u));
+            var halfValue = _module.AddInstruction(
+                SpirvOp.GroupNonUniformBroadcast,
+                _uintType,
+                UInt(3),
+                value,
+                IAdd(halfBase, firstInHalf));
+            var exchange = BeginWave64Exchange();
+            EmitConditional(IsHalfWaveLeader(lane), () =>
+            {
+                Store(Wave64ExchangePointer(exchange, upperHalf), halfValue);
+                Store(
+                    Wave64ExchangePointer(exchange, IAdd(UInt(2), upperHalf)),
+                    _module.AddInstruction(SpirvOp.Select, _uintType, halfHasActive, UInt(1), UInt(0)));
+            });
             EmitWave64Barrier();
-            var result = Load(_uintType, WaveBroadcastScratchPointer());
-            EmitWave64Barrier();
-            return result;
+            var lowerValue = Load(_uintType, Wave64ExchangePointer(exchange, UInt(0)));
+            var upperValue = Load(_uintType, Wave64ExchangePointer(exchange, UInt(1)));
+            var lowerActive = IsNotZero(Load(_uintType, Wave64ExchangePointer(exchange, UInt(2))));
+            var upperActive = IsNotZero(Load(_uintType, Wave64ExchangePointer(exchange, UInt(3))));
+            return _module.AddInstruction(
+                SpirvOp.Select,
+                _uintType,
+                _module.AddInstruction(SpirvOp.LogicalAnd, _boolType, LogicalNot(lowerActive), upperActive),
+                upperValue,
+                lowerValue);
         }
 
         private void StoreCarryOut(
@@ -5015,13 +4909,7 @@ public static partial class Gen5SpirvTranslator
             {
                 // The selected guest lane can belong to another host subgroup.
                 // Read it even when the guest execution mask disables that lane.
-                var isSelectedLane = _module.AddInstruction(
-                    SpirvOp.IEqual, _boolType, GuestWaveLane(), selectedLane);
-                EmitConditional(isSelectedLane, () => Store(WaveBroadcastScratchPointer(), sourceValue));
-                EmitWave64Barrier();
-                var broadcast = Load(_uintType, WaveBroadcastScratchPointer());
-                EmitWave64Barrier();
-                StoreS(destination, broadcast);
+                StoreS(destination, BroadcastWave64Lane(sourceValue, selectedLane));
             }
             else if (_subgroupInvocationIdInput != 0)
             {

src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Resources.cs

@@ -1904,10 +1904,7 @@ public static partial class Gen5SpirvTranslator
             uint broadcast;
             if (_emulateWave64)
             {
-                EmitConditional(isFirstActive, () => Store(WaveBroadcastScratchPointer(), firstValue));
-                EmitWave64Barrier();
-                broadcast = Load(_uintType, WaveBroadcastScratchPointer());
-                EmitWave64Barrier();
+                broadcast = ExchangeWave64Value(isFirstActive, firstValue);
             }
             else if (_stage == Gen5SpirvStage.Compute || _enableGraphicsSubgroupOperations)
             {

src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.cs

@@ -17,6 +17,8 @@ public static partial class Gen5SpirvTranslator
     // so a smaller array is safe.
     private const uint PrivateLdsDwordCount = 2048;
     private const uint RdnaWaveLaneCount = 32;
+    private const uint Wave64ExchangeSlotCount = 4;
+    private const uint Wave64ExchangeDwordCount = Wave64ExchangeSlotCount * 2;
 
     internal static SpirvImageFormat DecodeStorageImageFormat(
         uint dataFormat,
@@ -142,7 +144,7 @@ public static partial class Gen5SpirvTranslator
         private uint _uvec4Type;
         private uint _privateUintPointer;
         private uint _privateVec2Pointer;
-        private uint _privateBoolPointer;
+        private readonly HashSet<uint> _flagVariables = [];
         private uint _runtimeBufferBiases;
         private uint _scalarRegisters;
         private uint _vectorRegisters;
@@ -185,12 +187,13 @@ public static partial class Gen5SpirvTranslator
         private uint _localInvocationIndexInput;
         private uint _workGroupIdInput;
         private uint _pushConstantUintPointer;
-        private uint _subgroupSizeInput;
         private uint _subgroupInvocationIdInput;
-        private uint _waveMaskScratch;
-        private uint _waveMaskScratchElementPointer;
-        private uint _waveBroadcastScratch;
-        private bool _waveScratchInLds;
+        private uint _wave64Exchange;
+        private uint _wave64ExchangeElementPointer;
+        private uint _wave64ExchangeOffset;
+        private uint _wave64ExchangeParity;
+        private Ir.Gen5Wave64HalfMaskPlan? _halfMaskPlan;
+        private uint? _emittingPc;
         private uint _glsl;
 
         private enum ImageComponentKind
@@ -564,8 +567,6 @@ public static partial class Gen5SpirvTranslator
                 _module.TypePointer(SpirvStorageClass.Private, _uintType);
             _privateVec2Pointer =
                 _module.TypePointer(SpirvStorageClass.Private, _vec2Type);
-            _privateBoolPointer =
-                _module.TypePointer(SpirvStorageClass.Private, _boolType);
 
             var scalarArrayType = _module.TypeArray(_uintType, ScalarRegisterCount);
             var vectorArrayType = _module.TypeArray(_uintType, VectorRegisterCount);
@@ -622,7 +623,7 @@ public static partial class Gen5SpirvTranslator
 
         private void DeclareWave64Scratch()
         {
-            if (!_emulateWave64 || !UsesSubgroupOperations())
+            if (!UsesWave64Exchange())
             {
                 return;
             }
@@ -630,36 +631,39 @@ public static partial class Gen5SpirvTranslator
             // Metal exposes 32 KiB of threadgroup memory on the Apple GPUs we
             // target. Some PS5 compute shaders legitimately request all of it,
             // so allocating another workgroup variable for the wave64 bridge
-            // makes pipeline creation fail. Reuse the final three dwords of the
+            // makes pipeline creation fail. Reuse the final dwords of the
             // existing LDS allocation in that case. The translator already
             // bounds guest LDS accesses to this fixed allocation; keeping the
             // bridge inside it preserves the host limit and still provides the
             // cross-subgroup rendezvous needed to model one 64-lane guest wave.
             if (_lds != 0)
             {
-                _waveScratchInLds = true;
-                _waveMaskScratchElementPointer = _ldsElementPointer;
+                _wave64Exchange = _lds;
+                _wave64ExchangeElementPointer = _ldsElementPointer;
+                _wave64ExchangeOffset = _ldsDwordMask + 1 < LdsDwordCount
+                    ? _ldsDwordMask + 1
+                    : LdsDwordCount - Wave64ExchangeDwordCount;
                 return;
             }
 
-            var maskArrayType = _module.TypeArray(_uintType, 2);
-            var maskArrayPointer =
-                _module.TypePointer(SpirvStorageClass.Workgroup, maskArrayType);
-            _waveMaskScratchElementPointer =
-                _module.TypePointer(SpirvStorageClass.Workgroup, _uintType);
-            _waveMaskScratch = _module.AddGlobalVariable(
-                maskArrayPointer,
+            var exchangeArrayType = _module.TypeArray(_uintType, Wave64ExchangeDwordCount);
+            _wave64ExchangeElementPointer = _module.TypePointer(SpirvStorageClass.Workgroup, _uintType);
+            _wave64Exchange = _module.AddGlobalVariable(
+                _module.TypePointer(SpirvStorageClass.Workgroup, exchangeArrayType),
                 SpirvStorageClass.Workgroup);
-            _module.AddName(_waveMaskScratch, "wave64MaskScratch");
-            _interfaces.Add(_waveMaskScratch);
+            _wave64ExchangeOffset = 0;
+            _module.AddName(_wave64Exchange, "wave64Exchange");
+            _interfaces.Add(_wave64Exchange);
+        }
 
-            var uintPointer =
-                _module.TypePointer(SpirvStorageClass.Workgroup, _uintType);
-            _waveBroadcastScratch = _module.AddGlobalVariable(
-                uintPointer,
-                SpirvStorageClass.Workgroup);
-            _module.AddName(_waveBroadcastScratch, "wave64BroadcastScratch");
-            _interfaces.Add(_waveBroadcastScratch);
+        private bool UsesWave64Exchange() => _emulateWave64 && UsesSubgroupOperations();
+
+        private uint ComputeLdsGuestDwordCount()
+        {
+            var declared = _request.LocalDataShareDwords;
+            return declared == 0 || declared > LdsDwordCount
+                ? LdsDwordCount
+                : System.Numerics.BitOperations.RoundUpToPowerOf2(declared);
         }
 
         private void DeclareLds()
@@ -681,11 +685,14 @@ public static partial class Gen5SpirvTranslator
                 ? SpirvStorageClass.Workgroup
                 : SpirvStorageClass.Private;
             var dwordCount = _stage == Gen5SpirvStage.Compute
-                ? LdsDwordCount
+                ? ComputeLdsGuestDwordCount()
                 : PrivateLdsDwordCount;
             _ldsDwordMask = dwordCount - 1;
+            var arrayDwordCount = UsesWave64Exchange() && dwordCount < LdsDwordCount
+                ? dwordCount + Wave64ExchangeDwordCount
+                : dwordCount;
 
-            var ldsArrayType = _module.TypeArray(_uintType, dwordCount);
+            var ldsArrayType = _module.TypeArray(_uintType, arrayDwordCount);
             var ldsPointer = _module.TypePointer(storageClass, ldsArrayType);
             _ldsElementPointer = _module.TypePointer(storageClass, _uintType);
             _lds64ElementPointer = _module.TypePointer(storageClass, _ulongType);
@@ -769,24 +776,6 @@ public static partial class Gen5SpirvTranslator
                 }
                 _interfaces.Add(_subgroupInvocationIdInput);
 
-                if (_emulateWave64)
-                {
-                    _subgroupSizeInput = _module.AddGlobalVariable(
-                        subgroupPointer,
-                        SpirvStorageClass.Input);
-                    _module.AddDecoration(
-                        _subgroupSizeInput,
-                        SpirvDecoration.BuiltIn,
-                        (uint)SpirvBuiltIn.SubgroupSize);
-                    if (_stage == Gen5SpirvStage.Pixel)
-                    {
-                        _module.AddDecoration(
-                            _subgroupSizeInput,
-                            SpirvDecoration.Flat);
-                    }
-                    _interfaces.Add(_subgroupSizeInput);
-                }
-
                 // LocalInvocationIndex is a compute-stage built-in. Graphics
                 // stages can still use native subgroup operations, but they
                 // cannot combine two subgroup32 halves through a workgroup.
@@ -1103,7 +1092,12 @@ public static partial class Gen5SpirvTranslator
             {
                 Store(_pixelValidMaskActive, _module.ConstantBool(true));
             }
-            if (_subgroupInvocationIdInput != 0)
+            if (_subgroupInvocationIdInput != 0 && _emulateWave64)
+            {
+                StoreS64(106, BothHalvesOfOwnBallot(_module.ConstantBool(false)));
+                StoreS64(126, BothHalvesOfOwnBallot(_module.ConstantBool(true)));
+            }
+            else if (_subgroupInvocationIdInput != 0)
             {
                 StoreWaveMask(106, _module.ConstantBool(false));
                 StoreWaveMask(126, _module.ConstantBool(true));
@@ -1362,13 +1356,28 @@ public static partial class Gen5SpirvTranslator
         {
             error = string.Empty;
             var block = blocks[blockIndex];
+            var halfMaskPlan = HalfMaskPlan();
             // One guest wave can span two host subgroups. Keep its shared-memory phases ordered.
-            // Restrict added barriers to a single block, where all invocations follow the same path.
-            var synchronizeSharedMemory = _emulateWave64 && blocks.Count == 1;
+            // The half-mask plan inserts its own barriers; other wave64 programs also
+            // need LDS phase ordering when reads and writes span basic blocks.
+            var synchronizeSharedMemory = _emulateWave64 && halfMaskPlan is null;
             var sharedMemoryPhase = SharedMemoryPhase.None;
             for (var index = block.StartIndex; index < block.EndIndex; index++)
             {
                 var instruction = _request.Program.Instructions[index];
+                if (halfMaskPlan is not null)
+                {
+                    if (halfMaskPlan.ExactPairsBefore.TryGetValue(instruction.Pc, out var exactPairs))
+                    {
+                        EmitExactWaveMasks(exactPairs);
+                    }
+
+                    if (halfMaskPlan.SharedMemoryBarriersBefore.Contains(instruction.Pc))
+                    {
+                        EmitWave64Barrier();
+                    }
+                }
+
                 if (IsBranch(instruction.Opcode) || instruction.Opcode == "SEndpgm")
                 {
                     continue;
@@ -1389,7 +1398,9 @@ public static partial class Gen5SpirvTranslator
                 }
 
                 _execKnownFull = IsExecKnownFull(instruction.Pc);
+                _emittingPc = instruction.Pc;
                 var emitted = TryEmitInstruction(instruction, out error);
+                _emittingPc = null;
                 _execKnownFull = false;
                 if (!emitted)
                 {
@@ -1653,10 +1664,10 @@ public static partial class Gen5SpirvTranslator
             {
                 "SCbranchScc0" => LogicalNot(Load(_boolType, _scc)),
                 "SCbranchScc1" => Load(_boolType, _scc),
-                "SCbranchVccz" => LogicalNot(SubgroupAny(Load(_boolType, _vcc))),
-                "SCbranchVccnz" => SubgroupAny(Load(_boolType, _vcc)),
-                "SCbranchExecz" => LogicalNot(SubgroupAny(Load(_boolType, _exec))),
-                "SCbranchExecnz" => SubgroupAny(Load(_boolType, _exec)),
+                "SCbranchVccz" => LogicalNot(WaveMaskAny(106, _vcc)),
+                "SCbranchVccnz" => WaveMaskAny(106, _vcc),
+                "SCbranchExecz" => LogicalNot(WaveMaskAny(126, _exec)),
+                "SCbranchExecnz" => WaveMaskAny(126, _exec),
                 // The emulator does not expose a shader debug session.
                 "SCbranchCdbgsys" or
                 "SCbranchCdbguser" or
@@ -2191,11 +2202,7 @@ public static partial class Gen5SpirvTranslator
             uint broadcast;
             if (_emulateWave64)
             {
-                EmitConditional(isFirstActive, () =>
-                    Store(WaveBroadcastScratchPointer(), firstValue));
-                EmitWave64Barrier();
-                broadcast = Load(_uintType, WaveBroadcastScratchPointer());
-                EmitWave64Barrier();
+                broadcast = ExchangeWave64Value(isFirstActive, firstValue);
             }
             else
             {
@@ -7006,6 +7013,13 @@ public static partial class Gen5SpirvTranslator
                 return variable;
             }
 
+            uint Flag(bool initialValue)
+            {
+                var variable = Variable(_uintType, UInt(initialValue ? 1u : 0u));
+                _flagVariables.Add(variable);
+                return variable;
+            }
+
             _scalarRegisters = Variable(_scalarArrayType, _module.ConstantNull(_scalarArrayType));
             _vectorRegisters = Variable(_vectorArrayType, _module.ConstantNull(_vectorArrayType));
             if (_functionScopeState)
@@ -7016,18 +7030,23 @@ public static partial class Gen5SpirvTranslator
                 _module.AddName(_packedHalfRegisters, "vgprPackedHalf");
             }
 
-            _scc = Variable(_boolType, _module.ConstantBool(false));
-            _vcc = Variable(_boolType, _module.ConstantBool(false));
-            _exec = Variable(_boolType, _module.ConstantBool(true));
-            _reachedPixelExport = Variable(_boolType, _module.ConstantBool(false));
+            _scc = Flag(false);
+            _vcc = Flag(false);
+            _exec = Flag(true);
+            _reachedPixelExport = Flag(false);
             if (_usesPixelValidMask)
             {
-                _pixelValidMaskActive = Variable(_boolType, _module.ConstantBool(true));
+                _pixelValidMaskActive = Flag(true);
                 _module.AddName(_pixelValidMaskActive, "pixelValidMaskActive");
             }
 
             _programCounter = Variable(_uintType, _module.Constant(_uintType, 0));
-            _programActive = Variable(_boolType, _module.ConstantBool(true));
+            _programActive = Flag(true);
+            if (UsesWave64Exchange())
+            {
+                _wave64ExchangeParity = Variable(_uintType, _module.Constant(_uintType, 0));
+                _module.AddName(_wave64ExchangeParity, "wave64ExchangeParity");
+            }
             if (_maxDispatcherSteps > 0)
             {
                 _iterationGuard = Variable(_uintType, _module.Constant(_uintType, 0));
@@ -7307,11 +7326,32 @@ public static partial class Gen5SpirvTranslator
                     "SPIR-V generator attempted OpLoad from id 0.");
             }
 
+            if (_flagVariables.Contains(pointer))
+            {
+                return _module.AddInstruction(
+                    SpirvOp.INotEqual,
+                    _boolType,
+                    _module.AddInstruction(SpirvOp.Load, _uintType, pointer),
+                    UInt(0));
+            }
+
             return _module.AddInstruction(SpirvOp.Load, type, pointer);
         }
 
-        private void Store(uint pointer, uint value) =>
+        private void Store(uint pointer, uint value)
+        {
+            if (_flagVariables.Contains(pointer))
+            {
+                value = _module.AddInstruction(
+                    SpirvOp.Select,
+                    _uintType,
+                    value,
+                    UInt(1),
+                    UInt(0));
+            }
+
             _module.AddStatement(SpirvOp.Store, pointer, value);
+        }
 
         private uint UInt(uint value) => _module.Constant(_uintType, value);
 
@@ -7346,6 +7386,31 @@ public static partial class Gen5SpirvTranslator
                     Bitcast(_intType, left),
                     BitwiseAnd(right, UInt(31))));
 
+        private uint _uintPairType;
+        private uint _intPairType;
+
+        private (uint Low, uint High) MultiplyExtended(uint left, uint right, bool signed)
+        {
+            var elementType = signed ? _intType : _uintType;
+            if (signed && _intPairType == 0)
+            {
+                _intPairType = _module.TypeStruct(_intType, _intType);
+            }
+            else if (!signed && _uintPairType == 0)
+            {
+                _uintPairType = _module.TypeStruct(_uintType, _uintType);
+            }
+
+            var product = _module.AddInstruction(
+                signed ? SpirvOp.SMulExtended : SpirvOp.UMulExtended,
+                signed ? _intPairType : _uintPairType,
+                left,
+                right);
+            return (
+                _module.AddInstruction(SpirvOp.CompositeExtract, elementType, product, 0),
+                _module.AddInstruction(SpirvOp.CompositeExtract, elementType, product, 1));
+        }
+
         private uint ShiftLeftLogical64(uint left, uint right) =>
             _module.AddInstruction(
                 SpirvOp.ShiftLeftLogical,
@@ -7495,6 +7560,19 @@ public static partial class Gen5SpirvTranslator
                 return BooleanToLaneMask(condition);
             }
 
+            if (_emulateWave64)
+            {
+                var lane = GuestWaveLane();
+                var owned = OwnHalfBallot(condition);
+                var exchange = BeginWave64Exchange();
+                EmitConditional(IsHalfWaveLeader(lane), () =>
+                    Store(Wave64ExchangePointer(exchange, ShiftRightLogical(lane, UInt(5))), owned));
+                EmitWave64Barrier();
+                return Pair64(
+                    Load(_uintType, Wave64ExchangePointer(exchange, UInt(0))),
+                    Load(_uintType, Wave64ExchangePointer(exchange, UInt(1))));
+            }
+
             var ballot = _module.AddInstruction(
                 SpirvOp.GroupNonUniformBallot,
                 _uvec4Type,
@@ -7505,57 +7583,6 @@ public static partial class Gen5SpirvTranslator
                 _uintType,
                 ballot,
                 0);
-            if (_emulateWave64)
-            {
-                var high = _module.AddInstruction(
-                    SpirvOp.CompositeExtract,
-                    _uintType,
-                    ballot,
-                    1);
-                var subgroupLane =
-                    Load(_uintType, _subgroupInvocationIdInput);
-                var firstLane = _module.AddInstruction(
-                    SpirvOp.IEqual,
-                    _boolType,
-                    subgroupLane,
-                    UInt(0));
-                var half = ShiftRightLogical(GuestWaveLane(), UInt(5));
-                EmitConditional(firstLane, () =>
-                {
-                    Store(WaveMaskScratchPointer(half), low);
-
-                    var nativeWave64 = _module.AddInstruction(
-                        SpirvOp.UGreaterThanEqual,
-                        _boolType,
-                        Load(_uintType, _subgroupSizeInput),
-                        UInt(64));
-                    EmitConditional(nativeWave64, () =>
-                    {
-                        Store(WaveMaskScratchPointer(UInt(1)), high);
-                    });
-                });
-                EmitWave64Barrier();
-                var lowMask = Load(
-                    _uintType,
-                    WaveMaskScratchPointer(UInt(0)));
-                var highMask = Load(
-                    _uintType,
-                    WaveMaskScratchPointer(UInt(1)));
-                var combined = BitwiseOr64(
-                    _module.AddInstruction(
-                        SpirvOp.UConvert,
-                        _ulongType,
-                        lowMask),
-                    ShiftLeftLogical64(
-                        _module.AddInstruction(
-                            SpirvOp.UConvert,
-                            _ulongType,
-                            highMask),
-                        _module.Constant64(_ulongType, 32)));
-                EmitWave64Barrier();
-                return combined;
-            }
-
             var widened = _module.AddInstruction(SpirvOp.UConvert, _ulongType, low);
             if (_waveLaneCount != 64)
             {
@@ -7576,21 +7603,127 @@ public static partial class Gen5SpirvTranslator
                 widened);
         }
 
-        private uint WaveMaskScratchPointer(uint index) =>
+        private uint OwnHalfBallot(uint condition)
+        {
+            var ballot = _module.AddInstruction(
+                SpirvOp.GroupNonUniformBallot,
+                _uvec4Type,
+                UInt(3),
+                condition);
+            return _module.AddInstruction(
+                SpirvOp.VectorExtractDynamic,
+                _uintType,
+                ballot,
+                ShiftRightLogical(Load(_uintType, _subgroupInvocationIdInput), UInt(5)));
+        }
+
+        private uint BooleanToHalfWaveMask(uint condition)
+        {
+            var owned = _module.AddInstruction(SpirvOp.UConvert, _ulongType, OwnHalfBallot(condition));
+            return _module.AddInstruction(
+                SpirvOp.Select,
+                _ulongType,
+                _module.AddInstruction(SpirvOp.UGreaterThanEqual, _boolType, GuestWaveLane(), UInt(32)),
+                ShiftLeftLogical64(owned, _module.Constant64(_ulongType, 32)),
+                owned);
+        }
+
+        private uint BothHalvesOfOwnBallot(uint condition)
+        {
+            var owned = _module.AddInstruction(SpirvOp.UConvert, _ulongType, OwnHalfBallot(condition));
+            return BitwiseOr64(owned, ShiftLeftLogical64(owned, _module.Constant64(_ulongType, 32)));
+        }
+
+        private uint IsHalfWaveLeader(uint lane) =>
+            _module.AddInstruction(SpirvOp.IEqual, _boolType, BitwiseAnd(lane, UInt(31)), UInt(0));
+
+        private uint BeginWave64Exchange()
+        {
+            var parity = Load(_uintType, _wave64ExchangeParity);
+            Store(_wave64ExchangeParity, BitwiseXor(parity, UInt(1)));
+            return IAdd(
+                UInt(_wave64ExchangeOffset),
+                _module.AddInstruction(SpirvOp.IMul, _uintType, parity, UInt(Wave64ExchangeSlotCount)));
+        }
+
+        private uint Wave64ExchangePointer(uint exchange, uint slot) =>
             _module.AddInstruction(
                 SpirvOp.AccessChain,
-                _waveMaskScratchElementPointer,
-                _waveScratchInLds ? _lds : _waveMaskScratch,
-                _waveScratchInLds ? IAdd(UInt(LdsDwordCount - 3), index) : index);
+                _wave64ExchangeElementPointer,
+                _wave64Exchange,
+                IAdd(exchange, slot));
 
-        private uint WaveBroadcastScratchPointer() =>
-            _waveScratchInLds
-                ? _module.AddInstruction(
-                    SpirvOp.AccessChain,
-                    _ldsElementPointer,
-                    _lds,
-                    UInt(LdsDwordCount - 1))
-                : _waveBroadcastScratch;
+        private uint ExchangeWave64Value(uint writer, uint value)
+        {
+            var exchange = BeginWave64Exchange();
+            EmitConditional(writer, () => Store(Wave64ExchangePointer(exchange, UInt(0)), value));
+            EmitWave64Barrier();
+            return Load(_uintType, Wave64ExchangePointer(exchange, UInt(0)));
+        }
+
+        private void EmitExactWaveMasks(IReadOnlyList<uint> pairs)
+        {
+            var lane = GuestWaveLane();
+            var upperHalf = ShiftRightLogical(lane, UInt(5));
+            var lowerHalf = _module.AddInstruction(SpirvOp.IEqual, _boolType, upperHalf, UInt(0));
+            const int pairsPerExchange = (int)Wave64ExchangeSlotCount / 2;
+            for (var start = 0; start < pairs.Count; start += pairsPerExchange)
+            {
+                var count = Math.Min(pairsPerExchange, pairs.Count - start);
+                var owned = new uint[count];
+                for (var index = 0; index < count; index++)
+                {
+                    var pair = pairs[start + index];
+                    owned[index] = _module.AddInstruction(SpirvOp.Select, _uintType, lowerHalf, LoadS(pair), LoadS(pair + 1));
+                }
+
+                var exchange = BeginWave64Exchange();
+                EmitConditional(IsHalfWaveLeader(lane), () =>
+                {
+                    for (var index = 0; index < count; index++)
+                    {
+                        Store(Wave64ExchangePointer(exchange, IAdd(UInt((uint)index * 2), upperHalf)), owned[index]);
+                    }
+                });
+                EmitWave64Barrier();
+                for (var index = 0; index < count; index++)
+                {
+                    var pair = pairs[start + index];
+                    StoreS(pair, Load(_uintType, Wave64ExchangePointer(exchange, UInt((uint)index * 2))));
+                    StoreS(pair + 1, Load(_uintType, Wave64ExchangePointer(exchange, UInt((uint)index * 2 + 1))));
+                }
+            }
+        }
+
+        private uint WaveMaskAny(uint register, uint laneActive) =>
+            _emulateWave64 && _subgroupInvocationIdInput != 0
+                ? IsNotZero64(LoadS64(register))
+                : SubgroupAny(Load(_boolType, laneActive));
+
+        private static readonly bool HalfWaveMasks = !string.Equals(
+            Environment.GetEnvironmentVariable("SHARPEMU_WAVE64_HALF_MASKS"), "0", StringComparison.Ordinal);
+
+        private bool _halfMaskPlanBuilt;
+
+        private Ir.Gen5Wave64HalfMaskPlan? HalfMaskPlan()
+        {
+            if (_halfMaskPlanBuilt)
+            {
+                return _halfMaskPlan;
+            }
+
+            _halfMaskPlanBuilt = true;
+            if (HalfWaveMasks && UsesWave64Exchange() && _subgroupInvocationIdInput != 0)
+            {
+                var sharedFlatPcs = _request.Memory.Entries
+                    .Where(static memory => memory.AddressSpace is FlatAddressSpace.Shared or FlatAddressSpace.SharedOrPrivate)
+                    .Select(static memory => memory.Pc)
+                    .ToHashSet();
+                _halfMaskPlan = Ir.Gen5Wave64HalfMaskAnalysis.Analyze(_request.Program, sharedFlatPcs);
+            }
+
+            return _halfMaskPlan;
+        }
 
         private void EmitWave64Barrier()
         {
@@ -7635,6 +7768,17 @@ public static partial class Gen5SpirvTranslator
                 return;
             }
 
+            if (_emulateWave64 &&
+                _subgroupInvocationIdInput != 0 &&
+                _emittingPc is { } pc &&
+                HalfMaskPlan() is { } plan &&
+                plan.HalfMaskWrites.TryGetValue(pc, out var halfRegister) &&
+                halfRegister == register)
+            {
+                StoreS64(register, BooleanToHalfWaveMask(condition));
+                return;
+            }
+
             StoreS64(register, BooleanToWaveMask(condition));
         }
 

src/SharpEmu.ShaderCompiler/Ir/Gen5Wave64HalfMaskAnalysis.cs

@@ -0,0 +1,565 @@
+// Copyright (C) 2026 SharpEmu Emulator Project
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+namespace SharpEmu.ShaderCompiler.Ir;
+
+public sealed class Gen5Wave64HalfMaskPlan
+{
+    public required IReadOnlyDictionary<uint, uint[]> ExactPairsBefore { get; init; }
+
+    public required IReadOnlyDictionary<uint, uint> HalfMaskWrites { get; init; }
+
+    public required IReadOnlySet<uint> SharedMemoryBarriersBefore { get; init; }
+}
+
+public static class Gen5Wave64HalfMaskAnalysis
+{
+    private const uint Vcc = 106;
+    private const uint Exec = 126;
+    private const uint VcczSource = 251;
+    private const uint ExeczSource = 252;
+    private const uint SccSource = 253;
+
+    private static readonly UInt128 EvenRegisters = new(0x5555_5555_5555_5555UL, 0x5555_5555_5555_5555UL);
+    private static readonly UInt128 OddRegisters = EvenRegisters << 1;
+
+    private static readonly HashSet<string> CarryIn = new(StringComparer.Ordinal)
+    {
+        "VAddcU32", "VAddCoCiU32", "VSubbU32", "VSubCoCiU32", "VSubbrevU32", "VSubrevCoCiU32",
+    };
+
+    private static readonly HashSet<string> CarryOut = new(CarryIn, StringComparer.Ordinal)
+    {
+        "VAddCoU32", "VSubCoU32", "VSubrevCoU32", "VMadU64U32", "VMadI64I32",
+    };
+
+    private static readonly HashSet<string> BitwiseWithScc = new(StringComparer.Ordinal)
+    {
+        "SNotB64", "SAndB64", "SOrB64", "SXorB64", "SAndn1B64", "SAndn2B64", "SOrn1B64", "SOrn2B64",
+        "SNandB64", "SNorB64", "SXnorB64",
+        "SAndSaveexecB64", "SOrSaveexecB64", "SXorSaveexecB64", "SAndn1SaveexecB64", "SAndn2SaveexecB64",
+        "SOrn1SaveexecB64", "SOrn2SaveexecB64", "SNandSaveexecB64", "SNorSaveexecB64", "SXnorSaveexecB64",
+    };
+
+    private static readonly HashSet<string> BitwiseWithoutScc = new(StringComparer.Ordinal)
+    {
+        "SMovB64", "SWqmB64", "SCselectB64",
+    };
+
+    private static readonly HashSet<string> SccReaders = new(StringComparer.Ordinal)
+    {
+        "SCbranchScc0", "SCbranchScc1", "SAddcU32", "SSubbU32", "SCselectB32", "SCselectB64",
+    };
+
+    private static readonly HashSet<string> SccWriters = new(StringComparer.Ordinal)
+    {
+        "SNotB32", "SAbsI32", "SWqmB32", "SBrevB32", "SBcnt1I32B32", "SBcnt1I32B64",
+        "SAddU32", "SSubU32", "SAddI32", "SSubI32", "SAddcU32", "SSubbU32",
+        "SAndB32", "SOrB32", "SXorB32", "SAndn2B32", "SOrn2B32", "SNandB32", "SNorB32", "SXnorB32",
+        "SLshlB32", "SLshrB32", "SAshrI32", "SBfeU32", "SBfeI32", "SMinU32", "SMinI32", "SMaxU32", "SMaxI32",
+        "SLshlB64", "SLshrB64", "SAshrI64", "SBfeU64", "SBfeI64", "SBfmB64",
+    };
+
+    private static readonly HashSet<string> DynamicControlFlow = new(StringComparer.Ordinal)
+    {
+        "SSetpcB64", "SSwappcB64", "SRfeB64", "SCbranchJoin", "SCbranchGFork", "SCbranchIFork",
+    };
+
+    private readonly record struct State(bool Reached, UInt128 Half, bool PendingRead, bool PendingWrite)
+    {
+        public static State Meet(State left, State right) =>
+            !left.Reached ? right
+            : !right.Reached ? left
+            : new State(true, left.Half | right.Half, left.PendingRead | right.PendingRead, left.PendingWrite | right.PendingWrite);
+    }
+
+    public static Gen5Wave64HalfMaskPlan? Analyze(Gen5ShaderProgram program, IReadOnlySet<uint> sharedFlatPcs)
+    {
+        var instructions = program.Instructions;
+        if (instructions.Count == 0)
+        {
+            return null;
+        }
+
+        var successors = BuildSuccessors(instructions);
+        if (successors is null)
+        {
+            return null;
+        }
+
+        var sccLiveAfter = ComputeSccLiveAfter(instructions, successors);
+        var states = new State[instructions.Count];
+        states[0] = new State(true, 0, false, false);
+        var pending = new Queue<int>();
+        var queued = new bool[instructions.Count];
+        pending.Enqueue(0);
+        queued[0] = true;
+        while (pending.TryDequeue(out var index))
+        {
+            queued[index] = false;
+            var output = Step(instructions[index], states[index], sccLiveAfter[index], sharedFlatPcs, out _, out _, out _);
+            foreach (var successor in successors[index])
+            {
+                var merged = State.Meet(states[successor], output);
+                if (merged != states[successor])
+                {
+                    states[successor] = merged;
+                    if (!queued[successor])
+                    {
+                        queued[successor] = true;
+                        pending.Enqueue(successor);
+                    }
+                }
+            }
+        }
+
+        var exactPairs = new Dictionary<uint, uint[]>();
+        var halfWrites = new Dictionary<uint, uint>();
+        var barriers = new HashSet<uint>();
+        for (var index = 0; index < instructions.Count; index++)
+        {
+            if (!states[index].Reached)
+            {
+                continue;
+            }
+
+            var instruction = instructions[index];
+            Step(instruction, states[index], sccLiveAfter[index], sharedFlatPcs, out var exact, out var barrier, out var halfWrite);
+            if (exact != 0)
+            {
+                exactPairs[instruction.Pc] = PairBases(exact);
+            }
+
+            if (barrier)
+            {
+                barriers.Add(instruction.Pc);
+            }
+
+            if (halfWrite && TryGetHalfMaskDestination(instruction, out var maskDestination))
+            {
+                halfWrites[instruction.Pc] = maskDestination;
+            }
+        }
+
+        return new Gen5Wave64HalfMaskPlan
+        {
+            ExactPairsBefore = exactPairs,
+            HalfMaskWrites = halfWrites,
+            SharedMemoryBarriersBefore = barriers,
+        };
+    }
+
+    private static State Step(
+        Gen5ShaderInstruction instruction,
+        State input,
+        bool sccLiveAfter,
+        IReadOnlySet<uint> sharedFlatPcs,
+        out UInt128 exact,
+        out bool sharedBarrier,
+        out bool halfWrite)
+    {
+        var opcode = instruction.Opcode;
+        var half = input.Half;
+        var bitwise = IsBitwiseMaskOperation(instruction);
+        UInt128 needed = 0;
+        UInt128 bitwiseInputs = 0;
+        for (var index = 0; index < instruction.Sources.Count; index++)
+        {
+            var operand = instruction.Sources[index];
+            if (operand.Kind == Gen5OperandKind.EncodedConstant)
+            {
+                if (operand.Value == VcczSource)
+                {
+                    needed |= Pair(Vcc);
+                }
+                else if (operand.Value == ExeczSource)
+                {
+                    needed |= Pair(Exec);
+                }
+
+                continue;
+            }
+
+            if (operand.Kind != Gen5OperandKind.ScalarRegister)
+            {
+                continue;
+            }
+
+            if (bitwise)
+            {
+                bitwiseInputs |= Pair(operand.Value);
+                continue;
+            }
+
+            if (index == 2 && (opcode == "VCndmaskB32" || CarryIn.Contains(opcode)))
+            {
+                continue;
+            }
+
+            needed |= ReadRegisters(instruction, index, operand.Value);
+        }
+
+        if (opcode is "SCbranchExecz" or "SCbranchExecnz")
+        {
+            needed |= Pair(Exec);
+        }
+        else if (opcode is "SCbranchVccz" or "SCbranchVccnz")
+        {
+            needed |= Pair(Vcc);
+        }
+
+        if ((instruction.Encoding == Gen5ShaderEncoding.Sopk || opcode.StartsWith("SBitset", StringComparison.Ordinal)) &&
+            instruction.Destinations is [{ Kind: Gen5OperandKind.ScalarRegister } readDestination, ..])
+        {
+            needed |= Range(readDestination.Value, opcode.Contains("64", StringComparison.Ordinal) ? 2u : 1u);
+        }
+
+        if (opcode.StartsWith("SMovrel", StringComparison.Ordinal))
+        {
+            needed |= half;
+        }
+
+        if (bitwise)
+        {
+            if (opcode.EndsWith("SaveexecB64", StringComparison.Ordinal))
+            {
+                bitwiseInputs |= Pair(Exec);
+            }
+
+            if (sccLiveAfter && BitwiseWithScc.Contains(opcode))
+            {
+                needed |= bitwiseInputs;
+            }
+        }
+
+        exact = Expand(needed & half);
+        half &= ~exact;
+
+        var pendingRead = input.PendingRead;
+        var pendingWrite = input.PendingWrite;
+        if (exact != 0)
+        {
+            pendingRead = false;
+            pendingWrite = false;
+        }
+
+        var (reads, writes) = SharedMemoryAccess(instruction, sharedFlatPcs);
+        sharedBarrier = (writes && (pendingRead || pendingWrite)) || (reads && pendingWrite);
+        if (sharedBarrier)
+        {
+            pendingRead = false;
+            pendingWrite = false;
+        }
+
+        pendingRead |= reads;
+        pendingWrite |= writes;
+        if (opcode == "SBarrier")
+        {
+            pendingRead = false;
+            pendingWrite = false;
+        }
+
+        halfWrite = false;
+        if (TryGetHalfMaskDestination(instruction, out var maskDestination))
+        {
+            half |= Pair(maskDestination);
+            halfWrite = true;
+        }
+        else if (bitwise)
+        {
+            var destination = instruction.Destinations[0].Value;
+            var outputHalf = (bitwiseInputs & half) != 0;
+            half = outputHalf ? half | Pair(destination) : half & ~Pair(destination);
+            if (opcode.EndsWith("SaveexecB64", StringComparison.Ordinal))
+            {
+                half = outputHalf ? half | Pair(Exec) : half & ~Pair(Exec);
+            }
+        }
+        else
+        {
+            var wide = IsWideScalar(opcode);
+            foreach (var destination in instruction.Destinations)
+            {
+                if (destination.Kind != Gen5OperandKind.ScalarRegister)
+                {
+                    continue;
+                }
+
+                half &= ~Bit(destination.Value);
+                if (wide)
+                {
+                    half &= ~Bit(destination.Value + 1);
+                }
+            }
+
+            if (opcode.EndsWith("SaveexecB32", StringComparison.Ordinal))
+            {
+                half &= ~Pair(Exec);
+            }
+        }
+
+        return new State(true, half, pendingRead, pendingWrite);
+    }
+
+    private static bool TryGetHalfMaskDestination(Gen5ShaderInstruction instruction, out uint destination)
+    {
+        var opcode = instruction.Opcode;
+        if (opcode.StartsWith("VCmpx", StringComparison.Ordinal))
+        {
+            destination = Exec;
+        }
+        else if (opcode.StartsWith("VCmp", StringComparison.Ordinal))
+        {
+            destination = instruction.Control switch
+            {
+                Gen5SdwaControl { ScalarDestination: { } sdwa } => sdwa,
+                Gen5Vop3Control { ScalarDestination: { } vop3 } => vop3,
+                _ => Vcc,
+            };
+        }
+        else if (CarryOut.Contains(opcode))
+        {
+            destination = instruction.Control is Gen5Vop3Control { ScalarDestination: { } carry } ? carry : Vcc;
+        }
+        else
+        {
+            destination = 0;
+            return false;
+        }
+
+        return (destination & 1) == 0 && (destination < 124 || destination == Exec);
+    }
+
+    private static bool IsBitwiseMaskOperation(Gen5ShaderInstruction instruction)
+    {
+        if (!BitwiseWithScc.Contains(instruction.Opcode) && !BitwiseWithoutScc.Contains(instruction.Opcode))
+        {
+            return false;
+        }
+
+        if (instruction.Destinations is not [{ Kind: Gen5OperandKind.ScalarRegister } destination] ||
+            (destination.Value & 1) != 0 ||
+            (destination.Value >= 124 && destination.Value != Exec))
+        {
+            return false;
+        }
+
+        foreach (var source in instruction.Sources)
+        {
+            if (source.Kind == Gen5OperandKind.VectorRegister)
+            {
+                return false;
+            }
+
+            if (source.Kind == Gen5OperandKind.ScalarRegister &&
+                ((source.Value & 1) != 0 || (source.Value >= 124 && source.Value != Exec)))
+            {
+                return false;
+            }
+        }
+
+        return true;
+    }
+
+    private static (bool Reads, bool Writes) SharedMemoryAccess(Gen5ShaderInstruction instruction, IReadOnlySet<uint> sharedFlatPcs)
+    {
+        if (instruction.Control is Gen5DataShareControl { Gds: false })
+        {
+            var opcode = instruction.Opcode;
+            if (opcode is "DsSwizzleB32" or "DsBpermuteB32" or "DsPermuteB32" or "DsNop")
+            {
+                return (false, false);
+            }
+
+            if (opcode.StartsWith("DsRead", StringComparison.Ordinal))
+            {
+                return (true, false);
+            }
+
+            if (opcode.StartsWith("DsWrite", StringComparison.Ordinal))
+            {
+                return (false, true);
+            }
+
+            return (true, true);
+        }
+
+        return sharedFlatPcs.Contains(instruction.Pc) ? (true, true) : (false, false);
+    }
+
+    private static int[][]? BuildSuccessors(IReadOnlyList<Gen5ShaderInstruction> instructions)
+    {
+        var indexByPc = new Dictionary<uint, int>(instructions.Count);
+        for (var index = 0; index < instructions.Count; index++)
+        {
+            if (DynamicControlFlow.Contains(instructions[index].Opcode))
+            {
+                return null;
+            }
+
+            indexByPc[instructions[index].Pc] = index;
+        }
+
+        var last = instructions[^1];
+        var endPc = last.Pc + (uint)(last.Words.Count * sizeof(uint));
+        var successors = new int[instructions.Count][];
+        for (var index = 0; index < instructions.Count; index++)
+        {
+            var instruction = instructions[index];
+            var next = index + 1 < instructions.Count ? index + 1 : -1;
+            if (Gen5IrBranchResolver.IsTerminator(instruction))
+            {
+                successors[index] = [];
+                continue;
+            }
+
+            var unconditional = Gen5IrBranchResolver.IsUnconditionalBranch(instruction);
+            if (!unconditional && !Gen5IrBranchResolver.Instance.IsConditional(instruction))
+            {
+                successors[index] = next < 0 ? [] : [next];
+                continue;
+            }
+
+            if (!Gen5IrBranchResolver.Instance.TryGetBranchTarget(instruction, out var targetPc))
+            {
+                return null;
+            }
+
+            var targets = new List<int>(2);
+            if (indexByPc.TryGetValue(targetPc, out var target))
+            {
+                targets.Add(target);
+            }
+            else if (targetPc < endPc)
+            {
+                return null;
+            }
+
+            if (!unconditional && next >= 0)
+            {
+                targets.Add(next);
+            }
+
+            successors[index] = [.. targets];
+        }
+
+        return successors;
+    }
+
+    private static bool[] ComputeSccLiveAfter(IReadOnlyList<Gen5ShaderInstruction> instructions, int[][] successors)
+    {
+        var liveBefore = new bool[instructions.Count];
+        var liveAfter = new bool[instructions.Count];
+        bool changed;
+        do
+        {
+            changed = false;
+            for (var index = instructions.Count - 1; index >= 0; index--)
+            {
+                var after = false;
+                foreach (var successor in successors[index])
+                {
+                    after |= liveBefore[successor];
+                }
+
+                var instruction = instructions[index];
+                var before = ReadsScc(instruction) || (!WritesScc(instruction) && after);
+                if (after != liveAfter[index] || before != liveBefore[index])
+                {
+                    liveAfter[index] = after;
+                    liveBefore[index] = before;
+                    changed = true;
+                }
+            }
+        }
+        while (changed);
+
+        return liveAfter;
+    }
+
+    private static bool ReadsScc(Gen5ShaderInstruction instruction)
+    {
+        if (SccReaders.Contains(instruction.Opcode))
+        {
+            return true;
+        }
+
+        foreach (var source in instruction.Sources)
+        {
+            if (source.Kind == Gen5OperandKind.EncodedConstant && source.Value == SccSource)
+            {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    private static bool WritesScc(Gen5ShaderInstruction instruction) =>
+        instruction.Encoding == Gen5ShaderEncoding.Sopc ||
+        instruction.Opcode.StartsWith("SCmpk", StringComparison.Ordinal) ||
+        instruction.Opcode.EndsWith("SaveexecB32", StringComparison.Ordinal) ||
+        SccWriters.Contains(instruction.Opcode) ||
+        BitwiseWithScc.Contains(instruction.Opcode);
+
+    private static bool IsWideScalar(string opcode) =>
+        opcode.StartsWith('S') &&
+        (opcode.EndsWith("B64", StringComparison.Ordinal) ||
+         opcode.EndsWith("U64", StringComparison.Ordinal) ||
+         opcode.EndsWith("I64", StringComparison.Ordinal));
+
+    private static UInt128 ReadRegisters(Gen5ShaderInstruction instruction, int sourceIndex, uint register)
+    {
+        var scalarOrdinal = 0;
+        for (var index = 0; index < sourceIndex; index++)
+        {
+            if (instruction.Sources[index].Kind == Gen5OperandKind.ScalarRegister)
+            {
+                scalarOrdinal++;
+            }
+        }
+
+        var count = instruction.Encoding switch
+        {
+            Gen5ShaderEncoding.Smem or Gen5ShaderEncoding.Smrd => scalarOrdinal != 0 ? 1u
+                : instruction.Opcode.StartsWith("SLoad", StringComparison.Ordinal) ? 2u : 4u,
+            Gen5ShaderEncoding.Mubuf or Gen5ShaderEncoding.Mtbuf => scalarOrdinal == 0 ? 4u : 1u,
+            Gen5ShaderEncoding.Mimg => scalarOrdinal == 0 ? 8u : 4u,
+            Gen5ShaderEncoding.Flat => 2u,
+            _ => instruction.Opcode.Contains("64", StringComparison.Ordinal) ? 2u : 1u,
+        };
+        return Range(register, count);
+    }
+
+    private static UInt128 Range(uint register, uint count)
+    {
+        UInt128 registers = 0;
+        for (var offset = 0u; offset < count; offset++)
+        {
+            registers |= Bit(register + offset);
+        }
+
+        return registers;
+    }
+
+    private static UInt128 Expand(UInt128 registers) =>
+        registers | ((registers & EvenRegisters) << 1) | ((registers & OddRegisters) >> 1);
+
+    private static uint[] PairBases(UInt128 registers)
+    {
+        var bases = new List<uint>();
+        for (uint register = 0; register < 128; register += 2)
+        {
+            if ((registers & Pair(register)) != 0)
+            {
+                bases.Add(register);
+            }
+        }
+
+        return [.. bases];
+    }
+
+    private static UInt128 Pair(uint register) => Expand(Bit(register));
+
+    private static UInt128 Bit(uint register) => register < 128 ? UInt128.One << (int)register : 0;
+}

src/SharpEmu.ShaderCompiler/Resources/ResourceTracker.DenseImages.cs

@@ -1,6 +1,8 @@
 // Copyright (C) 2026 SharpEmu Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
+using SharpEmu.ShaderCompiler.Ir;
+
 namespace SharpEmu.ShaderCompiler.Resources;
 
 public sealed partial class ResourceTracker
@@ -195,9 +197,11 @@ public sealed partial class ResourceTracker
 
         var instructions = _graph.Program.Instructions;
         var firstLaneIndex = FindInstructionIndex(instructions, (uint)key.Payload);
-        if (firstLaneIndex < 0 || instructions[firstLaneIndex] is not { Opcode: "VReadfirstlaneB32", Sources.Count: 1, Destinations.Count: 1 } firstLane ||
+        if (firstLaneIndex < 0 || instructions[firstLaneIndex] is not { Destinations.Count: 1 } firstLane ||
+            firstLane.Sources.Count == 0 ||
             firstLane.Sources[0] is not { Kind: Gen5OperandKind.VectorRegister } vectorSource ||
-            firstLane.Destinations[0] is not { Kind: Gen5OperandKind.ScalarRegister } scalarDestination)
+            firstLane.Destinations[0] is not { Kind: Gen5OperandKind.ScalarRegister } ||
+            !(firstLane is { Opcode: "VReadfirstlaneB32", Sources.Count: 1 } || ReadsActiveLane(instructions, firstLaneIndex)))
         {
             return null;
         }
@@ -222,12 +226,8 @@ public sealed partial class ResourceTracker
         var descriptorLoadIndex = FindInstructionIndex(instructions, _plan.Memory[descriptorMemoryIndex].Pc);
         if (descriptorLoadIndex < 0 || instructions[descriptorLoadIndex] is not
             {
-                Control: Gen5ScalarMemoryControl
-            {
-                DestinationCount: >= 8,
-                DynamicOffsetRegister: { } dynamicOffsetRegister,
-            },
-            } descriptorLoad || dynamicOffsetRegister != scalarDestination.Value ||
+                Control: Gen5ScalarMemoryControl { DynamicOffsetRegister: not null },
+            } descriptorLoad ||
             descriptorLoad.Sources.Count == 0 || descriptorLoad.Sources[0] != Gen5Operand.Scalar(global.ScalarAddress))
         {
             return null;
@@ -244,8 +244,7 @@ public sealed partial class ResourceTracker
         if (TryGetSelfAddedConstant(instructions[addressDefinitionIndex], address, out _) &&
             TryGetWaveIndexedStride(instructions, addressDefinitionIndex, address, heapAddress,
                 out var maskRegister, out var bitRegister, out var maskOffset, out var indexStride) &&
-            indexDataOffset != 0 && indexStride != 0 && instructions.Any(instruction => instruction.Opcode == "SBitset0B32" &&
-                instruction.Destinations.Contains(maskRegister) && instruction.Sources.Contains(bitRegister)))
+            indexDataOffset != 0 && indexStride != 0 && ClearsMaskBit(instructions, 0, instructions.Count, maskRegister, bitRegister))
         {
             return new(maskOffset, indexDataOffset, indexStride);
         }
@@ -283,9 +282,7 @@ public sealed partial class ResourceTracker
             {
                 if (instructions[scanIndex] is not { Opcode: "SFF1I32B32", Sources.Count: 1, Destinations.Count: 1 } scan ||
                     scan.Sources[0] != maskRegister || scan.Destinations[0] is not { Kind: Gen5OperandKind.ScalarRegister } bitRegister ||
-                    !instructions.Skip(scanIndex + 1).Take(before - scanIndex - 1)
-                        .Any(instruction => instruction.Opcode == "SBitset0B32" && instruction.Destinations.Contains(maskRegister) &&
-                            instruction.Sources.Contains(bitRegister)))
+                    !ClearsMaskBit(instructions, scanIndex + 1, before, maskRegister, bitRegister))
                     continue;
 
                 for (var multiplyIndex = scanIndex + 1; multiplyIndex < before; multiplyIndex++)
@@ -358,6 +355,130 @@ public sealed partial class ResourceTracker
         return true;
     }
 
+    private static bool ClearsMaskBit(
+        IReadOnlyList<Gen5ShaderInstruction> instructions,
+        int start,
+        int end,
+        Gen5Operand mask,
+        Gen5Operand bit)
+    {
+        end = Math.Min(end, instructions.Count);
+        for (var index = Math.Max(0, start); index < end; index++)
+        {
+            var instruction = instructions[index];
+            if (instruction.Opcode == "SBitset0B32" && instruction.Destinations.Contains(mask) && instruction.Sources.Contains(bit))
+                return true;
+
+            if (instruction is not { Opcode: "SLshlB32", Sources.Count: 2, Destinations.Count: 1 } shift ||
+                !TryGetConstant(shift.Sources[0], out var one) || one != 1 || shift.Sources[1] != bit ||
+                shift.Destinations[0] is not { Kind: Gen5OperandKind.ScalarRegister } single)
+                continue;
+
+            for (var clearIndex = index + 1; clearIndex < end; clearIndex++)
+            {
+                var clear = instructions[clearIndex];
+                if (clear.Destinations.Contains(mask) && clear.Sources.Count == 2 &&
+                    ((clear.Opcode == "SXorB32" && clear.Sources.Contains(mask) && clear.Sources.Contains(single)) ||
+                     (clear.Opcode == "SAndn2B32" && clear.Sources[0] == mask && clear.Sources[1] == single)))
+                    return true;
+
+                if (clear.Destinations.Contains(single) || clear.Destinations.Contains(mask) || clear.Destinations.Contains(bit) ||
+                    (single.Value is 106 or 107 && clear.Opcode.StartsWith('V')))
+                    break;
+            }
+        }
+
+        return false;
+    }
+
+    private static bool ReadsActiveLane(IReadOnlyList<Gen5ShaderInstruction> instructions, int readLaneIndex)
+    {
+        var exec = Gen5Operand.Scalar(126);
+        var readLane = instructions[readLaneIndex];
+        if (readLane is not { Opcode: "VReadlaneB32" } || readLane.Sources.Count < 2 ||
+            readLane.Sources[1] is not { Kind: Gen5OperandKind.ScalarRegister } lane)
+            return false;
+
+        var scanIndex = FindLastDefinition(instructions, readLaneIndex, lane);
+        if (scanIndex < 0 || instructions[scanIndex] is not { Opcode: "SFF1I32B64", Sources.Count: 1 } scan ||
+            scan.Sources[0] is not { Kind: Gen5OperandKind.ScalarRegister } candidates)
+            return false;
+
+        var copyIndex = FindLastDefinition(instructions, scanIndex, candidates);
+        if (copyIndex < 0 || instructions[copyIndex] is not { Opcode: "SMovB64", Sources.Count: 1 } copy || copy.Sources[0] != exec)
+            return false;
+
+        for (var index = copyIndex + 1; index < readLaneIndex; index++)
+        {
+            if (WritesScalarPair(instructions[index], exec) || WritesScalarPair(instructions[index], candidates) ||
+                Gen5IrBranchResolver.Instance.TryGetBranchTarget(instructions[index], out _))
+                return false;
+        }
+
+        var backIndex = -1;
+        for (var index = readLaneIndex + 1; index < instructions.Count; index++)
+        {
+            if (!Gen5IrBranchResolver.Instance.TryGetBranchTarget(instructions[index], out var target))
+                continue;
+            if (target > instructions[copyIndex].Pc && target <= instructions[scanIndex].Pc)
+            {
+                backIndex = index;
+                break;
+            }
+        }
+
+        if (backIndex < 0)
+            return false;
+
+        var restoreIndex = -1;
+        for (var index = backIndex - 1; index > readLaneIndex; index--)
+        {
+            if (WritesScalarPair(instructions[index], exec))
+            {
+                restoreIndex = index;
+                break;
+            }
+        }
+
+        if (restoreIndex < 0 || instructions[restoreIndex] is not { Opcode: "SMovB64", Sources.Count: 1 } restore ||
+            restore.Sources[0] is not { Kind: Gen5OperandKind.ScalarRegister } saved)
+            return false;
+
+        var saveIndex = FindLastDefinition(instructions, restoreIndex, saved);
+        if (saveIndex <= readLaneIndex || instructions[saveIndex].Opcode != "SAndSaveexecB64")
+            return false;
+
+        for (var index = readLaneIndex + 1; index < backIndex; index++)
+        {
+            var instruction = instructions[index];
+            if (index > saveIndex && index < restoreIndex && WritesScalarPair(instruction, saved))
+                return false;
+            if (WritesScalarPair(instruction, candidates) &&
+                (instruction is not { Opcode: "SAndn2B64", Sources.Count: 2 } || instruction.Sources[0] != candidates))
+                return false;
+            if (Gen5IrBranchResolver.Instance.TryGetBranchTarget(instruction, out var target) &&
+                (target <= instruction.Pc || target > instructions[restoreIndex].Pc))
+                return false;
+        }
+
+        return true;
+    }
+
+    private static bool WritesScalarPair(Gen5ShaderInstruction instruction, Gen5Operand register)
+    {
+        if (register.Value is 106 or 107 && instruction.Opcode.StartsWith('V'))
+            return true;
+        if (instruction.Control is Gen5Vop3Control { ScalarDestination: { } scalarDestination } &&
+            scalarDestination + 1 >= register.Value && scalarDestination <= register.Value + 1)
+            return true;
+        if (register.Value == 126 && (instruction.Opcode.Contains("Saveexec", StringComparison.Ordinal) ||
+            instruction.Opcode.StartsWith("VCmpx", StringComparison.Ordinal)))
+            return true;
+        var width = instruction.Opcode.Contains("64", StringComparison.Ordinal) ? 2u : 1u;
+        return instruction.Destinations.Any(destination => destination.Kind == Gen5OperandKind.ScalarRegister &&
+            destination.Value + width > register.Value && destination.Value <= register.Value + 1);
+    }
+
     private static int FindLastDefinition(IReadOnlyList<Gen5ShaderInstruction> instructions, int before, Gen5Operand destination)
     {
         for (var index = before - 1; index >= 0; index--)

src/SharpEmu.ShaderCompiler/Resources/ScalarValueGraph.Builder.cs

@@ -1558,15 +1558,22 @@ public sealed partial class ScalarValueGraph
 
             var lane = instruction.Sources.Count > 1 ? Read(instruction.Sources[1], state) : _graph.Undefined(ScalarValueType.U32);
             if (instruction.Sources.Count < 2 ||
-                instruction.Sources[0] is not { Kind: Gen5OperandKind.VectorRegister } source ||
-                !lane.IsConstant ||
-                !state.Lanes.TryGetValue((source.Value, lane.ConstantU32 & 63), out var value))
+                instruction.Sources[0] is not { Kind: Gen5OperandKind.VectorRegister } source)
             {
                 state.WriteScalar(destination.Value, _graph.Undefined(ScalarValueType.U32));
                 return;
             }
 
-            state.WriteScalar(destination.Value, value);
+            if (lane.IsConstant && state.Lanes.TryGetValue((source.Value, lane.ConstantU32 & 63), out var value))
+            {
+                state.WriteScalar(destination.Value, value);
+                return;
+            }
+
+            var sourceValue = state.ReadVector(source.Value);
+            state.WriteScalar(destination.Value, !lane.IsConstant && sourceValue.IsUndefined
+                ? _graph.FirstLane(sourceValue, state.Exec, instruction.Pc)
+                : _graph.Undefined(ScalarValueType.U32));
         }
 
         // ---- memory instructions ----

src/SharpEmu.ShaderCompiler/Resources/ShaderResourcePlan.cs

@@ -181,7 +181,8 @@ public sealed class ShaderResourcePlan
             var value = TableReads[slot].Value;
             if (value.Kind == ScalarValueKind.ScalarAddressWord && value.Operands.Length != 0 &&
                 value.Operands[0].Kind == ScalarValueKind.AddressHandle &&
-                value.Operands[0].Operands.SequenceEqual(heap.Dwords))
+                value.Operands[0].Operands.Length == heap.Dwords.Length &&
+                value.Operands[0].Operands.Zip(heap.Dwords).All(pair => ReferenceEquals(pair.First, pair.Second) || Graph.Equivalent(pair.First, pair.Second)))
             {
                 slots[slot] = 1;
             }

src/SharpEmu.ShaderCompiler/ShaderCompileRequest.cs

@@ -186,6 +186,7 @@ public sealed class ShaderCompileRequest
     public uint LocalSizeX { get; init; } = 1;
     public uint LocalSizeY { get; init; } = 1;
     public uint LocalSizeZ { get; init; } = 1;
+    public uint LocalDataShareDwords { get; init; }
 
     // Fixed bounds for standalone modules; runtime-limit layouts read the dispatch data.
     public uint ThreadCountX { get; init; } = UnboundedThreadCount;

tests/SharpEmu.Libs.Tests/Gpu/Images/GuestImageCacheTests.Metadata.cs

@@ -156,6 +156,36 @@ public sealed partial class GuestImageCacheTests
         harness.Shutdown();
     }
 
+    [Fact]
+    public void GuestDccClear_ReadsOnlyUniformClearSlices()
+    {
+        if (!GatePrerequisites.Ready(_vulkan)) return;
+        using var harness = new CacheHarness(_vulkan);
+        const ulong SliceSize = 0x2000;
+        var metadata = harness.MapBacked(0x10000, ReadWrite);
+        harness.Write(metadata, Enumerable.Repeat((byte)0x80, (int)SliceSize).ToArray());
+        harness.Write(metadata + SliceSize, Enumerable.Repeat((byte)0x00, (int)SliceSize).ToArray());
+        var mixed = Enumerable.Repeat((byte)0x40, (int)SliceSize).ToArray();
+        mixed[^1] = 0xff;
+        harness.Write(metadata + 2 * SliceSize, mixed);
+        harness.Write(metadata + 3 * SliceSize, Enumerable.Repeat((byte)0xff, (int)SliceSize).ToArray());
+
+        Assert.True(harness.Images.TryReadGuestDccClear(metadata, SliceSize, 0, out var slice, out var code));
+        Assert.Equal(metadata, slice);
+        Assert.Equal(0x80, code);
+        Assert.True(harness.Images.TryReadGuestDccClear(metadata, SliceSize, 1, out slice, out code));
+        Assert.Equal(metadata + SliceSize, slice);
+        Assert.Equal(0x00, code);
+        Assert.False(harness.Images.TryReadGuestDccClear(metadata, SliceSize, 2, out _, out _));
+        Assert.False(harness.Images.TryReadGuestDccClear(metadata, SliceSize, 3, out _, out _));
+        Assert.False(harness.Images.TryReadGuestDccClear(metadata, 0, 0, out _, out _));
+
+        harness.Worker.Run(() => harness.Cache.FillBuffer(metadata, SliceSize, uint.MaxValue, isGds: false));
+        Assert.False(harness.Images.TryReadGuestDccClear(metadata, SliceSize, 0, out _, out _));
+        Assert.True(harness.Images.TryReadGuestDccClear(metadata, SliceSize, 1, out _, out _));
+        harness.Shutdown();
+    }
+
     [Fact]
     public void Unregister_DropsMetadataInTheRange()
     {

tests/SharpEmu.Libs.Tests/Gpu/Images/ImageDescriptionTests.cs

@@ -259,6 +259,28 @@ public sealed class ImageDescriptionTests
         Reject(association, "stencil-association image", fatal);
     }
 
+    [Theory]
+    [InlineData(2560u, 1440u, 8u, 1u, 1u, 0x1E000ul)]
+    [InlineData(1920u, 1080u, 4u, 1u, 1u, 0xC000ul)]
+    [InlineData(3840u, 2160u, 4u, 1u, 1u, 0x28000ul)]
+    [InlineData(640u, 360u, 16u, 1u, 1u, 0x6000ul)]
+    [InlineData(1920u, 1080u, 12u, 1u, 1u, 0ul)]
+    [InlineData(1920u, 1080u, 4u, 2u, 1u, 0ul)]
+    [InlineData(1920u, 1080u, 4u, 1u, 2u, 0ul)]
+    public void DccSliceSize_CoversOneMetadataBytePerColorBlock(uint width, uint height, uint bytesPerElement, uint levels, uint samples, ulong expected)
+    {
+        var description = Color(Base, 0x100, width, height, levels);
+        description.BytesPerBlock = bytesPerElement;
+        description.Samples = samples;
+        description.TileMode = GuestTileMode.RenderTarget;
+        description.Metadata.Kind = MetadataKind.Dcc;
+        description.Metadata.Range = new GuestSpan(0x200000, 0);
+        Assert.Equal(expected, description.DccSliceSize);
+
+        description.TileMode = GuestTileMode.Linear;
+        Assert.Equal(0ul, description.DccSliceSize);
+    }
+
     [Fact]
     public void Predicates_ReadTheDescription()
     {

tests/SharpEmu.Libs.Tests/Gpu/Pipelines/ShaderPipelineCacheTests.cs

@@ -303,6 +303,26 @@ public sealed class ShaderPipelineCacheTests : IDisposable
         Assert.Equal(expectedMask, description.StaticParameters.GetColorMask(0));
     }
 
+    [Fact]
+    public void ColorTarget_WithoutAPixelStage_IsNotWritten()
+    {
+        var banks = Banks();
+        banks.Context.RenderTargetMask = 0xF;
+        var programs = Programs();
+        var resolution = new ColorTargetResolution(
+            default, 0x1000, 0x10000, new Extent2D(64, 64), 0, 0, 1, ColorComponentMap.Identity, false, false, default);
+        ColorTargetState[] colors = [new(in resolution, 0, new SharpEmu.Libs.Gpu.Buffers.ResourceSlotIdentifier(1, 1))];
+        var rendering = new RenderingState { Samples = 1, ColorAttachmentCount = 1 };
+        rendering.ColorAttachments[0] = new RenderingAttachment(
+            default, ImageLayout.ColorAttachmentOptimal, Format.R8G8B8A8Unorm, 0, 0, 0, 0, false, false, false, false, false);
+
+        var description = ShaderPipelineCache.BuildGraphicsDescription(
+            colors, default, programs.VertexInput, null, banks.Context, in rendering, PrimitiveTopology.TriangleList, false, false,
+            programs.Vertex, programs.Pixel, SampleCountFlags.Count1Bit);
+
+        Assert.Equal(0u, description.StaticParameters.GetColorMask(0));
+    }
+
     [Fact]
     public void NoAttachments_AtAnUnsupportedSampleCount_AreFatal()
     {

tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorStateTests.cs

@@ -338,6 +338,19 @@ public sealed class RenderExecutorStateTests : IDisposable
         Assert.Equal((expectedColors, expectedExtent, expectedExtent), (rendering.ColorAttachmentCount, rendering.Width, rendering.Height));
     }
 
+    [Fact]
+    public void Attachments_AColorTargetWithoutAnActivePixelShaderIsDroppedFromTheDepthPass()
+    {
+        var banks = Banks(withDepth: true);
+        banks.Context.ColorTargets[0] = RegisterWords.Color(ColorBase, 32, 32);
+        banks.Context.ShaderInterface.ColorShaderMask = 0;
+        _executor.DrawIndexed(1, banks, Indexed(3));
+
+        var rendering = _host.BegunRenderings[0];
+        Assert.Equal((0u, 64u, 64u), (rendering.ColorAttachmentCount, rendering.Width, rendering.Height));
+        Assert.Contains("create_graphics_pipeline colors=0 depth=True topology=TriangleList restart=False", _pipelines.Calls);
+    }
+
     [Fact]
     public void Attachments_StencilFormatIsReportedOnlyWithAStencilAspect()
     {

tests/SharpEmu.Libs.Tests/Gpu/Vulkan/MultiplyHighDeviceTests.cs

@@ -0,0 +1,69 @@
+// Copyright (C) 2026 SharpEmu Emulator Project
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+using System.Buffers.Binary;
+using SharpEmu.Libs.Gpu.Buffers;
+using SharpEmu.Libs.Tests.Gpu.Images;
+using SharpEmu.ShaderCompiler;
+using SharpEmu.ShaderCompiler.Resources;
+using SharpEmu.ShaderCompiler.Vulkan;
+using Xunit;
+using static SharpEmu.ShaderCompiler.Tests.Resources.ResourceTestProgram;
+
+namespace SharpEmu.Libs.Tests.Gpu.Vulkan;
+
+public sealed class MultiplyHighDeviceTests(HeadlessVulkanFixture fixture) : IClassFixture<HeadlessVulkanFixture>
+{
+    private const uint Factor = 0x9E37_79B9;
+    private const uint SignedFactor = 0x8000_0005;
+    private const ulong Addend = 0x1_FFFF_FFF0;
+    private const uint LaneBase = 0x7FFF_FFF0;
+    private const uint ThreadCount = 32;
+
+    [Fact]
+    public void WideProductsMatchTheHost()
+    {
+        var vulkan = fixture.Vulkan;
+        if (!GatePrerequisites.Ready(vulkan, shaderInt64: true)) return;
+        var program = Program(
+            Vop2(0, "VAddI32", 10, Operand(LaneBase), Gen5Operand.Vector(0)),
+            MoveScalar(8, 8, Factor),
+            MoveScalar(16, 9, SignedFactor),
+            MoveScalar(24, 10, unchecked((uint)Addend)),
+            MoveScalar(32, 11, (uint)(Addend >> 32)),
+            Vop3(40, "VMadU64U32", 2, Gen5Operand.Scalar(8), Gen5Operand.Vector(10), Gen5Operand.Scalar(10)),
+            Vop3(48, "VMulHiU32", 4, Gen5Operand.Vector(10), Gen5Operand.Scalar(9)),
+            Vop3(56, "VMulHiI32", 5, Gen5Operand.Vector(10), Gen5Operand.Scalar(9)),
+            Sop2(64, "SMulHiU32", 12, Gen5Operand.Scalar(8), Gen5Operand.Scalar(9)),
+            MoveVectorFromScalar(68, 6, 12),
+            Vop2(72, "VLshlrevB32", 7, Operand(5), Gen5Operand.Vector(0)),
+            BufferAccess(76, "BufferStoreDwordx4", 4, vectorData: 2, dwords: 4, offsetEnabled: true, vectorAddress: 7),
+            BufferAccess(84, "BufferStoreDword", 4, offset: 16, vectorData: 6, offsetEnabled: true, vectorAddress: 7),
+            EndProgram(92));
+        var (plan, resources, layout) = Prepare(program);
+        var request = new ShaderCompileRequest(plan, resources, layout)
+        {
+            LocalSizeX = ThreadCount, ThreadCountX = ThreadCount, WaveSize = 32,
+        };
+        Assert.True(Gen5SpirvTranslator.TryCompileProgram(request, out var shader, out var error), error);
+        using var harness = new ImageTestHarness(vulkan);
+        using var runner = new LayoutComputeRunner(harness, request, shader.Spirv);
+        var result = runner.CreateBuffer(ThreadCount * 32);
+        var registers = new uint[256];
+        registers[6] = ThreadCount * 32;
+        harness.Run(() => runner.Dispatch(registers,
+            new Dictionary<DescriptorBindingKind, GpuBuffer[]> { [DescriptorBindingKind.Buffers] = [result] }, 1));
+        var actual = runner.ReadBack(result, 0, ThreadCount * 32);
+        for (var lane = 0u; lane < ThreadCount; lane++)
+        {
+            var value = LaneBase + lane;
+            var mad = (ulong)Factor * value + Addend;
+            var row = actual.AsSpan((int)(lane * 32));
+            Assert.Equal((uint)mad, BinaryPrimitives.ReadUInt32LittleEndian(row));
+            Assert.Equal((uint)(mad >> 32), BinaryPrimitives.ReadUInt32LittleEndian(row[4..]));
+            Assert.Equal((uint)(((ulong)value * SignedFactor) >> 32), BinaryPrimitives.ReadUInt32LittleEndian(row[8..]));
+            Assert.Equal((uint)(((long)(int)value * unchecked((int)SignedFactor)) >> 32), BinaryPrimitives.ReadUInt32LittleEndian(row[12..]));
+            Assert.Equal((uint)(((ulong)Factor * SignedFactor) >> 32), BinaryPrimitives.ReadUInt32LittleEndian(row[16..]));
+        }
+    }
+}

tests/SharpEmu.Libs.Tests/Gpu/Vulkan/Wave64HalfMaskDeviceTests.cs

@@ -0,0 +1,269 @@
+// Copyright (C) 2026 SharpEmu Emulator Project
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+using System.Buffers.Binary;
+using SharpEmu.Libs.Gpu.Buffers;
+using SharpEmu.Libs.Tests.Gpu.Images;
+using SharpEmu.ShaderCompiler;
+using SharpEmu.ShaderCompiler.Resources;
+using SharpEmu.ShaderCompiler.Vulkan;
+using Xunit;
+using static SharpEmu.ShaderCompiler.Tests.Resources.ResourceTestProgram;
+
+namespace SharpEmu.Libs.Tests.Gpu.Vulkan;
+
+public sealed class Wave64HalfMaskDeviceTests(HeadlessVulkanFixture fixture) : IClassFixture<HeadlessVulkanFixture>
+{
+    private const uint Vcc = 106;
+    private const uint Exec = 126;
+    private static readonly Gen5Operand AllLanes = Gen5Operand.Source(193);
+
+    private readonly List<Gen5ShaderInstruction> _instructions = [];
+    private uint _pc;
+
+    private uint Add(Gen5ShaderInstruction instruction)
+    {
+        var pc = _pc;
+        _instructions.Add(instruction with { Pc = pc });
+        _pc += 8;
+        return pc;
+    }
+
+    private void Point(uint branchPc, uint targetPc)
+    {
+        var index = _instructions.FindIndex(instruction => instruction.Pc == branchPc);
+        var offset = (short)(((int)targetPc - (int)branchPc - 4) / 4);
+        _instructions[index] = _instructions[index] with { Words = [unchecked((uint)(ushort)offset)] };
+    }
+
+    private void Lane()
+    {
+        Add(Vop2(0, "VLshlrevB32", 2, Operand(3), Gen5Operand.Vector(1)));
+        Add(Vop2(0, "VAddI32", 2, Gen5Operand.Vector(0), Gen5Operand.Vector(2)));
+    }
+
+    private uint CompareTo(string opcode, uint destination, Gen5Operand left, Gen5Operand right) =>
+        Add(new Gen5ShaderInstruction(0, Gen5ShaderEncoding.Vop3, opcode, [0u, 0u], [left, right], [Gen5Operand.Scalar(destination)],
+            new Gen5Vop3Control(0, 0, 0, false, 0, destination)));
+
+    private uint[] Run(uint localDataShareDwords = 0)
+    {
+        Add(Vop2(0, "VLshlrevB32", 5, Operand(2), Gen5Operand.Vector(2)));
+        Add(BufferAccess(0, "BufferStoreDword", 4, vectorData: 4, offsetEnabled: true, vectorAddress: 5));
+        Add(EndProgram(0));
+        var (plan, resources, layout) = Prepare(Program([.. _instructions]));
+        var request = new ShaderCompileRequest(plan, resources, layout)
+        {
+            LocalSizeX = 8, LocalSizeY = 8, ThreadCountX = 8, ThreadCountY = 8, WaveSize = 64,
+            LocalDataShareDwords = localDataShareDwords,
+        };
+        Assert.True(Gen5SpirvTranslator.TryCompileProgram(request, out var shader, out var error), error);
+        Assert.NotNull(fixture.Vulkan);
+        using var harness = new ImageTestHarness(fixture.Vulkan);
+        using var runner = new LayoutComputeRunner(harness, request, shader.Spirv);
+        var output = runner.CreateBuffer(256);
+        var registers = new uint[256];
+        registers[6] = 256;
+        harness.Run(() => runner.Dispatch(registers,
+            new Dictionary<DescriptorBindingKind, GpuBuffer[]> { [DescriptorBindingKind.Buffers] = [output] }, 1));
+        var bytes = runner.ReadBack(output, 0, 256);
+        harness.AssertNoValidationMessages();
+        var lanes = new uint[64];
+        for (var lane = 0; lane < lanes.Length; lane++)
+        {
+            lanes[lane] = BinaryPrimitives.ReadUInt32LittleEndian(bytes.AsSpan(lane * sizeof(uint)));
+        }
+
+        return lanes;
+    }
+
+    private bool Ready() => GatePrerequisites.Ready(fixture.Vulkan, shaderInt64: true);
+
+    [Fact]
+    public void MasksAcrossTheHalfBoundary_StayExactPerLaneAndWhenCounted()
+    {
+        if (!Ready()) return;
+        Lane();
+        CompareTo("VCmpLtU32", 10, Gen5Operand.Vector(2), Operand(40));
+        Add(Vop2(0, "VAndB32", 6, Operand(1), Gen5Operand.Vector(2)));
+        Add(Vopc(0, "VCmpEqU32", Operand(1), 6));
+        Add(Sop2(0, "SAndB64", 12, Gen5Operand.Scalar(10), Gen5Operand.Scalar(Vcc)));
+        Add(Vop3(0, "VCndmaskB32", 4, Operand(0), Operand(1000), Gen5Operand.Scalar(12)));
+        Add(Sop1(0, "SBcnt1I32B64", 14, Gen5Operand.Scalar(12)));
+        Add(Vop2(0, "VAddU32", 4, Gen5Operand.Scalar(14), Gen5Operand.Vector(4)));
+
+        var lanes = Run();
+        for (uint lane = 0; lane < 64; lane++)
+        {
+            Assert.Equal((lane & 1) != 0 && lane < 40 ? 1020u : 20u, lanes[lane]);
+        }
+    }
+
+    [Theory]
+    [InlineData(48u, 77u)]
+    [InlineData(8u, 77u)]
+    [InlineData(64u, 5u)]
+    public void AnExecBranch_IsDecidedForTheWholeWave(uint firstActiveLane, uint expected)
+    {
+        if (!Ready()) return;
+        Lane();
+        Add(MoveScalar(0, 20, 5));
+        Add(Vopc(0, "VCmpxLeU32", Operand(firstActiveLane), 2));
+        var skip = Add(Branch(0, "SCbranchExecz", 0));
+        Add(MoveScalar(0, 20, 77));
+        var join = Add(Sop1(0, "SMovB64", Exec, AllLanes));
+        Point(skip, join);
+        Add(MoveVectorFromScalar(0, 4, 20));
+
+        Assert.All(Run(), value => Assert.Equal(expected, value));
+    }
+
+    [Theory]
+    [InlineData(50u, 9u)]
+    [InlineData(10u, 9u)]
+    [InlineData(64u, 3u)]
+    public void AMaskWhoseSccIsRead_CountsBothHalves(uint threshold, uint expected)
+    {
+        if (!Ready()) return;
+        Lane();
+        Add(MoveScalar(0, 20, 3));
+        CompareTo("VCmpGeU32", 10, Gen5Operand.Vector(2), Operand(threshold));
+        Add(Sop2(0, "SAndB64", 12, Gen5Operand.Scalar(10), Gen5Operand.Scalar(Exec)));
+        var skip = Add(Branch(0, "SCbranchScc0", 0));
+        Add(MoveScalar(0, 20, 9));
+        var join = Add(MoveVectorFromScalar(0, 4, 20));
+        Point(skip, join);
+
+        Assert.All(Run(), value => Assert.Equal(expected, value));
+    }
+
+    [Theory]
+    [InlineData(0u)]
+    [InlineData(64u)]
+    public void SharedMemoryWrittenByOneHalf_IsReadByTheOther(uint localDataShareDwords)
+    {
+        if (!Ready()) return;
+        Lane();
+        Add(Vop2(0, "VLshlrevB32", 3, Operand(1), Gen5Operand.Vector(2)));
+        Add(Vop2(0, "VAddU32", 3, Gen5Operand.Vector(2), Gen5Operand.Vector(3)));
+        Add(Vop2(0, "VLshlrevB32", 6, Operand(2), Gen5Operand.Vector(2)));
+        Add(Vopc(0, "VCmpxGtU32", Operand(32), 2));
+        Add(BufferAccess(0, "BufferLoadDword", 4, vectorData: 8, offsetEnabled: true, vectorAddress: 6));
+        Add(Vop2(0, "VAddU32", 3, Gen5Operand.Vector(8), Gen5Operand.Vector(3)));
+        Add(DataShare(0, "DsWriteB32", false, [Gen5Operand.Vector(6), Gen5Operand.Vector(3)], []));
+        Add(Sop1(0, "SMovB64", Exec, AllLanes));
+        Add(Vop2(0, "VAndB32", 7, Operand(31), Gen5Operand.Vector(2)));
+        Add(Vop2(0, "VLshlrevB32", 7, Operand(2), Gen5Operand.Vector(7)));
+        Add(DataShare(0, "DsReadB32", false, [Gen5Operand.Vector(7)], [4]));
+
+        var lanes = Run(localDataShareDwords);
+        for (uint lane = 0; lane < 64; lane++)
+        {
+            Assert.Equal((lane & 31) * 3, lanes[lane]);
+        }
+    }
+
+    [Fact]
+    public void AnExecLoop_RunsEveryLaneItsOwnCount()
+    {
+        if (!Ready()) return;
+        Lane();
+        Add(Vop2(0, "VAndB32", 3, Operand(7), Gen5Operand.Vector(2)));
+        Add(Vop2(0, "VAddU32", 3, Operand(1), Gen5Operand.Vector(3)));
+        Add(MoveVector(0, 4, 0));
+        Add(Sop1(0, "SMovB64", 20, Gen5Operand.Scalar(Exec)));
+        var header = Add(Vopc(0, "VCmpxGtU32", Gen5Operand.Vector(3), 4));
+        var exit = Add(Branch(0, "SCbranchExecz", 0));
+        Add(Vop2(0, "VAddU32", 4, Operand(1), Gen5Operand.Vector(4)));
+        var back = Add(Branch(0, "SBranch", 0));
+        var done = Add(Sop1(0, "SMovB64", Exec, Gen5Operand.Scalar(20)));
+        Point(exit, done);
+        Point(back, header);
+
+        var lanes = Run();
+        for (uint lane = 0; lane < 64; lane++)
+        {
+            Assert.Equal((lane & 7) + 1, lanes[lane]);
+        }
+    }
+
+    [Theory]
+    [InlineData(40u, 140u)]
+    [InlineData(3u, 103u)]
+    [InlineData(64u, 100u)]
+    public void ReadFirstLane_TakesTheFirstActiveLaneOfTheWholeWave(uint firstActiveLane, uint expected)
+    {
+        if (!Ready()) return;
+        Lane();
+        Add(Vop2(0, "VAddU32", 3, Operand(100), Gen5Operand.Vector(2)));
+        Add(Vopc(0, "VCmpxLeU32", Operand(firstActiveLane), 2));
+        Add(ReadFirstLane(0, 20, 3));
+        Add(Sop1(0, "SMovB64", Exec, AllLanes));
+        Add(MoveVectorFromScalar(0, 4, 20));
+
+        Assert.All(Run(), value => Assert.Equal(expected, value));
+    }
+
+    [Fact]
+    public void ADemonsSoulsIndexRemap_CompilesOnTheDevice()
+    {
+        if (!Ready()) return;
+        uint[] words =
+        [
+            0xBEFC03FFu, 0xA65CD91Eu, 0xBF960000u, 0xBFA00003u, 0xD7460002u, 0x04010C0Cu, 0xF4201A84u, 0xFA000000u,
+            0xBF8CC07Fu, 0x7DA8046Au, 0xBF88002Au, 0xF4200304u, 0xFA000004u, 0xBF8CC07Fu, 0x7E000C0Cu, 0xBF070C80u,
+            0x8588807Eu, 0x7E005700u, 0x100000FFu, 0x4F800000u, 0x7E060F00u, 0xD5766A00u, 0x0202060Cu, 0x7D8A0280u,
+            0x4C020080u, 0x02000101u, 0xD56A0001u, 0x00020700u, 0x4C000303u, 0x4A020303u, 0x02000101u, 0xD56A0000u,
+            0x00020500u, 0xD5690001u, 0x0002000Cu, 0x4C060302u, 0x7D8C02F9u, 0x06068A02u, 0x7D86060Cu, 0x87EA6A0Au,
+            0x50000080u, 0xD5286A00u, 0x002A00C1u, 0xD5010000u, 0x002200C1u, 0xD5690000u, 0x0002000Cu, 0x4C000102u,
+            0xE0002000u, 0x80000000u, 0xBF8C3F70u, 0xE0102000u, 0x80010002u, 0xBF810000u,
+        ];
+        var bytes = new byte[words.Length * sizeof(uint)];
+        for (var index = 0; index < words.Length; index++)
+            BinaryPrimitives.WriteUInt32LittleEndian(bytes.AsSpan(index * sizeof(uint)), words[index]);
+        Assert.True(Gen5ShaderTranslator.TryDecodeProgram(new SharpEmu.HLE.CpuContext(new WordMemory(bytes), SharpEmu.HLE.Generation.Gen5),
+            0x1000, out var program, out var decodeError), decodeError);
+        var (plan, resources, layout) = Prepare(program, userDataCount: 12);
+        var request = new ShaderCompileRequest(plan, resources, layout)
+        {
+            LocalSizeX = 64, WaveSize = 64, EnableExecGuardElision = false,
+            ComputeSystemRegisters = new Gen5ComputeSystemRegisters(12, null, null, null),
+        };
+        Assert.True(Gen5SpirvTranslator.TryCompileProgram(request, out var shader, out var error), error);
+        Assert.NotNull(fixture.Vulkan);
+        using var harness = new ImageTestHarness(fixture.Vulkan);
+        using var runner = new LayoutComputeRunner(harness, request, shader.Spirv);
+        harness.AssertNoValidationMessages();
+    }
+
+    private sealed class WordMemory(byte[] bytes) : SharpEmu.HLE.ICpuMemory
+    {
+        public bool TryRead(ulong address, Span<byte> destination)
+        {
+            if (address < 0x1000 || address - 0x1000 > (ulong)(bytes.Length - destination.Length)) return false;
+            bytes.AsSpan((int)(address - 0x1000), destination.Length).CopyTo(destination);
+            return true;
+        }
+
+        public bool TryWrite(ulong address, ReadOnlySpan<byte> source) => false;
+    }
+
+    [Fact]
+    public void ASavedExec_RestoresAfterAHalfMaskedBody()
+    {
+        if (!Ready()) return;
+        Lane();
+        Add(MoveVector(0, 4, 1));
+        CompareTo("VCmpGeU32", 10, Gen5Operand.Vector(2), Operand(28));
+        Add(Sop1(0, "SAndSaveexecB64", 12, Gen5Operand.Scalar(10)));
+        Add(MoveVector(0, 4, 2));
+        Add(Sop2(0, "SOrB64", Exec, Gen5Operand.Scalar(Exec), Gen5Operand.Scalar(12)));
+        Add(Vop2(0, "VAddU32", 4, Operand(10), Gen5Operand.Vector(4)));
+
+        var lanes = Run();
+        for (uint lane = 0; lane < 64; lane++)
+        {
+            Assert.Equal(lane >= 28 ? 12u : 11u, lanes[lane]);
+        }
+    }
+}

tests/SharpEmu.Libs.Tests/VideoOut/RenderHostDeviceTests.cs

@@ -524,6 +524,46 @@ public sealed unsafe partial class RenderHostDeviceTests : IClassFixture<Headles
         harness.Shutdown();
     }
 
+    [Fact]
+    public void PrepareBindings_MergesDeviceAddressRangesBeforeBuffersAreBound()
+    {
+        if (!Ready()) return;
+        using var presenter = new PresenterUnderTest(_vulkan!);
+        using var fatal = new FatalScope();
+        presenter.SetField("_minStorageBufferOffsetAlignment", 256UL);
+        presenter.LoadRenderingCommands();
+        var harness = presenter.Harness;
+        var address = harness.MapBacked(0x10000, ReadWrite);
+        var written = address + 0x100;
+        var resource = new BufferResource { Read = true, Written = true, MaxByteExtent = 32 };
+        var program = FixedProgramProvider.EmptyProgram(ShaderStageKind.Compute, 3,
+            new ShaderResourceInfo { Buffers = [resource] });
+        var snapshot = new ResourceSnapshot
+        {
+            Buffers = [[(uint)written, (uint)(written >> 32), 32, 0]],
+            DeviceAddressRanges = [new DeviceAddressRange(0, address, 2 * GuestBufferCache.CachingPageSize, true, false)],
+        };
+        GuestGpuMemoryHook.Attach(harness.Gpu);
+        try
+        {
+            presenter.Run(() =>
+            {
+                using var preparation = presenter.RenderHost.BeginPreparation();
+                var prepared = presenter.RenderHost.PrepareBindings(new ShaderStageResources(program, snapshot));
+                presenter.RenderHost.PrepareDeviceAddresses();
+                var owner = harness.Cache.GetBuffer(harness.Cache.FindBuffer(written, 32));
+                presenter.RenderHost.BindResources(prepared);
+                Assert.Same(owner, harness.Cache.GetBuffer(harness.Cache.FindBuffer(written, 32)));
+                Assert.Equal(1, harness.Cache.BufferCount);
+            });
+        }
+        finally
+        {
+            GuestGpuMemoryHook.Attach(null);
+        }
+        harness.Shutdown();
+    }
+
     [Fact]
     public void PrepareBindings_PrivateBufferUploadsCurrentBytesAndStillRejectsGpuWrites()
     {

tests/SharpEmu.ShaderCompiler.Tests/Gen5MultiplyHighTests.cs

@@ -0,0 +1,68 @@
+// Copyright (C) 2026 SharpEmu Emulator Project
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+using System.Runtime.InteropServices;
+using SharpEmu.ShaderCompiler.Tests.Resources;
+using SharpEmu.ShaderCompiler.Vulkan;
+using Xunit;
+using static SharpEmu.ShaderCompiler.Tests.Resources.ResourceTestProgram;
+
+namespace SharpEmu.ShaderCompiler.Tests;
+
+public sealed class Gen5MultiplyHighTests
+{
+    [Theory]
+    [InlineData("VMulHiU32", false)]
+    [InlineData("VMulHiU32U24", false)]
+    [InlineData("VMulHiI32", true)]
+    [InlineData("VMadU64U32", false)]
+    public void VectorProducts_UseExtendedMultiplies(string opcode, bool signed)
+    {
+        var program = Program(
+            Vop3(0, opcode, 2, Gen5Operand.Vector(4), Gen5Operand.Scalar(8), Gen5Operand.Scalar(10)),
+            EndProgram(8));
+        AssertExtendedMultiply(program, signed);
+    }
+
+    [Theory]
+    [InlineData("SMulHiU32", false)]
+    [InlineData("SMulHiI32", true)]
+    public void ScalarProducts_UseExtendedMultiplies(string opcode, bool signed)
+    {
+        var program = Program(
+            Sop2(0, opcode, 12, Gen5Operand.Scalar(8), Gen5Operand.Scalar(9)),
+            EndProgram(4));
+        AssertExtendedMultiply(program, signed);
+    }
+
+    private static void AssertExtendedMultiply(Gen5ShaderProgram program, bool signed)
+    {
+        var (plan, resources, layout) = Prepare(program);
+        var request = new ShaderCompileRequest(plan, resources, layout) { LocalSizeX = 1, ThreadCountX = 1 };
+        Assert.True(Gen5SpirvTranslator.TryCompileProgram(request, out var shader, out var error), error);
+        var inspector = new SpirvModuleInspector(shader.Spirv);
+        Assert.Contains((ushort)(signed ? SpirvOp.SMulExtended : SpirvOp.UMulExtended), inspector.Opcodes);
+        Assert.Empty(WideMultiplies(shader.Spirv));
+    }
+
+    private static List<uint> WideMultiplies(byte[] spirv)
+    {
+        var words = MemoryMarshal.Cast<byte, uint>(spirv);
+        var wideTypes = new HashSet<uint>();
+        var multiplies = new List<uint>();
+        for (var index = 5; index < words.Length; index += (int)(words[index] >> 16))
+        {
+            var opcode = (ushort)(words[index] & 0xFFFF);
+            if (opcode == (ushort)SpirvOp.TypeInt && words[index + 2] == 64)
+            {
+                wideTypes.Add(words[index + 1]);
+            }
+            else if (opcode == (ushort)SpirvOp.IMul && wideTypes.Contains(words[index + 1]))
+            {
+                multiplies.Add(words[index + 2]);
+            }
+        }
+
+        return multiplies;
+    }
+}

tests/SharpEmu.ShaderCompiler.Tests/Gen5SharedMemoryBarrierTests.cs

@@ -11,12 +11,14 @@ namespace SharpEmu.ShaderCompiler.Tests;
 public sealed class Gen5SharedMemoryBarrierTests
 {
     [Theory]
-    [InlineData(64u, 64u, false, 2)]
-    [InlineData(64u, 64u, true, 2)]
-    [InlineData(32u, 64u, false, 0)]
-    [InlineData(64u, 128u, false, 0)]
+    [InlineData(64u, 64u, false, false, 2)]
+    [InlineData(64u, 64u, true, false, 2)]
+    [InlineData(32u, 64u, false, false, 0)]
+    [InlineData(64u, 128u, false, false, 0)]
+    [InlineData(64u, 64u, false, true, 2)]
+    [InlineData(32u, 64u, false, true, 0)]
     public void SharedMemoryPhases_SynchronizeOnlySingleWaveGroups(
-        uint waveSize, uint threadCount, bool explicitBarrier, int expectedBarriers)
+        uint waveSize, uint threadCount, bool explicitBarrier, bool splitBlocks, int expectedBarriers)
     {
         var instructions = new List<Gen5ShaderInstruction>
         {
@@ -25,6 +27,8 @@ public sealed class Gen5SharedMemoryBarrierTests
         };
         if (explicitBarrier)
             instructions.Add(new(8, Gen5ShaderEncoding.Sopp, "SBarrier", [0u], [], [], null));
+        if (splitBlocks)
+            instructions.Add(new(8, Gen5ShaderEncoding.Sopp, "SBranch", [0u], [], [], null));
         instructions.Add(new(12, Gen5ShaderEncoding.Ds, "DsReadB32", [0u, 0u],
             [Gen5Operand.Vector(0)], [Gen5Operand.Vector(2)], new Gen5DataShareControl(0, 0, false)));
         instructions.Add(new(20, Gen5ShaderEncoding.Sopp, "SEndpgm", [0u], [], [], null));

tests/SharpEmu.ShaderCompiler.Tests/Gen5Wave64HalfMaskAnalysisTests.cs

@@ -0,0 +1,221 @@
+// Copyright (C) 2026 SharpEmu Emulator Project
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+using SharpEmu.ShaderCompiler;
+using SharpEmu.ShaderCompiler.Ir;
+using Xunit;
+
+namespace SharpEmu.ShaderCompiler.Tests;
+
+public sealed class Gen5Wave64HalfMaskAnalysisTests
+{
+    private const uint Vcc = 106;
+    private const uint Exec = 126;
+
+    private readonly List<Gen5ShaderInstruction> _program = [];
+    private uint _pc;
+
+    private uint Add(string opcode, Gen5ShaderEncoding encoding, Gen5Operand[] destinations, Gen5Operand[] sources,
+        Gen5InstructionControl? control = null, uint word = 0)
+    {
+        var pc = _pc;
+        _program.Add(new Gen5ShaderInstruction(pc, encoding, opcode, [word], sources, destinations, control));
+        _pc += 4;
+        return pc;
+    }
+
+    private uint CompareToVcc() =>
+        Add("VCmpLtU32", Gen5ShaderEncoding.Vopc, [], [Gen5Operand.Vector(0), Gen5Operand.Vector(1)]);
+
+    private uint CompareTo(uint destination) =>
+        Add("VCmpLtU32", Gen5ShaderEncoding.Vop3, [Gen5Operand.Scalar(destination)], [Gen5Operand.Vector(0), Gen5Operand.Vector(1)],
+            new Gen5Vop3Control(0, 0, 0, false, 0, destination));
+
+    private uint CompareToExec() =>
+        Add("VCmpxLtU32", Gen5ShaderEncoding.Vopc, [], [Gen5Operand.Vector(0), Gen5Operand.Vector(1)]);
+
+    private uint Select(uint mask) =>
+        Add("VCndmaskB32", Gen5ShaderEncoding.Vop3, [Gen5Operand.Vector(2)], [Gen5Operand.Vector(3), Gen5Operand.Vector(4), Gen5Operand.Scalar(mask)],
+            new Gen5Vop3Control(0, 0, 0, false, 0, null));
+
+    private uint Scalar1(string opcode, uint destination, params Gen5Operand[] sources) =>
+        Add(opcode, Gen5ShaderEncoding.Sop1, [Gen5Operand.Scalar(destination)], sources);
+
+    private uint Scalar2(string opcode, uint destination, params Gen5Operand[] sources) =>
+        Add(opcode, Gen5ShaderEncoding.Sop2, [Gen5Operand.Scalar(destination)], sources);
+
+    private uint Branch(string opcode) => Add(opcode, Gen5ShaderEncoding.Sopp, [], [], word: 0xBF840000u);
+
+    private uint DataShare(string opcode) =>
+        Add(opcode, Gen5ShaderEncoding.Ds, opcode.StartsWith("DsRead", StringComparison.Ordinal) ? [Gen5Operand.Vector(5)] : [],
+            [Gen5Operand.Vector(0), Gen5Operand.Vector(1)], new Gen5DataShareControl(0, 0, false));
+
+    private uint End() => Add("SEndpgm", Gen5ShaderEncoding.Sopp, [], [], word: 0xBF810000u);
+
+    private void PointBranch(uint branchPc, uint targetPc)
+    {
+        var index = _program.FindIndex(instruction => instruction.Pc == branchPc);
+        var original = _program[index];
+        var offset = (ushort)(short)((int)(targetPc - branchPc - 4) / 4);
+        _program[index] = original with { Words = [(original.Words[0] & 0xFFFF0000u) | offset] };
+    }
+
+    private Gen5Wave64HalfMaskPlan Analyze() =>
+        Gen5Wave64HalfMaskAnalysis.Analyze(new Gen5ShaderProgram(0x1000, _program), new HashSet<uint>())
+            ?? throw new InvalidOperationException("analysis gave up");
+
+    [Fact]
+    public void ACompareReadPerLane_NeedsNoWholeWaveMask()
+    {
+        var compare = CompareTo(20);
+        var select = Select(20);
+        End();
+
+        var plan = Analyze();
+        Assert.Equal(20u, plan.HalfMaskWrites[compare]);
+        Assert.False(plan.ExactPairsBefore.ContainsKey(select));
+    }
+
+    [Fact]
+    public void ABranchOnVcc_CombinesVccFirst()
+    {
+        CompareToVcc();
+        var branch = Branch("SCbranchVccz");
+        End();
+        PointBranch(branch, _pc - 4);
+
+        Assert.Equal(new[] { Vcc }, Analyze().ExactPairsBefore[branch]);
+    }
+
+    [Fact]
+    public void ABranchOnExec_CombinesExecFirst()
+    {
+        CompareToExec();
+        var branch = Branch("SCbranchExecz");
+        End();
+        PointBranch(branch, _pc - 4);
+
+        Assert.Equal(new[] { Exec }, Analyze().ExactPairsBefore[branch]);
+    }
+
+    [Fact]
+    public void ABitwiseMaskOperationWithADeadScc_StaysHalf()
+    {
+        CompareTo(20);
+        CompareToVcc();
+        var and = Scalar2("SAndB64", 22, Gen5Operand.Scalar(20), Gen5Operand.Scalar(Vcc));
+        var select = Select(22);
+        End();
+
+        var plan = Analyze();
+        Assert.False(plan.ExactPairsBefore.ContainsKey(and));
+        Assert.False(plan.ExactPairsBefore.ContainsKey(select));
+    }
+
+    [Fact]
+    public void ABitwiseMaskOperationWhoseSccIsRead_CombinesItsInputs()
+    {
+        CompareTo(20);
+        CompareToVcc();
+        var and = Scalar2("SAndB64", 22, Gen5Operand.Scalar(20), Gen5Operand.Scalar(Vcc));
+        var branch = Branch("SCbranchScc0");
+        End();
+        PointBranch(branch, _pc - 4);
+
+        var plan = Analyze();
+        Assert.Equal(new[] { 20u, Vcc }, plan.ExactPairsBefore[and]);
+        Assert.False(plan.ExactPairsBefore.ContainsKey(branch));
+    }
+
+    [Fact]
+    public void AScalarReadOfAMaskHalf_CombinesThePair()
+    {
+        CompareTo(20);
+        var move = Scalar1("SMovB32", 30, Gen5Operand.Scalar(21));
+        End();
+
+        Assert.Equal(new[] { 20u }, Analyze().ExactPairsBefore[move]);
+    }
+
+    [Fact]
+    public void ACombinedMask_IsNotCombinedAgain()
+    {
+        CompareTo(20);
+        var first = Scalar1("SMovB32", 30, Gen5Operand.Scalar(20));
+        var second = Scalar1("SMovB32", 31, Gen5Operand.Scalar(21));
+        End();
+
+        var plan = Analyze();
+        Assert.Equal(new[] { 20u }, plan.ExactPairsBefore[first]);
+        Assert.False(plan.ExactPairsBefore.ContainsKey(second));
+    }
+
+    [Fact]
+    public void ASaveexecCopiesAHalfExec_SoTheRestoreStaysHalf()
+    {
+        CompareTo(20);
+        var save = Scalar1("SAndSaveexecB64", 24, Gen5Operand.Scalar(20));
+        Select(Exec);
+        var restore = Scalar2("SOrB64", Exec, Gen5Operand.Scalar(Exec), Gen5Operand.Scalar(24));
+        var read = Scalar1("SMovB32", 30, Gen5Operand.Scalar(24));
+        End();
+
+        var plan = Analyze();
+        Assert.False(plan.ExactPairsBefore.ContainsKey(save));
+        Assert.False(plan.ExactPairsBefore.ContainsKey(restore));
+        Assert.Equal(new[] { 24u }, plan.ExactPairsBefore[read]);
+    }
+
+    [Fact]
+    public void ALoopExitOnExec_CombinesExecEveryIteration()
+    {
+        var header = CompareToExec();
+        var exit = Branch("SCbranchExecz");
+        Select(Exec);
+        var back = Branch("SBranch");
+        var end = End();
+        PointBranch(exit, end);
+        PointBranch(back, header);
+
+        var plan = Analyze();
+        Assert.Equal(new[] { Exec }, plan.ExactPairsBefore[exit]);
+        Assert.Contains(header, plan.HalfMaskWrites.Keys);
+    }
+
+    [Fact]
+    public void ASharedReadAfterAWrite_WaitsForTheOtherHalf()
+    {
+        var write = DataShare("DsWriteB32");
+        var read = DataShare("DsReadB32");
+        var again = DataShare("DsReadB32");
+        End();
+
+        var plan = Analyze();
+        Assert.DoesNotContain(write, plan.SharedMemoryBarriersBefore);
+        Assert.Contains(read, plan.SharedMemoryBarriersBefore);
+        Assert.DoesNotContain(again, plan.SharedMemoryBarriersBefore);
+    }
+
+    [Fact]
+    public void ACombineBarrier_AlsoOrdersSharedMemory()
+    {
+        DataShare("DsWriteB32");
+        CompareTo(20);
+        var move = Scalar1("SMovB32", 30, Gen5Operand.Scalar(20));
+        var read = DataShare("DsReadB32");
+        End();
+
+        var plan = Analyze();
+        Assert.True(plan.ExactPairsBefore.ContainsKey(move));
+        Assert.DoesNotContain(read, plan.SharedMemoryBarriersBefore);
+    }
+
+    [Fact]
+    public void ADynamicJump_GivesUp()
+    {
+        Scalar1("SSetpcB64", 0, Gen5Operand.Scalar(4));
+        End();
+
+        Assert.Null(Gen5Wave64HalfMaskAnalysis.Analyze(new Gen5ShaderProgram(0x1000, _program), new HashSet<uint>()));
+    }
+}

tests/SharpEmu.ShaderCompiler.Tests/Resources/DirectImageTableTests.cs

@@ -30,6 +30,41 @@ public sealed class DirectImageTableTests
             EndProgram(72));
     }
 
+    internal static Gen5ShaderProgram CreateWaveIndexedReadLaneProgram(bool selfAddressed, bool restoreExec = true)
+    {
+        var instructions = new List<Gen5ShaderInstruction>
+        {
+            ScalarLoad(0, 0, 16, immediateOffset: 0x80),
+            Sop1(8, "SFF1I32B32", 18, Gen5Operand.Scalar(16)),
+            Sop2(12, "SMulI32", 19, Gen5Operand.Scalar(18), new Gen5Operand(Gen5OperandKind.LiteralConstant, 0x90)),
+            MoveVectorFromScalar(16, 34, 18),
+            Vop2(20, "VLshlrevB32", 15, Operand(4), Gen5Operand.Vector(34)),
+            Vop3(24, "VLshlAddU32", selfAddressed ? 15u : 16u, Gen5Operand.Vector(15), Operand(3), Gen5Operand.Vector(15)),
+            Sop2(32, "SLshlB32", 20, Operand(1), Gen5Operand.Scalar(18)),
+            Sop2(36, "SXorB32", 16, Gen5Operand.Scalar(20), Gen5Operand.Scalar(16)),
+            Vop2(40, "VAddI32", 15, new Gen5Operand(Gen5OperandKind.LiteralConstant, 0x40), Gen5Operand.Vector(selfAddressed ? 15u : 16u)),
+            GlobalMemory(44, "GlobalLoadDword", 0, 15, 22, 0),
+            Sop1(52, "SMovB64", 12, Gen5Operand.Scalar(126)),
+            Sop1(56, "SFF1I32B64", 24, Gen5Operand.Scalar(12)),
+            new(60, Gen5ShaderEncoding.Vop3, "VReadlaneB32", [0u, 0u],
+                [Gen5Operand.Vector(22), Gen5Operand.Scalar(24), Gen5Operand.Scalar(24)], [Gen5Operand.Scalar(106)], null),
+            new(68, Gen5ShaderEncoding.Vop3, "VCmpEqU32", [0u, 0u],
+                [Gen5Operand.Scalar(106), Gen5Operand.Vector(22)], [Gen5Operand.Scalar(14)], new Gen5Vop3Control(0, 0, 0, false, 0, 14)),
+            Sop1(76, "SAndSaveexecB64", 28, Gen5Operand.Scalar(14)),
+            Branch(80, "SCbranchExecz", 8),
+            Sop2(84, "SLshlB32", 106, Gen5Operand.Scalar(106), Operand(5)),
+            Sop2(88, "SAddI32", 107, Gen5Operand.Scalar(106), Operand(16)),
+            ScalarLoad(92, 0, 4, 4, immediateOffset: 0x100, dynamicOffsetRegister: 106),
+            ScalarLoad(100, 0, 8, 4, immediateOffset: 0x100, dynamicOffsetRegister: 107),
+            Image(108, "ImageLoad", 4, dmask: 1, vectorAddress: 1),
+            Sop2(116, "SAndn2B64", 12, Gen5Operand.Scalar(12), Gen5Operand.Scalar(14)),
+            restoreExec ? Sop1(120, "SMovB64", 126, Gen5Operand.Scalar(28)) : Nop(120),
+            Branch(124, "SCbranchScc1", -18),
+            EndProgram(128),
+        };
+        return Program([.. instructions]);
+    }
+
     public static Gen5ShaderProgram CreateProgram(uint mask = 1, bool split = true, bool bitScan = true)
     {
         var instructions = new List<Gen5ShaderInstruction>
@@ -107,41 +142,72 @@ public sealed class DirectImageTableTests
         Assert.Equal(0u, selector.KeyBound);
         Assert.Equal(new WaveIndexedImageSelector(0x80, 0x40, 0x90), selector.WaveIndexed);
 
-        bool Read(ulong address, out uint word)
-        {
-            word = 0;
-            if (address == 0x1000 + 0x80)
-            {
-                word = (1u << 1) | (1u << 4);
-                return true;
-            }
+        var snapshot = new ResourceSnapshot();
+        var specialization = new ResourceSpecialization();
+        Assert.True(ResourceMaterializer.Materialize(plan, Inputs([0x1000, 0], readCleanMemory: ReadWaveIndexedMemory), ref snapshot, ref specialization));
+        Assert.Equal(2, snapshot.Images.Length);
+    }
 
-            if (address == 0x1000 + 0x40 + 0x90 || address == 0x1000 + 0x40 + 4 * 0x90)
-            {
-                word = address == 0x1000 + 0x40 + 0x90 ? 2u : 5u;
-                return true;
-            }
-
-            if (address < 0x1000 + 0x100 || address >= 0x1000 + 0x100 + 6 * 32)
-                return false;
-            var relative = address - 0x1000 - 0x100;
-            var record = relative / 32;
-            if (record is not (2 or 5))
-                return false;
-            word = (relative % 32 / 4) switch
-            {
-                0 => (record & 1) == 0 ? 0x2000u : 0x1000u,
-                1 => 20u << 20,
-                3 => 0xFACu | (9u << 28),
-                _ => 0,
-            };
-            return true;
-        }
+    [Theory]
+    [InlineData(false)]
+    [InlineData(true)]
+    public void ReadLaneWaterfallSelectsWaveIndexedDescriptors(bool selfAddressed)
+    {
+        var program = CreateWaveIndexedReadLaneProgram(selfAddressed);
+        var plan = ShaderResourcePlan.Extract(program, ShaderStage.Compute, Hash, 0, 2);
+        var selector = plan.DescriptorSources[(int)plan.Info.Images[0].Source].IndirectImage!;
+        Assert.True(selector.Dense, "dense");
+        Assert.Equal(0x100u, selector.TableOffset);
+        Assert.Equal(new WaveIndexedImageSelector(0x80, 0x40, 0x90), selector.WaveIndexed);
+        Assert.True(Assert.Single(plan.IndirectImages).KeyIsAddressOffset, "address-offset key");
 
         var snapshot = new ResourceSnapshot();
         var specialization = new ResourceSpecialization();
-        Assert.True(ResourceMaterializer.Materialize(plan, Inputs([0x1000, 0], readCleanMemory: Read), ref snapshot, ref specialization));
+        Assert.True(ResourceMaterializer.Materialize(plan, Inputs([0x1000, 0], readCleanMemory: ReadWaveIndexedMemory), ref snapshot, ref specialization, out var failure), $"materialize {failure}");
         Assert.Equal(2, snapshot.Images.Length);
+        var resources = ResourceMaterializer.ApplyTo(plan, specialization);
+        var layout = BindingLayout.Allocate(resources.Info, BindingLayout.CollectUserDataRegisters(program, 0, 2),
+            false, ShaderCompileRequest.RequiresFlattenedTable(plan, resources), false);
+        var request = new ShaderCompileRequest(plan, resources, layout) { LocalSizeX = 64, ThreadCountX = 64 };
+        Assert.True(Gen5SpirvTranslator.TryCompileProgram(request, out _, out var error), error);
+    }
+
+    [Fact]
+    public void ReadLaneWithoutRestoredExecutionIsNotWaveIndexed()
+    {
+        Assert.Throws<ResourcePlanException>(() =>
+            ShaderResourcePlan.Extract(CreateWaveIndexedReadLaneProgram(selfAddressed: true, restoreExec: false), ShaderStage.Compute, Hash, 0, 2));
+    }
+
+    private static bool ReadWaveIndexedMemory(ulong address, out uint word)
+    {
+        word = 0;
+        if (address == 0x1000 + 0x80)
+        {
+            word = (1u << 1) | (1u << 4);
+            return true;
+        }
+
+        if (address == 0x1000 + 0x40 + 0x90 || address == 0x1000 + 0x40 + 4 * 0x90)
+        {
+            word = address == 0x1000 + 0x40 + 0x90 ? 2u : 5u;
+            return true;
+        }
+
+        if (address < 0x1000 + 0x100 || address >= 0x1000 + 0x100 + 6 * 32)
+            return false;
+        var relative = address - 0x1000 - 0x100;
+        var record = relative / 32;
+        if (record is not (2 or 5))
+            return false;
+        word = (relative % 32 / 4) switch
+        {
+            0 => (record & 1) == 0 ? 0x2000u : 0x1000u,
+            1 => 20u << 20,
+            3 => 0xFACu | (9u << 28),
+            _ => 0,
+        };
+        return true;
     }
 
     public static (ResourceSnapshot Snapshot, ShaderCompileRequest Request) PrepareMixedDimensions(uint mask, bool arrayFirst)