[GPU] Keep sampled depth attachments in one rendering scope (#1037)

Draws that sample the depth attachment need its read-only layout; every
other draw got the writable layout, even one that only tests depth. A run
of depth-sampling particles between test-only draws flipped the layout on
every draw, and each flip ended the rendering scope. Image state tracking
also recorded a barrier whenever the access changed, even from one read
to another in the same layout.

A draw that neither samples nor writes the attachment now keeps the
read-only layout it is already in, and reads after reads in one layout
record no barrier: the tracked state accumulates every reader, so the next
writer still waits for all of them. Depth-transition scope breaks in
Silent Hill: The Short Message's menu drop from ~1250 to ~820 per 5 s.

(cherry picked from commit 5563417d72fadefb115b300291b1e019b3171466)

src/SharpEmu.Libs/Gpu/Images/CachedImage.Transfers.cs

@@ -14,6 +14,14 @@ public sealed unsafe partial class CachedImage
 {
     private const AccessFlags WriteAccess = AccessFlags.TransferWriteBit | AccessFlags.ShaderWriteBit | AccessFlags.MemoryWriteBit;
     private const AccessFlags TransferAccess = AccessFlags.TransferReadBit | AccessFlags.TransferWriteBit;
+    private const AccessFlags AnyWriteAccess = WriteAccess | AccessFlags.ColorAttachmentWriteBit | AccessFlags.DepthStencilAttachmentWriteBit;
+
+    // Reads after reads in one layout need no barrier: the state keeps every reader, so the next
+    // writer still waits for all of them. (A sampled depth attachment followed by a draw that only
+    // tests depth would otherwise end the rendering scope.)
+    private static bool IsReadAfterRead(ImageAccessState state, ImageLayout layout, AccessFlags access) =>
+        state.Layout == layout && state.Layout != ImageLayout.Undefined && access != 0 && state.Access != 0 &&
+        (state.Access & AnyWriteAccess) == 0 && (access & AnyWriteAccess) == 0;
     private const AccessFlags MemoryAccess = AccessFlags.MemoryReadBit | AccessFlags.MemoryWriteBit;
     private const ImageLayout ReadyLayout = ImageLayout.General;
     private const AccessFlags ReadyAccess = AccessFlags.ShaderReadBit | AccessFlags.TransferReadBit;
@@ -81,7 +89,11 @@ public sealed unsafe partial class CachedImage
 
                     var state = states[index];
                     var repeatedWrite = (state.Access & WriteAccess) != 0;
-                    if (state.Layout != layout || state.Access != access || repeatedWrite)
+                    if (IsReadAfterRead(state, layout, access))
+                    {
+                        states[index] = new ImageAccessState(state.Stage | stage, state.Access | access, layout);
+                    }
+                    else if (state.Layout != layout || state.Access != access || repeatedWrite)
                     {
                         barriers.Add(MakeBarrier(state, layout, access, level, 1, layer, 1));
                         sourceStages |= state.Stage;
@@ -104,6 +116,12 @@ public sealed unsafe partial class CachedImage
                 return (barriers, sourceStages);
             }
 
+            if (IsReadAfterRead(state, layout, access))
+            {
+                Backing.State = new ImageAccessState(state.Stage | stage, state.Access | access, layout);
+                return (barriers, sourceStages);
+            }
+
             barriers.Add(MakeBarrier(state, layout, access, 0, Vk.RemainingMipLevels, 0, Vk.RemainingArrayLayers));
             sourceStages |= state.Stage;
         }
@@ -112,6 +130,39 @@ public sealed unsafe partial class CachedImage
         return (barriers, sourceStages);
     }
 
+    // The layout every subresource in the range is in, or null when they differ.
+    public ImageLayout? UniformLayout(SubresourceRange range)
+    {
+        var states = Backing.SubresourceStates;
+        if (states == null)
+        {
+            return Backing.State.Layout;
+        }
+
+        var layers = Description.Resources.Layers;
+        if (Description.IsVolume)
+        {
+            range = range with { BaseLayer = 0, LayerCount = 1 };
+        }
+
+        ImageLayout? layout = null;
+        for (var level = range.BaseLevel; level < range.BaseLevel + range.LevelCount; level++)
+        {
+            for (var layer = range.BaseLayer; layer < range.BaseLayer + range.LayerCount; layer++)
+            {
+                var index = (int)(level * layers + layer);
+                if (index >= states.Count || (layout is { } known && known != states[index].Layout))
+                {
+                    return null;
+                }
+
+                layout = states[index].Layout;
+            }
+        }
+
+        return layout;
+    }
+
     public void Transition(ImageLayout layout, AccessFlags access, SubresourceRange? range, CommandBuffer command)
     {
         PipelineStageFlags stage = 0;

src/SharpEmu.Libs/Gpu/Rendering/DepthTargetResolver.cs

@@ -76,6 +76,20 @@ public readonly record struct DepthStencilState(
             : ImageLayout.DepthAttachmentOptimal;
     }
 
+    // A sampled attachment needs the read-only layouts; a draw that writes needs the writable
+    // one. A draw that neither samples nor writes keeps the read-only layout it is already in,
+    // so sampling and depth-testing draws can alternate inside one rendering scope.
+    public static ImageLayout AttachmentLayoutFor(DepthStencilState state, Format format, bool sampled, ImageLayout? current)
+    {
+        var readOnly = state.AttachmentLayout(format);
+        if (sampled)
+        {
+            return readOnly;
+        }
+
+        return state.IsReadOnly(format) && current == readOnly ? readOnly : WritableAttachmentLayout(format);
+    }
+
     public ImageLayout AttachmentLayout(Format format)
     {
         var available = ViewFormatRules.DepthAspects(format);

src/SharpEmu.Libs/Gpu/Rendering/IRenderHost.cs

@@ -140,6 +140,9 @@ public interface IRenderHost
     // Whether a texture bound for the draw being prepared reads the depth attachment's subresources.
     bool SamplesDepthAttachment(in DepthAttachmentState depth) => true;
 
+    // The layout the depth attachment's subresources are in now, or null when unknown or mixed.
+    ImageLayout? DepthAttachmentLayout(in DepthAttachmentState depth) => null;
+
     // The next draw stores to buffers or storage images; called before its BeginRendering.
     void PrepareMemoryWritingDraw() { }
 

src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Targets.cs

@@ -310,9 +310,11 @@ public sealed partial class RenderExecutor
             }
 
             var loadState = depth.LoadState;
-            var layout = _host.SamplesDepthAttachment(in depth)
-                ? loadState.AttachmentLayout(target.Format)
-                : DepthStencilState.WritableAttachmentLayout(target.Format);
+            var layout = DepthStencilState.AttachmentLayoutFor(
+                loadState,
+                target.Format,
+                _host.SamplesDepthAttachment(in depth),
+                _host.DepthAttachmentLayout(in depth));
             _host.TransitionDepthAttachment(in depth, layout, loadState.AttachmentWriteAspects(target.Format));
             var view = target.Request.View;
             rendering.Width = Math.Min(rendering.Width, target.Width);

src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Descriptors.cs

@@ -475,6 +475,12 @@ internal static unsafe partial class VulkanVideoPresenter
             return false;
         }
 
+        ImageLayout? IRenderHost.DepthAttachmentLayout(in DepthAttachmentState depth)
+        {
+            var view = depth.Target.Target.Request.View;
+            return _imageCache.GetImage(depth.Image).UniformLayout(new SubresourceRange(view.BaseLevel, view.LevelCount, view.BaseLayer, view.LayerCount));
+        }
+
         private BufferView NullStorageBuffer() => new(_bufferCache.GetBuffer(GuestBufferCache.NullBufferId).Handle, 0, NullStorageBufferBytes);
 
         // A storage buffer view on the cache buffer, aligned down with the adjustment carried in the memory offsets.

tests/SharpEmu.Libs.Tests/Gpu/Images/CachedImageTests.cs

@@ -203,6 +203,27 @@ public sealed unsafe class CachedImageTests : IClassFixture<HeadlessVulkanFixtur
         harness.AssertNoValidationMessages();
     }
 
+    [Fact]
+    public void Barriers_SkipReadsAfterReadsInOneLayoutButKeepEveryReaderForTheNextWriter()
+    {
+        if (!GatePrerequisites.Ready(_vulkan)) return;
+        using var harness = new ImageTestHarness(_vulkan);
+        var image = harness.CreateImage(Color2D(16, 16));
+        const ImageLayout readOnly = ImageLayout.ShaderReadOnlyOptimal;
+
+        Assert.Single(image.GetBarriers(readOnly, AccessFlags.ShaderReadBit, PipelineStageFlags.FragmentShaderBit, null).Barriers);
+        var (reread, _) = image.GetBarriers(readOnly, AccessFlags.TransferReadBit, PipelineStageFlags.TransferBit, null);
+        Assert.Empty(reread);
+        Assert.Equal(new ImageAccessState(PipelineStageFlags.FragmentShaderBit | PipelineStageFlags.TransferBit,
+            AccessFlags.ShaderReadBit | AccessFlags.TransferReadBit, readOnly), image.Backing.State);
+
+        var (write, writeStages) = image.GetBarriers(ImageLayout.General, AccessFlags.ShaderWriteBit, PipelineStageFlags.ComputeShaderBit, null);
+        var barrier = Assert.Single(write);
+        Assert.Equal(PipelineStageFlags.FragmentShaderBit | PipelineStageFlags.TransferBit, writeStages);
+        Assert.Equal(AccessFlags2.ShaderReadBit | AccessFlags2.TransferReadBit, barrier.SrcAccessMask);
+        harness.AssertNoValidationMessages();
+    }
+
     [Fact]
     public void Barriers_IgnoreLayerWindowsOnVolumes()
     {

tests/SharpEmu.Libs.Tests/Gpu/Rendering/TargetResolverTests.cs

@@ -234,6 +234,25 @@ public sealed class TargetResolverTests : IClassFixture<HeadlessVulkanFixture>
         Assert.Equal(ImageLayout.DepthAttachmentStencilReadOnlyOptimal, depthOnlyWrite.AttachmentLayout(Format.D32SfloatS8Uint));
     }
 
+    [Fact]
+    public void AttachmentLayoutFor_KeepsTheReadOnlyLayoutForDrawsThatOnlyTest()
+    {
+        var context = StencilContext(pass: 4, writeMask: 0xFF, operationValue: 0x10, DepthControl(CompareOp.Less, CompareOp.Always));
+        var writer = DepthTargetResolver.ResolveState(context, true, Fatal) with { StencilTestEnabled = false };
+        var tester = writer with { DepthWriteEnabled = false };
+        const Format format = Format.D32SfloatS8Uint;
+        var readOnly = ImageLayout.DepthStencilReadOnlyOptimal;
+
+        // A sampled attachment is read-only whatever it was in before.
+        Assert.Equal(readOnly, DepthStencilState.AttachmentLayoutFor(tester, format, sampled: true, ImageLayout.DepthStencilAttachmentOptimal));
+        // A test-only draw after a sampling draw stays read-only, so the scope continues.
+        Assert.Equal(readOnly, DepthStencilState.AttachmentLayoutFor(tester, format, sampled: false, readOnly));
+        // Otherwise the writable layout keeps toggling depth writes inside one scope.
+        Assert.Equal(ImageLayout.DepthStencilAttachmentOptimal, DepthStencilState.AttachmentLayoutFor(tester, format, sampled: false, ImageLayout.DepthStencilAttachmentOptimal));
+        Assert.Equal(ImageLayout.DepthStencilAttachmentOptimal, DepthStencilState.AttachmentLayoutFor(tester, format, sampled: false, null));
+        Assert.Equal(ImageLayout.DepthStencilAttachmentOptimal, DepthStencilState.AttachmentLayoutFor(writer, format, sampled: false, readOnly));
+    }
+
     [Fact]
     public void DepthResolve_UsesTheBuilderThenAddsTheState()
     {