[GPU] Sweep CPU writes at memory visibility points instead of before every device-address program (#1033)

* [GPU] Sweep CPU writes for device-address programs at memory visibility points

A device-address program may read any mapped page, so before each such
draw or dispatch the buffer cache uploaded every CPU-dirty page whenever
the global CPU-dirty epoch had moved. A guest that streams its per-draw
ring while the command stream is being translated moves the epoch between
almost every pair of draws, and each upload ended the rendering scope and
drained the GPU with an ALL_COMMANDS barrier pair - about 670 times per
frame in Silent Hill: The Short Message's menu.

CPU writes only need to be visible at memory visibility points: when a
submission slice starts, when a wait packet is satisfied, and after a
command-processor memory write (later commands may read those bytes through
device addresses). The sweep now runs at the first device-address program
after such a point, and its uploads share one barrier pair instead of one
per buffer. SHARPEMU_BDA_SWEEP_EVERY_DRAW=1 restores the per-draw sweep.
Menu GPU time drops from about 435 to 290 ms per frame.

(cherry picked from commit fafe92b71271cf750dd89251891bac20fb87f1ff)

* [GPU] Let command-processor DMA writes turn pages CPU-dirty instead of copying

A DMA fill or copy into memory without GPU-written bytes is executed on
the CPU, and the bytes were then pushed into every overlapping cache
buffer through a staging copy and a barrier pair, which ends the
rendering scope (~60 per frame in Silent Hill: The Short Message's menu).

When no GPU-written byte shares the written pages, the pages now simply
turn CPU-dirty: every later GPU reader synchronizes CPU-dirty pages before
reading (bound buffers on obtain, device-address programs at the next
memory visibility point, which the write now raises), so the bytes still
arrive in command order. Pages that hold GPU-written bytes keep the
staging copy. SHARPEMU_DEFER_HOST_WRITES=0 restores the copy.

(cherry picked from commit 3fe21faa3821551fa99b9398064f0652e45d2e84)

* [GPU] Keep hot CPU-written pages writable across device-address sweeps

A page the guest rewrites every frame turns hot after two upload/write
cycles, and the bound-buffer path then serves it from the stream ring so
the page stays writable. The device-address sweep and the range
synchronization still re-protected hot pages after every upload, so the
guest's next write to each ring page took a protection fault - about
5,000 faults per second on Silent Hill: The Short Message's render
thread, each a signal round trip under Rosetta plus page-guard lock
traffic shared with the command worker.

Both sweeps now preserve hot pages: they stay dirty and writable and are
copied again on the next sweep, a few hundred KiB of ring memory per
sweep. Faults drop to a third and the menu goes from 3.8 to 4.2 FPS.
SHARPEMU_BDA_SWEEP_REPROTECT_HOT=1 restores the re-protection.

(cherry picked from commit 65d2b55ddc07644d52e5ad3112cbe323a2421abd)

* [GPU] Refresh writable hot pages at every BDA visibility point

src/SharpEmu.Libs/Gpu/Buffers/GuestBufferCache.cs

@@ -354,6 +354,27 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable
 
         // Registered buffers are ordered and disjoint: walk only the ones the write overlaps.
         var end = guestAddress + (ulong)data.Length;
+        if (DeferHostWrites)
+        {
+            // When no GPU-written byte shares the written pages, the pages can simply turn
+            // CPU-dirty: every later GPU reader synchronizes CPU-dirty pages first (bound buffers
+            // on obtain, device-address programs at the next visibility point), so the bytes
+            // arrive in command order without a staging copy that ends the rendering scope.
+            var firstPage = guestAddress & ~(TrackerLayout.PageBytes - 1);
+            var pagesEnd = (end + TrackerLayout.PageBytes - 1) & ~(TrackerLayout.PageBytes - 1);
+            if (!_gpuModifiedRanges.Overlaps(firstPage, pagesEnd - firstPage) &&
+                !_tracker.HasGpuDirtyPages(firstPage, pagesEnd - firstPage))
+            {
+                if (IsRegionRegistered(guestAddress, (ulong)data.Length))
+                {
+                    _tracker.MarkCpuDirtyPages(guestAddress, (ulong)data.Length);
+                    NoteMemoryVisibilityPoint();
+                }
+
+                return;
+            }
+        }
+
         for (var index = _registry.FindFirstOverlappingIndex(guestAddress);
              index < _registry.RegisteredCount && _registry.GetRegisteredAddress(index) < end;
              index++)
@@ -373,6 +394,9 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable
         }
     }
 
+    private static readonly bool DeferHostWrites =
+        Environment.GetEnvironmentVariable("SHARPEMU_DEFER_HOST_WRITES") != "0";
+
     public void FillBuffer(ulong guestAddress, ulong size, uint value, bool isGds)
     {
         if ((guestAddress & 3) != 0 || size == 0 || (size & 3) != 0 || size > ulong.MaxValue - guestAddress)
@@ -679,13 +703,28 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable
                 _bdaTouchMapping = mapping;
             }
 
-            var epoch = _tracker.CpuDirtyEpoch;
+            // Device-address programs may read any mapped page, so CPU writes are swept into the
-            if (epoch != _bdaSweepEpoch || mapping != _bdaSweepMapping)
+            // buffers - but only at memory visibility points (a submission slice starting, a wait
+            // packet satisfied). A CPU write after the submission began carries no ordering
+            // guarantee for its commands, and sweeping on every draw would end the render pass
+            // for every page the guest streams while the stream is being translated.
+            // Hot pages remain writable, so later CPU writes need not advance CpuDirtyEpoch.
+            // Every visibility point must revisit their dirty ranges even with an unchanged epoch.
+            if (mapping != _bdaSweepMapping || _bdaVisibilityPending || SweepBdaOnEveryDraw)
             {
-                foreach (var span in spans)
+                BeginUploadBatch();
-                    _tracker.ForEachPossiblyCpuDirtyRange(span.Address, span.Size, _uploadDirtyBuffersInRange ??= UploadDirtyBuffersInRange);
+                try
-                _bdaSweepEpoch = epoch;
+                {
+                    foreach (var span in spans)
+                        _tracker.ForEachPossiblyCpuDirtyRange(span.Address, span.Size, _uploadDirtyBuffersInRange ??= UploadDirtyBuffersInRange);
+                }
+                finally
+                {
+                    EndUploadBatch();
+                }
+
                 _bdaSweepMapping = mapping;
+                _bdaVisibilityPending = false;
             }
         }
 
@@ -696,7 +735,15 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable
     }
 
     private Action<ulong, ulong>? _uploadDirtyBuffersInRange;
-    private long _bdaSweepEpoch = -1;
+
+    private static readonly bool PreserveHotPagesInSweeps =
+        Environment.GetEnvironmentVariable("SHARPEMU_BDA_SWEEP_REPROTECT_HOT") != "1";
+    private bool _bdaVisibilityPending = true;
+
+    private static readonly bool SweepBdaOnEveryDraw =
+        Environment.GetEnvironmentVariable("SHARPEMU_BDA_SWEEP_EVERY_DRAW") == "1";
+
+    public void NoteMemoryVisibilityPoint() => _bdaVisibilityPending = true;
     private ulong _bdaSweepMapping;
 
     private void TouchBuffersInRange(ulong guestAddress, ulong size)
@@ -717,8 +764,10 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable
             var buffer = _registry.GetBuffer(_registry.GetRegisteredIdentifier(index));
             var start = Math.Max(buffer.CpuAddress, guestAddress);
             var finish = Math.Min(buffer.CpuAddress + buffer.Size, end);
+            // Hot pages stay writable and are copied again on the next sweep: a few hundred KiB of
+            // ring memory per sweep costs far less than a protection fault per page per frame.
             if (start < finish && _tracker.MayHaveCpuDirtyPages(start, finish - start))
-                _ = SynchronizeBuffer(buffer, start, finish - start, false, false, preserveCpuWriteHotPages: false);
+                _ = SynchronizeBuffer(buffer, start, finish - start, false, false, preserveCpuWriteHotPages: PreserveHotPagesInSweeps);
         }
     }
 
@@ -744,7 +793,7 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable
                 // SynchronizeBuffer takes for this read-only call), and the block summary
                 // answers that without a lock for the common all-clean case.
                 if (_tracker.HasCpuDirtyPages(start, finish - start))
-                    _ = SynchronizeBuffer(buffer, start, finish - start, false, false, preserveCpuWriteHotPages: false);
+                    _ = SynchronizeBuffer(buffer, start, finish - start, false, false, preserveCpuWriteHotPages: PreserveHotPagesInSweeps);
             }
         }
     }
@@ -2059,6 +2108,17 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable
             command.EndRendering();
             var native = new CommandBuffer(command.Handle);
             var vk = _device.Vk;
+            if (_uploadBatchActive)
+            {
+                OpenUploadBatch(native);
+                var batched = CollectionsMarshal.AsSpan(copies);
+                fixed (BufferCopy* pointer = batched)
+                {
+                    vk.CmdCopyBuffer(native, source.Handle, buffer.Handle, (uint)batched.Length, pointer);
+                }
+            }
+            else
+            {
             var before = new BufferMemoryBarrier2
             {
                 SType = StructureType.BufferMemoryBarrier2,
@@ -2085,6 +2145,7 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable
             VulkanSynchronization.PipelineBarrier(vk,
                 native, PipelineStageFlags.TransferBit, PipelineStageFlags.AllCommandsBit, DependencyFlags.ByRegionBit,
                 0, null, 1, &after, 0, null);
+            }
         }
 
         if (BufferUploadProfile.Enabled)
@@ -2110,6 +2171,59 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable
         return false;
     }
 
+    // A sweep uploads many buffers back to back: one global barrier before the first copy and one
+    // after the last replace a barrier pair per buffer, so the copies share one transfer pass.
+    private bool _uploadBatchActive;
+    private ulong _uploadBatchCommand;
+
+    private void BeginUploadBatch()
+    {
+        _uploadBatchActive = true;
+        _uploadBatchCommand = 0;
+    }
+
+    private void OpenUploadBatch(CommandBuffer native)
+    {
+        if (_uploadBatchCommand == (ulong)native.Handle)
+        {
+            return;
+        }
+
+        // A new command buffer (or the first copy) orders the copies after all earlier work.
+        var barrier = new MemoryBarrier2
+        {
+            SType = StructureType.MemoryBarrier2,
+            SrcAccessMask = AccessFlags2.MemoryReadBit | AccessFlags2.MemoryWriteBit,
+            DstAccessMask = AccessFlags2.TransferWriteBit,
+        };
+        VulkanSynchronization.PipelineBarrier(_device.Vk,
+            native, PipelineStageFlags.AllCommandsBit, PipelineStageFlags.TransferBit, 0, 1, &barrier, 0, null, 0, null);
+        _uploadBatchCommand = (ulong)native.Handle;
+    }
+
+    private void EndUploadBatch()
+    {
+        _uploadBatchActive = false;
+        if (_uploadBatchCommand == 0)
+        {
+            return;
+        }
+
+        // Recorded on the current command buffer; submission order covers copies recorded on an
+        // earlier one.
+        var command = _scheduler.Current;
+        command.EndRendering();
+        var barrier = new MemoryBarrier2
+        {
+            SType = StructureType.MemoryBarrier2,
+            SrcAccessMask = AccessFlags2.TransferWriteBit,
+            DstAccessMask = AccessFlags2.MemoryReadBit | AccessFlags2.MemoryWriteBit,
+        };
+        VulkanSynchronization.PipelineBarrier(_device.Vk,
+            new CommandBuffer(command.Handle), PipelineStageFlags.TransferBit, PipelineStageFlags.AllCommandsBit, 0, 1, &barrier, 0, null, 0, null);
+        _uploadBatchCommand = 0;
+    }
+
     // The copies of one synchronization; the tracker forbids nesting, so one list per thread is reused.
     [ThreadStatic]
     private static List<BufferCopy>? _syncCopies;

src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Waits.cs

@@ -42,7 +42,10 @@ public sealed partial class GpuCommandInterpreter
         if (!satisfied)
         {
             Suspend();
+            return;
         }
+
+        _host.NoteMemoryVisibilityPoint();
     }
 
     // The wait bit is PredicationZPassWaitOp: 0 (kWaitForQueryResults) stalls the command

src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.cs

@@ -443,6 +443,8 @@ public sealed partial class GpuCommandInterpreter
         {
             throw _host.Fatal($"The command stream cannot write guest memory: address=0x{address:X16} size={source.Length}.");
         }
+        // Later commands in this stream may read the bytes through device addresses.
+        _host.NoteMemoryVisibilityPoint();
         _lastWriteLength = 0;
         if (source.Length is sizeof(uint) or sizeof(ulong))
         {

src/SharpEmu.Libs/Gpu/GpuCommands/ICommandStreamHost.cs

@@ -107,6 +107,13 @@ public interface ICommandStreamHost
 
     void CopyBuffer(ulong destination, ulong source, ulong size, bool destinationIsGds, bool sourceIsGds);
 
+    // A satisfied wait packet or a command-processor memory write: CPU writes released by the
+    // waited-on value, or the written bytes, must become visible to the commands that follow,
+    // the same as at the start of a submission slice.
+    void NoteMemoryVisibilityPoint()
+    {
+    }
+
     void ReadGds(Span<uint> destination, uint wordOffset, uint wordCount);
 
     void RecordEndOfPipe(in EndOfPipeWrite write);

src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CommandStream.cs

@@ -316,9 +316,12 @@ internal static unsafe partial class VulkanVideoPresenter
             _activeGuestQueue = new VulkanGuestQueueIdentity(_commandQueueNames[queueId], submissionId);
             BindSubmissionContext(_activeGuestQueue);
             _ = CurrentRecordingBuffer();
+            _bufferCache.NoteMemoryVisibilityPoint();
             _translation.BeginSubmission(queueId, submissionId, geometrySnapshots, _commandStream.GetInterpreter(queueId));
         }
 
+        public void NoteMemoryVisibilityPoint() => _bufferCache.NoteMemoryVisibilityPoint();
+
         public void Flush()
         {
             using var flushScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.Flush);

tests/SharpEmu.Libs.Tests/Gpu/Buffers/GuestBufferCacheTests.cs

@@ -674,7 +674,12 @@ public sealed class GuestBufferCacheTests : IClassFixture<HeadlessVulkanFixture>
         var address = harness.MapBacked(0x10000, ReadWrite);
         var (buffer, _) = harness.Worker.Run(() => harness.Cache.ObtainBuffer(address, 0x8000, isWritten: false));
 
-        harness.Worker.Run(() => harness.Cache.WriteHostMemory(address + 0x1000, Pattern(0x100, 7)));
+        harness.Worker.Run(() =>
+        {
+            harness.Cache.WriteHostMemory(address + 0x1000, Pattern(0x100, 7));
+            // Deferred host writes become visible when a command acquires its buffer.
+            harness.Cache.PrepareBda([new GuestSpan(address, 0x10000)]);
+        });
 
         Assert.Equal(Pattern(0x100, 7), harness.Read(address + 0x1000, 0x100));
         Assert.Equal(Pattern(0x100, 7), harness.ReadBack(buffer, 0x1000, 0x100));
@@ -801,8 +806,8 @@ public sealed class GuestBufferCacheTests : IClassFixture<HeadlessVulkanFixture>
             harness.Write(address, expected);
             harness.Worker.Run(() =>
             {
+                harness.Cache.NoteMemoryVisibilityPoint();
                 harness.Cache.PrepareBda([new GuestSpan(address, size)]);
-                Assert.False(harness.Cache.HasCpuDirtyPages(address, size));
                 Assert.False(harness.Cache.HasGpuDirtyPages(address, size));
                 harness.Cache.PrepareBda([new GuestSpan(address, size)]);
             });
@@ -835,7 +840,11 @@ public sealed class GuestBufferCacheTests : IClassFixture<HeadlessVulkanFixture>
         });
         Assert.Equal(Bytes(0x12345678u), harness.ReadBack(buffer, offset, 4));
         Assert.True(harness.Cache.TrySynchronizeCpuRead(address, size));
-        harness.Worker.Run(() => harness.Cache.PrepareBda([new GuestSpan(address, size)]));
+        harness.Worker.Run(() =>
+        {
+            harness.Cache.NoteMemoryVisibilityPoint();
+            harness.Cache.PrepareBda([new GuestSpan(address, size)]);
+        });
 
         var expected = Pattern((int)size, 7);
         await Task.Run(() =>
@@ -843,12 +852,48 @@ public sealed class GuestBufferCacheTests : IClassFixture<HeadlessVulkanFixture>
             Assert.True(harness.Store.MarkCpuWrite(address, size));
             harness.Write(address, expected);
         });
-        harness.Worker.Run(() => harness.Cache.PrepareBda([new GuestSpan(address, size)]));
+        harness.Worker.Run(() =>
+        {
+            harness.Cache.NoteMemoryVisibilityPoint();
+            harness.Cache.PrepareBda([new GuestSpan(address, size)]);
+        });
         Assert.False(harness.Cache.HasCpuDirtyPages(address, size));
         Assert.Equal(expected, harness.ReadBack(buffer, offset, size));
         harness.Shutdown();
     }
 
+    [Fact]
+    public void DeviceAddressVisibilityPointRefreshesHotPagesWithoutAnotherWriteFault()
+    {
+        if (!GatePrerequisites.Ready(_vulkan)) return;
+        using var harness = new CacheHarness(_vulkan);
+        var address = harness.MapBacked(0x20000, ReadWrite);
+        const ulong size = 0x8000;
+        var (buffer, offset) = harness.Worker.Run(() => harness.Cache.ObtainBuffer(address, size, false));
+        for (var write = 1; write <= 3; write++)
+        {
+            Assert.True(harness.Store.MarkCpuWrite(address, size));
+            harness.Write(address, Pattern((int)size, (byte)write));
+            harness.Worker.Run(() =>
+            {
+                harness.Cache.NoteMemoryVisibilityPoint();
+                harness.Cache.PrepareBda([new GuestSpan(address, size)]);
+            });
+        }
+
+        Assert.True(harness.Cache.HasCpuDirtyPages(address, size));
+        var expected = Pattern((int)size, 9);
+        // A permanently writable hot page changes without another protection fault.
+        harness.Write(address, expected);
+        harness.Worker.Run(() =>
+        {
+            harness.Cache.NoteMemoryVisibilityPoint();
+            harness.Cache.PrepareBda([new GuestSpan(address, size)]);
+        });
+        Assert.Equal(expected, harness.ReadBack(buffer, offset, size));
+        harness.Shutdown();
+    }
+
     [Theory]
     [InlineData(false)]
     [InlineData(true)]

tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/CommandStreamTestSupport.cs

@@ -31,6 +31,11 @@ internal sealed class RecordingCommandStreamHost : ICommandStreamHost
 
     public List<EndOfPipeWrite> EndOfPipeWrites { get; } = new();
 
+    // Satisfied waits and command-processor memory writes reported to the host.
+    public int VisibilityPoints { get; private set; }
+
+    public void NoteMemoryVisibilityPoint() => VisibilityPoints++;
+
     public List<ulong> GuestReads { get; } = new();
 
     public Action<ulong>? BeforeGuestRead { get; set; }

tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/GpuCommandInterpreterLoopTests.cs

@@ -261,6 +261,30 @@ public sealed class GpuCommandInterpreterLoopTests
         Assert.Contains(Label, runner.Host.GuestReads);
     }
 
+    [Fact]
+    public void Wait_IsAMemoryVisibilityPointOnlyOnceSatisfied()
+    {
+        var runner = new StreamRunner();
+
+        Assert.Equal(SubmissionProgress.Blocked, runner.Run(WaitEqual(Label, 1)));
+        Assert.Equal(0, runner.Host.VisibilityPoints);
+
+        runner.Host.WriteDword(Label, 1);
+        Assert.Equal(SubmissionProgress.Complete, runner.Run());
+        Assert.Equal(1, runner.Host.VisibilityPoints);
+    }
+
+    [Fact]
+    public void WriteData_IsAMemoryVisibilityPoint()
+    {
+        var runner = new StreamRunner();
+
+        runner.Run(WriteData(Label, 7));
+
+        Assert.Equal(7u, runner.Host.ReadDword(Label));
+        Assert.Equal(1, runner.Host.VisibilityPoints);
+    }
+
     [Fact]
     public void PendingCommands_RunBeforeEveryPacket()
     {