From 775141d58ca6e51514e6b89cf1f911d31e86c6fe Mon Sep 17 00:00:00 2001 From: kuba Date: Wed, 7 Oct 2026 11:52:12 +0200 Subject: [PATCH] [GPU] Sweep CPU writes at memory visibility points instead of before every device-address program (#1033) * [GPU] Sweep CPU writes for device-address programs at memory visibility points A device-address program may read any mapped page, so before each such draw or dispatch the buffer cache uploaded every CPU-dirty page whenever the global CPU-dirty epoch had moved. A guest that streams its per-draw ring while the command stream is being translated moves the epoch between almost every pair of draws, and each upload ended the rendering scope and drained the GPU with an ALL_COMMANDS barrier pair - about 670 times per frame in Silent Hill: The Short Message's menu. CPU writes only need to be visible at memory visibility points: when a submission slice starts, when a wait packet is satisfied, and after a command-processor memory write (later commands may read those bytes through device addresses). The sweep now runs at the first device-address program after such a point, and its uploads share one barrier pair instead of one per buffer. SHARPEMU_BDA_SWEEP_EVERY_DRAW=1 restores the per-draw sweep. Menu GPU time drops from about 435 to 290 ms per frame. (cherry picked from commit fafe92b71271cf750dd89251891bac20fb87f1ff) * [GPU] Let command-processor DMA writes turn pages CPU-dirty instead of copying A DMA fill or copy into memory without GPU-written bytes is executed on the CPU, and the bytes were then pushed into every overlapping cache buffer through a staging copy and a barrier pair, which ends the rendering scope (~60 per frame in Silent Hill: The Short Message's menu). When no GPU-written byte shares the written pages, the pages now simply turn CPU-dirty: every later GPU reader synchronizes CPU-dirty pages before reading (bound buffers on obtain, device-address programs at the next memory visibility point, which the write now raises), so the bytes still arrive in command order. Pages that hold GPU-written bytes keep the staging copy. SHARPEMU_DEFER_HOST_WRITES=0 restores the copy. (cherry picked from commit 3fe21faa3821551fa99b9398064f0652e45d2e84) * [GPU] Keep hot CPU-written pages writable across device-address sweeps A page the guest rewrites every frame turns hot after two upload/write cycles, and the bound-buffer path then serves it from the stream ring so the page stays writable. The device-address sweep and the range synchronization still re-protected hot pages after every upload, so the guest's next write to each ring page took a protection fault - about 5,000 faults per second on Silent Hill: The Short Message's render thread, each a signal round trip under Rosetta plus page-guard lock traffic shared with the command worker. Both sweeps now preserve hot pages: they stay dirty and writable and are copied again on the next sweep, a few hundred KiB of ring memory per sweep. Faults drop to a third and the menu goes from 3.8 to 4.2 FPS. SHARPEMU_BDA_SWEEP_REPROTECT_HOT=1 restores the re-protection. (cherry picked from commit 65d2b55ddc07644d52e5ad3112cbe323a2421abd) * [GPU] Refresh writable hot pages at every BDA visibility point --- .../Gpu/Buffers/GuestBufferCache.cs | 130 ++++++++++++++++-- .../GpuCommandInterpreter.Waits.cs | 3 + .../Gpu/GpuCommands/GpuCommandInterpreter.cs | 2 + .../Gpu/GpuCommands/ICommandStreamHost.cs | 7 + .../VulkanVideoPresenter.CommandStream.cs | 3 + .../Gpu/Buffers/GuestBufferCacheTests.cs | 53 ++++++- .../GpuCommands/CommandStreamTestSupport.cs | 5 + .../GpuCommandInterpreterLoopTests.cs | 24 ++++ 8 files changed, 215 insertions(+), 12 deletions(-) diff --git a/src/SharpEmu.Libs/Gpu/Buffers/GuestBufferCache.cs b/src/SharpEmu.Libs/Gpu/Buffers/GuestBufferCache.cs index 33a6253..76c9d1e 100644 --- a/src/SharpEmu.Libs/Gpu/Buffers/GuestBufferCache.cs +++ b/src/SharpEmu.Libs/Gpu/Buffers/GuestBufferCache.cs @@ -354,6 +354,27 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable // Registered buffers are ordered and disjoint: walk only the ones the write overlaps. var end = guestAddress + (ulong)data.Length; + if (DeferHostWrites) + { + // When no GPU-written byte shares the written pages, the pages can simply turn + // CPU-dirty: every later GPU reader synchronizes CPU-dirty pages first (bound buffers + // on obtain, device-address programs at the next visibility point), so the bytes + // arrive in command order without a staging copy that ends the rendering scope. + var firstPage = guestAddress & ~(TrackerLayout.PageBytes - 1); + var pagesEnd = (end + TrackerLayout.PageBytes - 1) & ~(TrackerLayout.PageBytes - 1); + if (!_gpuModifiedRanges.Overlaps(firstPage, pagesEnd - firstPage) && + !_tracker.HasGpuDirtyPages(firstPage, pagesEnd - firstPage)) + { + if (IsRegionRegistered(guestAddress, (ulong)data.Length)) + { + _tracker.MarkCpuDirtyPages(guestAddress, (ulong)data.Length); + NoteMemoryVisibilityPoint(); + } + + return; + } + } + for (var index = _registry.FindFirstOverlappingIndex(guestAddress); index < _registry.RegisteredCount && _registry.GetRegisteredAddress(index) < end; index++) @@ -373,6 +394,9 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable } } + private static readonly bool DeferHostWrites = + Environment.GetEnvironmentVariable("SHARPEMU_DEFER_HOST_WRITES") != "0"; + public void FillBuffer(ulong guestAddress, ulong size, uint value, bool isGds) { if ((guestAddress & 3) != 0 || size == 0 || (size & 3) != 0 || size > ulong.MaxValue - guestAddress) @@ -679,13 +703,28 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable _bdaTouchMapping = mapping; } - var epoch = _tracker.CpuDirtyEpoch; - if (epoch != _bdaSweepEpoch || mapping != _bdaSweepMapping) + // Device-address programs may read any mapped page, so CPU writes are swept into the + // buffers - but only at memory visibility points (a submission slice starting, a wait + // packet satisfied). A CPU write after the submission began carries no ordering + // guarantee for its commands, and sweeping on every draw would end the render pass + // for every page the guest streams while the stream is being translated. + // Hot pages remain writable, so later CPU writes need not advance CpuDirtyEpoch. + // Every visibility point must revisit their dirty ranges even with an unchanged epoch. + if (mapping != _bdaSweepMapping || _bdaVisibilityPending || SweepBdaOnEveryDraw) { - foreach (var span in spans) - _tracker.ForEachPossiblyCpuDirtyRange(span.Address, span.Size, _uploadDirtyBuffersInRange ??= UploadDirtyBuffersInRange); - _bdaSweepEpoch = epoch; + BeginUploadBatch(); + try + { + foreach (var span in spans) + _tracker.ForEachPossiblyCpuDirtyRange(span.Address, span.Size, _uploadDirtyBuffersInRange ??= UploadDirtyBuffersInRange); + } + finally + { + EndUploadBatch(); + } + _bdaSweepMapping = mapping; + _bdaVisibilityPending = false; } } @@ -696,7 +735,15 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable } private Action? _uploadDirtyBuffersInRange; - private long _bdaSweepEpoch = -1; + + private static readonly bool PreserveHotPagesInSweeps = + Environment.GetEnvironmentVariable("SHARPEMU_BDA_SWEEP_REPROTECT_HOT") != "1"; + private bool _bdaVisibilityPending = true; + + private static readonly bool SweepBdaOnEveryDraw = + Environment.GetEnvironmentVariable("SHARPEMU_BDA_SWEEP_EVERY_DRAW") == "1"; + + public void NoteMemoryVisibilityPoint() => _bdaVisibilityPending = true; private ulong _bdaSweepMapping; private void TouchBuffersInRange(ulong guestAddress, ulong size) @@ -717,8 +764,10 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable var buffer = _registry.GetBuffer(_registry.GetRegisteredIdentifier(index)); var start = Math.Max(buffer.CpuAddress, guestAddress); var finish = Math.Min(buffer.CpuAddress + buffer.Size, end); + // Hot pages stay writable and are copied again on the next sweep: a few hundred KiB of + // ring memory per sweep costs far less than a protection fault per page per frame. if (start < finish && _tracker.MayHaveCpuDirtyPages(start, finish - start)) - _ = SynchronizeBuffer(buffer, start, finish - start, false, false, preserveCpuWriteHotPages: false); + _ = SynchronizeBuffer(buffer, start, finish - start, false, false, preserveCpuWriteHotPages: PreserveHotPagesInSweeps); } } @@ -744,7 +793,7 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable // SynchronizeBuffer takes for this read-only call), and the block summary // answers that without a lock for the common all-clean case. if (_tracker.HasCpuDirtyPages(start, finish - start)) - _ = SynchronizeBuffer(buffer, start, finish - start, false, false, preserveCpuWriteHotPages: false); + _ = SynchronizeBuffer(buffer, start, finish - start, false, false, preserveCpuWriteHotPages: PreserveHotPagesInSweeps); } } } @@ -2059,6 +2108,17 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable command.EndRendering(); var native = new CommandBuffer(command.Handle); var vk = _device.Vk; + if (_uploadBatchActive) + { + OpenUploadBatch(native); + var batched = CollectionsMarshal.AsSpan(copies); + fixed (BufferCopy* pointer = batched) + { + vk.CmdCopyBuffer(native, source.Handle, buffer.Handle, (uint)batched.Length, pointer); + } + } + else + { var before = new BufferMemoryBarrier2 { SType = StructureType.BufferMemoryBarrier2, @@ -2085,6 +2145,7 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable VulkanSynchronization.PipelineBarrier(vk, native, PipelineStageFlags.TransferBit, PipelineStageFlags.AllCommandsBit, DependencyFlags.ByRegionBit, 0, null, 1, &after, 0, null); + } } if (BufferUploadProfile.Enabled) @@ -2110,6 +2171,59 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable return false; } + // A sweep uploads many buffers back to back: one global barrier before the first copy and one + // after the last replace a barrier pair per buffer, so the copies share one transfer pass. + private bool _uploadBatchActive; + private ulong _uploadBatchCommand; + + private void BeginUploadBatch() + { + _uploadBatchActive = true; + _uploadBatchCommand = 0; + } + + private void OpenUploadBatch(CommandBuffer native) + { + if (_uploadBatchCommand == (ulong)native.Handle) + { + return; + } + + // A new command buffer (or the first copy) orders the copies after all earlier work. + var barrier = new MemoryBarrier2 + { + SType = StructureType.MemoryBarrier2, + SrcAccessMask = AccessFlags2.MemoryReadBit | AccessFlags2.MemoryWriteBit, + DstAccessMask = AccessFlags2.TransferWriteBit, + }; + VulkanSynchronization.PipelineBarrier(_device.Vk, + native, PipelineStageFlags.AllCommandsBit, PipelineStageFlags.TransferBit, 0, 1, &barrier, 0, null, 0, null); + _uploadBatchCommand = (ulong)native.Handle; + } + + private void EndUploadBatch() + { + _uploadBatchActive = false; + if (_uploadBatchCommand == 0) + { + return; + } + + // Recorded on the current command buffer; submission order covers copies recorded on an + // earlier one. + var command = _scheduler.Current; + command.EndRendering(); + var barrier = new MemoryBarrier2 + { + SType = StructureType.MemoryBarrier2, + SrcAccessMask = AccessFlags2.TransferWriteBit, + DstAccessMask = AccessFlags2.MemoryReadBit | AccessFlags2.MemoryWriteBit, + }; + VulkanSynchronization.PipelineBarrier(_device.Vk, + new CommandBuffer(command.Handle), PipelineStageFlags.TransferBit, PipelineStageFlags.AllCommandsBit, 0, 1, &barrier, 0, null, 0, null); + _uploadBatchCommand = 0; + } + // The copies of one synchronization; the tracker forbids nesting, so one list per thread is reused. [ThreadStatic] private static List? _syncCopies; diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Waits.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Waits.cs index 7e6f8fd..0883dd6 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Waits.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Waits.cs @@ -42,7 +42,10 @@ public sealed partial class GpuCommandInterpreter if (!satisfied) { Suspend(); + return; } + + _host.NoteMemoryVisibilityPoint(); } // The wait bit is PredicationZPassWaitOp: 0 (kWaitForQueryResults) stalls the command diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.cs index ef91e56..d5c49e7 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.cs @@ -443,6 +443,8 @@ public sealed partial class GpuCommandInterpreter { throw _host.Fatal($"The command stream cannot write guest memory: address=0x{address:X16} size={source.Length}."); } + // Later commands in this stream may read the bytes through device addresses. + _host.NoteMemoryVisibilityPoint(); _lastWriteLength = 0; if (source.Length is sizeof(uint) or sizeof(ulong)) { diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/ICommandStreamHost.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/ICommandStreamHost.cs index cdc9461..cbcae94 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/ICommandStreamHost.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/ICommandStreamHost.cs @@ -107,6 +107,13 @@ public interface ICommandStreamHost void CopyBuffer(ulong destination, ulong source, ulong size, bool destinationIsGds, bool sourceIsGds); + // A satisfied wait packet or a command-processor memory write: CPU writes released by the + // waited-on value, or the written bytes, must become visible to the commands that follow, + // the same as at the start of a submission slice. + void NoteMemoryVisibilityPoint() + { + } + void ReadGds(Span destination, uint wordOffset, uint wordCount); void RecordEndOfPipe(in EndOfPipeWrite write); diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CommandStream.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CommandStream.cs index e0eb2df..ac7532c 100644 --- a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CommandStream.cs +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CommandStream.cs @@ -316,9 +316,12 @@ internal static unsafe partial class VulkanVideoPresenter _activeGuestQueue = new VulkanGuestQueueIdentity(_commandQueueNames[queueId], submissionId); BindSubmissionContext(_activeGuestQueue); _ = CurrentRecordingBuffer(); + _bufferCache.NoteMemoryVisibilityPoint(); _translation.BeginSubmission(queueId, submissionId, geometrySnapshots, _commandStream.GetInterpreter(queueId)); } + public void NoteMemoryVisibilityPoint() => _bufferCache.NoteMemoryVisibilityPoint(); + public void Flush() { using var flushScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.Flush); diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Buffers/GuestBufferCacheTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Buffers/GuestBufferCacheTests.cs index 6e2617c..632eeaa 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Buffers/GuestBufferCacheTests.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Buffers/GuestBufferCacheTests.cs @@ -674,7 +674,12 @@ public sealed class GuestBufferCacheTests : IClassFixture var address = harness.MapBacked(0x10000, ReadWrite); var (buffer, _) = harness.Worker.Run(() => harness.Cache.ObtainBuffer(address, 0x8000, isWritten: false)); - harness.Worker.Run(() => harness.Cache.WriteHostMemory(address + 0x1000, Pattern(0x100, 7))); + harness.Worker.Run(() => + { + harness.Cache.WriteHostMemory(address + 0x1000, Pattern(0x100, 7)); + // Deferred host writes become visible when a command acquires its buffer. + harness.Cache.PrepareBda([new GuestSpan(address, 0x10000)]); + }); Assert.Equal(Pattern(0x100, 7), harness.Read(address + 0x1000, 0x100)); Assert.Equal(Pattern(0x100, 7), harness.ReadBack(buffer, 0x1000, 0x100)); @@ -801,8 +806,8 @@ public sealed class GuestBufferCacheTests : IClassFixture harness.Write(address, expected); harness.Worker.Run(() => { + harness.Cache.NoteMemoryVisibilityPoint(); harness.Cache.PrepareBda([new GuestSpan(address, size)]); - Assert.False(harness.Cache.HasCpuDirtyPages(address, size)); Assert.False(harness.Cache.HasGpuDirtyPages(address, size)); harness.Cache.PrepareBda([new GuestSpan(address, size)]); }); @@ -835,7 +840,11 @@ public sealed class GuestBufferCacheTests : IClassFixture }); Assert.Equal(Bytes(0x12345678u), harness.ReadBack(buffer, offset, 4)); Assert.True(harness.Cache.TrySynchronizeCpuRead(address, size)); - harness.Worker.Run(() => harness.Cache.PrepareBda([new GuestSpan(address, size)])); + harness.Worker.Run(() => + { + harness.Cache.NoteMemoryVisibilityPoint(); + harness.Cache.PrepareBda([new GuestSpan(address, size)]); + }); var expected = Pattern((int)size, 7); await Task.Run(() => @@ -843,12 +852,48 @@ public sealed class GuestBufferCacheTests : IClassFixture Assert.True(harness.Store.MarkCpuWrite(address, size)); harness.Write(address, expected); }); - harness.Worker.Run(() => harness.Cache.PrepareBda([new GuestSpan(address, size)])); + harness.Worker.Run(() => + { + harness.Cache.NoteMemoryVisibilityPoint(); + harness.Cache.PrepareBda([new GuestSpan(address, size)]); + }); Assert.False(harness.Cache.HasCpuDirtyPages(address, size)); Assert.Equal(expected, harness.ReadBack(buffer, offset, size)); harness.Shutdown(); } + [Fact] + public void DeviceAddressVisibilityPointRefreshesHotPagesWithoutAnotherWriteFault() + { + if (!GatePrerequisites.Ready(_vulkan)) return; + using var harness = new CacheHarness(_vulkan); + var address = harness.MapBacked(0x20000, ReadWrite); + const ulong size = 0x8000; + var (buffer, offset) = harness.Worker.Run(() => harness.Cache.ObtainBuffer(address, size, false)); + for (var write = 1; write <= 3; write++) + { + Assert.True(harness.Store.MarkCpuWrite(address, size)); + harness.Write(address, Pattern((int)size, (byte)write)); + harness.Worker.Run(() => + { + harness.Cache.NoteMemoryVisibilityPoint(); + harness.Cache.PrepareBda([new GuestSpan(address, size)]); + }); + } + + Assert.True(harness.Cache.HasCpuDirtyPages(address, size)); + var expected = Pattern((int)size, 9); + // A permanently writable hot page changes without another protection fault. + harness.Write(address, expected); + harness.Worker.Run(() => + { + harness.Cache.NoteMemoryVisibilityPoint(); + harness.Cache.PrepareBda([new GuestSpan(address, size)]); + }); + Assert.Equal(expected, harness.ReadBack(buffer, offset, size)); + harness.Shutdown(); + } + [Theory] [InlineData(false)] [InlineData(true)] diff --git a/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/CommandStreamTestSupport.cs b/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/CommandStreamTestSupport.cs index 85814a3..9761146 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/CommandStreamTestSupport.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/CommandStreamTestSupport.cs @@ -31,6 +31,11 @@ internal sealed class RecordingCommandStreamHost : ICommandStreamHost public List EndOfPipeWrites { get; } = new(); + // Satisfied waits and command-processor memory writes reported to the host. + public int VisibilityPoints { get; private set; } + + public void NoteMemoryVisibilityPoint() => VisibilityPoints++; + public List GuestReads { get; } = new(); public Action? BeforeGuestRead { get; set; } diff --git a/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/GpuCommandInterpreterLoopTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/GpuCommandInterpreterLoopTests.cs index 4c93b35..c74ce4f 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/GpuCommandInterpreterLoopTests.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/GpuCommandInterpreterLoopTests.cs @@ -261,6 +261,30 @@ public sealed class GpuCommandInterpreterLoopTests Assert.Contains(Label, runner.Host.GuestReads); } + [Fact] + public void Wait_IsAMemoryVisibilityPointOnlyOnceSatisfied() + { + var runner = new StreamRunner(); + + Assert.Equal(SubmissionProgress.Blocked, runner.Run(WaitEqual(Label, 1))); + Assert.Equal(0, runner.Host.VisibilityPoints); + + runner.Host.WriteDword(Label, 1); + Assert.Equal(SubmissionProgress.Complete, runner.Run()); + Assert.Equal(1, runner.Host.VisibilityPoints); + } + + [Fact] + public void WriteData_IsAMemoryVisibilityPoint() + { + var runner = new StreamRunner(); + + runner.Run(WriteData(Label, 7)); + + Assert.Equal(7u, runner.Host.ReadDword(Label)); + Assert.Equal(1, runner.Host.VisibilityPoints); + } + [Fact] public void PendingCommands_RunBeforeEveryPacket() {