diff --git a/src/SharpEmu.Libs/Gpu/Buffers/GuestBufferCache.cs b/src/SharpEmu.Libs/Gpu/Buffers/GuestBufferCache.cs index 33a6253..76c9d1e 100644 --- a/src/SharpEmu.Libs/Gpu/Buffers/GuestBufferCache.cs +++ b/src/SharpEmu.Libs/Gpu/Buffers/GuestBufferCache.cs @@ -354,6 +354,27 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable // Registered buffers are ordered and disjoint: walk only the ones the write overlaps. var end = guestAddress + (ulong)data.Length; + if (DeferHostWrites) + { + // When no GPU-written byte shares the written pages, the pages can simply turn + // CPU-dirty: every later GPU reader synchronizes CPU-dirty pages first (bound buffers + // on obtain, device-address programs at the next visibility point), so the bytes + // arrive in command order without a staging copy that ends the rendering scope. + var firstPage = guestAddress & ~(TrackerLayout.PageBytes - 1); + var pagesEnd = (end + TrackerLayout.PageBytes - 1) & ~(TrackerLayout.PageBytes - 1); + if (!_gpuModifiedRanges.Overlaps(firstPage, pagesEnd - firstPage) && + !_tracker.HasGpuDirtyPages(firstPage, pagesEnd - firstPage)) + { + if (IsRegionRegistered(guestAddress, (ulong)data.Length)) + { + _tracker.MarkCpuDirtyPages(guestAddress, (ulong)data.Length); + NoteMemoryVisibilityPoint(); + } + + return; + } + } + for (var index = _registry.FindFirstOverlappingIndex(guestAddress); index < _registry.RegisteredCount && _registry.GetRegisteredAddress(index) < end; index++) @@ -373,6 +394,9 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable } } + private static readonly bool DeferHostWrites = + Environment.GetEnvironmentVariable("SHARPEMU_DEFER_HOST_WRITES") != "0"; + public void FillBuffer(ulong guestAddress, ulong size, uint value, bool isGds) { if ((guestAddress & 3) != 0 || size == 0 || (size & 3) != 0 || size > ulong.MaxValue - guestAddress) @@ -679,13 +703,28 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable _bdaTouchMapping = mapping; } - var epoch = _tracker.CpuDirtyEpoch; - if (epoch != _bdaSweepEpoch || mapping != _bdaSweepMapping) + // Device-address programs may read any mapped page, so CPU writes are swept into the + // buffers - but only at memory visibility points (a submission slice starting, a wait + // packet satisfied). A CPU write after the submission began carries no ordering + // guarantee for its commands, and sweeping on every draw would end the render pass + // for every page the guest streams while the stream is being translated. + // Hot pages remain writable, so later CPU writes need not advance CpuDirtyEpoch. + // Every visibility point must revisit their dirty ranges even with an unchanged epoch. + if (mapping != _bdaSweepMapping || _bdaVisibilityPending || SweepBdaOnEveryDraw) { - foreach (var span in spans) - _tracker.ForEachPossiblyCpuDirtyRange(span.Address, span.Size, _uploadDirtyBuffersInRange ??= UploadDirtyBuffersInRange); - _bdaSweepEpoch = epoch; + BeginUploadBatch(); + try + { + foreach (var span in spans) + _tracker.ForEachPossiblyCpuDirtyRange(span.Address, span.Size, _uploadDirtyBuffersInRange ??= UploadDirtyBuffersInRange); + } + finally + { + EndUploadBatch(); + } + _bdaSweepMapping = mapping; + _bdaVisibilityPending = false; } } @@ -696,7 +735,15 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable } private Action? _uploadDirtyBuffersInRange; - private long _bdaSweepEpoch = -1; + + private static readonly bool PreserveHotPagesInSweeps = + Environment.GetEnvironmentVariable("SHARPEMU_BDA_SWEEP_REPROTECT_HOT") != "1"; + private bool _bdaVisibilityPending = true; + + private static readonly bool SweepBdaOnEveryDraw = + Environment.GetEnvironmentVariable("SHARPEMU_BDA_SWEEP_EVERY_DRAW") == "1"; + + public void NoteMemoryVisibilityPoint() => _bdaVisibilityPending = true; private ulong _bdaSweepMapping; private void TouchBuffersInRange(ulong guestAddress, ulong size) @@ -717,8 +764,10 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable var buffer = _registry.GetBuffer(_registry.GetRegisteredIdentifier(index)); var start = Math.Max(buffer.CpuAddress, guestAddress); var finish = Math.Min(buffer.CpuAddress + buffer.Size, end); + // Hot pages stay writable and are copied again on the next sweep: a few hundred KiB of + // ring memory per sweep costs far less than a protection fault per page per frame. if (start < finish && _tracker.MayHaveCpuDirtyPages(start, finish - start)) - _ = SynchronizeBuffer(buffer, start, finish - start, false, false, preserveCpuWriteHotPages: false); + _ = SynchronizeBuffer(buffer, start, finish - start, false, false, preserveCpuWriteHotPages: PreserveHotPagesInSweeps); } } @@ -744,7 +793,7 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable // SynchronizeBuffer takes for this read-only call), and the block summary // answers that without a lock for the common all-clean case. if (_tracker.HasCpuDirtyPages(start, finish - start)) - _ = SynchronizeBuffer(buffer, start, finish - start, false, false, preserveCpuWriteHotPages: false); + _ = SynchronizeBuffer(buffer, start, finish - start, false, false, preserveCpuWriteHotPages: PreserveHotPagesInSweeps); } } } @@ -2059,6 +2108,17 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable command.EndRendering(); var native = new CommandBuffer(command.Handle); var vk = _device.Vk; + if (_uploadBatchActive) + { + OpenUploadBatch(native); + var batched = CollectionsMarshal.AsSpan(copies); + fixed (BufferCopy* pointer = batched) + { + vk.CmdCopyBuffer(native, source.Handle, buffer.Handle, (uint)batched.Length, pointer); + } + } + else + { var before = new BufferMemoryBarrier2 { SType = StructureType.BufferMemoryBarrier2, @@ -2085,6 +2145,7 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable VulkanSynchronization.PipelineBarrier(vk, native, PipelineStageFlags.TransferBit, PipelineStageFlags.AllCommandsBit, DependencyFlags.ByRegionBit, 0, null, 1, &after, 0, null); + } } if (BufferUploadProfile.Enabled) @@ -2110,6 +2171,59 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable return false; } + // A sweep uploads many buffers back to back: one global barrier before the first copy and one + // after the last replace a barrier pair per buffer, so the copies share one transfer pass. + private bool _uploadBatchActive; + private ulong _uploadBatchCommand; + + private void BeginUploadBatch() + { + _uploadBatchActive = true; + _uploadBatchCommand = 0; + } + + private void OpenUploadBatch(CommandBuffer native) + { + if (_uploadBatchCommand == (ulong)native.Handle) + { + return; + } + + // A new command buffer (or the first copy) orders the copies after all earlier work. + var barrier = new MemoryBarrier2 + { + SType = StructureType.MemoryBarrier2, + SrcAccessMask = AccessFlags2.MemoryReadBit | AccessFlags2.MemoryWriteBit, + DstAccessMask = AccessFlags2.TransferWriteBit, + }; + VulkanSynchronization.PipelineBarrier(_device.Vk, + native, PipelineStageFlags.AllCommandsBit, PipelineStageFlags.TransferBit, 0, 1, &barrier, 0, null, 0, null); + _uploadBatchCommand = (ulong)native.Handle; + } + + private void EndUploadBatch() + { + _uploadBatchActive = false; + if (_uploadBatchCommand == 0) + { + return; + } + + // Recorded on the current command buffer; submission order covers copies recorded on an + // earlier one. + var command = _scheduler.Current; + command.EndRendering(); + var barrier = new MemoryBarrier2 + { + SType = StructureType.MemoryBarrier2, + SrcAccessMask = AccessFlags2.TransferWriteBit, + DstAccessMask = AccessFlags2.MemoryReadBit | AccessFlags2.MemoryWriteBit, + }; + VulkanSynchronization.PipelineBarrier(_device.Vk, + new CommandBuffer(command.Handle), PipelineStageFlags.TransferBit, PipelineStageFlags.AllCommandsBit, 0, 1, &barrier, 0, null, 0, null); + _uploadBatchCommand = 0; + } + // The copies of one synchronization; the tracker forbids nesting, so one list per thread is reused. [ThreadStatic] private static List? _syncCopies; diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Waits.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Waits.cs index 7e6f8fd..0883dd6 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Waits.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Waits.cs @@ -42,7 +42,10 @@ public sealed partial class GpuCommandInterpreter if (!satisfied) { Suspend(); + return; } + + _host.NoteMemoryVisibilityPoint(); } // The wait bit is PredicationZPassWaitOp: 0 (kWaitForQueryResults) stalls the command diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.cs index ef91e56..d5c49e7 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.cs @@ -443,6 +443,8 @@ public sealed partial class GpuCommandInterpreter { throw _host.Fatal($"The command stream cannot write guest memory: address=0x{address:X16} size={source.Length}."); } + // Later commands in this stream may read the bytes through device addresses. + _host.NoteMemoryVisibilityPoint(); _lastWriteLength = 0; if (source.Length is sizeof(uint) or sizeof(ulong)) { diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/ICommandStreamHost.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/ICommandStreamHost.cs index cdc9461..cbcae94 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/ICommandStreamHost.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/ICommandStreamHost.cs @@ -107,6 +107,13 @@ public interface ICommandStreamHost void CopyBuffer(ulong destination, ulong source, ulong size, bool destinationIsGds, bool sourceIsGds); + // A satisfied wait packet or a command-processor memory write: CPU writes released by the + // waited-on value, or the written bytes, must become visible to the commands that follow, + // the same as at the start of a submission slice. + void NoteMemoryVisibilityPoint() + { + } + void ReadGds(Span destination, uint wordOffset, uint wordCount); void RecordEndOfPipe(in EndOfPipeWrite write); diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CommandStream.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CommandStream.cs index e0eb2df..ac7532c 100644 --- a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CommandStream.cs +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CommandStream.cs @@ -316,9 +316,12 @@ internal static unsafe partial class VulkanVideoPresenter _activeGuestQueue = new VulkanGuestQueueIdentity(_commandQueueNames[queueId], submissionId); BindSubmissionContext(_activeGuestQueue); _ = CurrentRecordingBuffer(); + _bufferCache.NoteMemoryVisibilityPoint(); _translation.BeginSubmission(queueId, submissionId, geometrySnapshots, _commandStream.GetInterpreter(queueId)); } + public void NoteMemoryVisibilityPoint() => _bufferCache.NoteMemoryVisibilityPoint(); + public void Flush() { using var flushScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.Flush); diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Buffers/GuestBufferCacheTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Buffers/GuestBufferCacheTests.cs index 6e2617c..632eeaa 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Buffers/GuestBufferCacheTests.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Buffers/GuestBufferCacheTests.cs @@ -674,7 +674,12 @@ public sealed class GuestBufferCacheTests : IClassFixture var address = harness.MapBacked(0x10000, ReadWrite); var (buffer, _) = harness.Worker.Run(() => harness.Cache.ObtainBuffer(address, 0x8000, isWritten: false)); - harness.Worker.Run(() => harness.Cache.WriteHostMemory(address + 0x1000, Pattern(0x100, 7))); + harness.Worker.Run(() => + { + harness.Cache.WriteHostMemory(address + 0x1000, Pattern(0x100, 7)); + // Deferred host writes become visible when a command acquires its buffer. + harness.Cache.PrepareBda([new GuestSpan(address, 0x10000)]); + }); Assert.Equal(Pattern(0x100, 7), harness.Read(address + 0x1000, 0x100)); Assert.Equal(Pattern(0x100, 7), harness.ReadBack(buffer, 0x1000, 0x100)); @@ -801,8 +806,8 @@ public sealed class GuestBufferCacheTests : IClassFixture harness.Write(address, expected); harness.Worker.Run(() => { + harness.Cache.NoteMemoryVisibilityPoint(); harness.Cache.PrepareBda([new GuestSpan(address, size)]); - Assert.False(harness.Cache.HasCpuDirtyPages(address, size)); Assert.False(harness.Cache.HasGpuDirtyPages(address, size)); harness.Cache.PrepareBda([new GuestSpan(address, size)]); }); @@ -835,7 +840,11 @@ public sealed class GuestBufferCacheTests : IClassFixture }); Assert.Equal(Bytes(0x12345678u), harness.ReadBack(buffer, offset, 4)); Assert.True(harness.Cache.TrySynchronizeCpuRead(address, size)); - harness.Worker.Run(() => harness.Cache.PrepareBda([new GuestSpan(address, size)])); + harness.Worker.Run(() => + { + harness.Cache.NoteMemoryVisibilityPoint(); + harness.Cache.PrepareBda([new GuestSpan(address, size)]); + }); var expected = Pattern((int)size, 7); await Task.Run(() => @@ -843,12 +852,48 @@ public sealed class GuestBufferCacheTests : IClassFixture Assert.True(harness.Store.MarkCpuWrite(address, size)); harness.Write(address, expected); }); - harness.Worker.Run(() => harness.Cache.PrepareBda([new GuestSpan(address, size)])); + harness.Worker.Run(() => + { + harness.Cache.NoteMemoryVisibilityPoint(); + harness.Cache.PrepareBda([new GuestSpan(address, size)]); + }); Assert.False(harness.Cache.HasCpuDirtyPages(address, size)); Assert.Equal(expected, harness.ReadBack(buffer, offset, size)); harness.Shutdown(); } + [Fact] + public void DeviceAddressVisibilityPointRefreshesHotPagesWithoutAnotherWriteFault() + { + if (!GatePrerequisites.Ready(_vulkan)) return; + using var harness = new CacheHarness(_vulkan); + var address = harness.MapBacked(0x20000, ReadWrite); + const ulong size = 0x8000; + var (buffer, offset) = harness.Worker.Run(() => harness.Cache.ObtainBuffer(address, size, false)); + for (var write = 1; write <= 3; write++) + { + Assert.True(harness.Store.MarkCpuWrite(address, size)); + harness.Write(address, Pattern((int)size, (byte)write)); + harness.Worker.Run(() => + { + harness.Cache.NoteMemoryVisibilityPoint(); + harness.Cache.PrepareBda([new GuestSpan(address, size)]); + }); + } + + Assert.True(harness.Cache.HasCpuDirtyPages(address, size)); + var expected = Pattern((int)size, 9); + // A permanently writable hot page changes without another protection fault. + harness.Write(address, expected); + harness.Worker.Run(() => + { + harness.Cache.NoteMemoryVisibilityPoint(); + harness.Cache.PrepareBda([new GuestSpan(address, size)]); + }); + Assert.Equal(expected, harness.ReadBack(buffer, offset, size)); + harness.Shutdown(); + } + [Theory] [InlineData(false)] [InlineData(true)] diff --git a/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/CommandStreamTestSupport.cs b/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/CommandStreamTestSupport.cs index 85814a3..9761146 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/CommandStreamTestSupport.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/CommandStreamTestSupport.cs @@ -31,6 +31,11 @@ internal sealed class RecordingCommandStreamHost : ICommandStreamHost public List EndOfPipeWrites { get; } = new(); + // Satisfied waits and command-processor memory writes reported to the host. + public int VisibilityPoints { get; private set; } + + public void NoteMemoryVisibilityPoint() => VisibilityPoints++; + public List GuestReads { get; } = new(); public Action? BeforeGuestRead { get; set; } diff --git a/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/GpuCommandInterpreterLoopTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/GpuCommandInterpreterLoopTests.cs index 4c93b35..c74ce4f 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/GpuCommandInterpreterLoopTests.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/GpuCommandInterpreterLoopTests.cs @@ -261,6 +261,30 @@ public sealed class GpuCommandInterpreterLoopTests Assert.Contains(Label, runner.Host.GuestReads); } + [Fact] + public void Wait_IsAMemoryVisibilityPointOnlyOnceSatisfied() + { + var runner = new StreamRunner(); + + Assert.Equal(SubmissionProgress.Blocked, runner.Run(WaitEqual(Label, 1))); + Assert.Equal(0, runner.Host.VisibilityPoints); + + runner.Host.WriteDword(Label, 1); + Assert.Equal(SubmissionProgress.Complete, runner.Run()); + Assert.Equal(1, runner.Host.VisibilityPoints); + } + + [Fact] + public void WriteData_IsAMemoryVisibilityPoint() + { + var runner = new StreamRunner(); + + runner.Run(WriteData(Label, 7)); + + Assert.Equal(7u, runner.Host.ReadDword(Label)); + Assert.Equal(1, runner.Host.VisibilityPoints); + } + [Fact] public void PendingCommands_RunBeforeEveryPacket() {