diff --git a/src/SharpEmu.Libs/Gpu/Buffers/GuestBufferCache.cs b/src/SharpEmu.Libs/Gpu/Buffers/GuestBufferCache.cs index 183f607..c84c5f1 100644 --- a/src/SharpEmu.Libs/Gpu/Buffers/GuestBufferCache.cs +++ b/src/SharpEmu.Libs/Gpu/Buffers/GuestBufferCache.cs @@ -896,7 +896,7 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable $"unsupported buffer readback from an asynchronous GPU completion, addr=0x{guestAddress:X16} size=0x{size:X16}"); } - if (!_relay.IsGpuQueueThread && AsyncReadback is { } readback && GuestReadsAwaitOffQueue) + if (!_relay.IsGpuQueueThread && GuestReadsAwaitOffQueue && (AsyncReadback is not null || MainQueuePendingReads)) { PendingDownload? pending = null; if (_relay.TryRunOnGpuQueue(() => pending = BeginReadMemoryOnGpu(guestAddress, size, isWrite, source, allowPending: true))) @@ -906,7 +906,15 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable return true; } - readback.Wait(pending.Ticket); + if (pending.Ticket is { } ticket) + { + (AsyncReadback ?? throw SubmissionScheduler.Fatal("The pending readback lost its queue.")).Wait(ticket); + } + else + { + _scheduler.WaitForSubmittedTick(pending.MainQueueTick); + } + if (_relay.TryRunOnGpuQueue(() => CompleteReadMemoryOnGpu(pending, retrySynchronously: true))) { return true; @@ -1144,9 +1152,14 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable public static string TakeAsyncReadbackReport() => FormattableString.Invariant( $"[PERF][ASYNC_READBACK] applied={Interlocked.Exchange(ref _reportedApplied, 0)} retried={Interlocked.Exchange(ref _reportedRetried, 0)} eager_started={Interlocked.Exchange(ref _reportedEagerStarted, 0)} eager_used={Interlocked.Exchange(ref _reportedEagerUsed, 0)} eager_stale={Interlocked.Exchange(ref _reportedEagerStale, 0)}"); + // Ticket is set when the readback queue carries the download; otherwise the main queue + // does, into MainQueueBuffer, and MainQueueTick is the submission the guest thread waits for. private sealed record PendingDownload { - public required VulkanAsyncReadback.Ticket Ticket { get; init; } + public VulkanAsyncReadback.Ticket? Ticket { get; init; } + public ulong MainQueueTick { get; init; } + public GpuBuffer? MainQueueBuffer { get; init; } + public ulong[] MainQueueOffsets { get; init; } = []; public required List Copies { get; init; } public required ulong[] WriteTicks { get; init; } public required ulong WindowBegin { get; init; } @@ -1201,6 +1214,31 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable }; } + if (copies.Count != 0 && allowPending && + TryBeginDownloadOnMainQueue(copies, out var mainTick, out var mainBuffer, out var mainOffsets, out var mainWriteTicks)) + { + foreach (var copy in copies) + { + copy.Buffer.RetainForeignRead(); + } + + return new PendingDownload + { + MainQueueTick = mainTick, + MainQueueBuffer = mainBuffer, + MainQueueOffsets = mainOffsets, + Copies = copies, + WriteTicks = mainWriteTicks, + WindowBegin = windowBegin, + WindowEnd = windowEnd, + GuestAddress = guestAddress, + Size = size, + IsWrite = isWrite, + Source = source, + Started = readbackStarted, + }; + } + if (copies.Count != 0) { DownloadBufferMemory(copies); @@ -1216,9 +1254,67 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable return null; } + // Without a readback queue the main queue copies the pieces into a buffer of their own and + // submits at once; the guest thread then waits for that submission instead of the worker + // waiting for the whole GPU (a guest read of GPU-written memory otherwise idled the worker + // for tens of milliseconds, several times per frame in Silent Hill: The Short Message). + private bool TryBeginDownloadOnMainQueue(List copies, out ulong tick, out GpuBuffer buffer, out ulong[] offsets, out ulong[] writeTicks) + { + tick = 0; + buffer = null!; + offsets = []; + writeTicks = []; + if (!MainQueuePendingReads || copies.Count == 0) + { + return false; + } + + var ticks = new ulong[copies.Count]; + var placements = new ulong[copies.Count]; + var total = 0UL; + for (var index = 0; index < copies.Count; index++) + { + var written = copies[index].Buffer.LastGpuWriteTick; + if (written == 0) + { + return false; + } + + ticks[index] = written; + placements[index] = total; + total += (copies[index].Size + BufferDownloadBatchPlanner.Alignment - 1) & ~(BufferDownloadBatchPlanner.Alignment - 1); + } + + if (total > AsyncReadbackLimit) + { + return false; + } + + buffer = new GpuBuffer(_device, _scheduler, GpuBufferUsage.Download, 0, BufferUsageFlags.TransferDstBit, total); + for (var index = 0; index < copies.Count; index++) + { + buffer.CopyFrom( + _scheduler.Current, copies[index].Buffer, copies[index].SourceOffset, placements[index], copies[index].Size, + AccessFlags.MemoryWriteBit, AccessFlags.None, AccessFlags.MemoryReadBit | AccessFlags.MemoryWriteBit, AccessFlags.HostReadBit); + } + + tick = _scheduler.Flush(); + offsets = placements; + writeTicks = ticks; + return true; + } + + private static readonly bool MainQueuePendingReads = + Environment.GetEnvironmentVariable("SHARPEMU_MAIN_QUEUE_PENDING_READS") != "0"; + private bool CompleteReadMemoryOnGpu(PendingDownload pending, bool retrySynchronously) { using var readbackScope = GuestMemoryProfile.Measure(GuestMemoryProfile.Operation.BufferReadback); + if (pending.Ticket is null) + { + return CompleteMainQueueRead(pending, retrySynchronously); + } + var readback = AsyncReadback ?? throw SubmissionScheduler.Fatal("The pending readback lost its queue."); try { @@ -1269,6 +1365,58 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable } } + private bool CompleteMainQueueRead(PendingDownload pending, bool retrySynchronously) + { + var buffer = pending.MainQueueBuffer ?? throw SubmissionScheduler.Fatal("The pending readback lost its buffer."); + try + { + if (!IsPendingDownloadCurrent(pending)) + { + if (retrySynchronously) + { + PendingReadbacksRetried++; + Interlocked.Increment(ref _reportedRetried); + ReadMemoryOnGpu(pending.GuestAddress, pending.Size, pending.IsWrite, pending.Source); + } + + return false; + } + + PendingReadbacksApplied++; + Interlocked.Increment(ref _reportedApplied); + var copies = pending.Copies; + buffer.Invalidate(0, buffer.Size); + for (var index = 0; index < copies.Count; index++) + { + var bytes = buffer.Mapped.Slice((int)pending.MainQueueOffsets[index], (int)copies[index].Size); + if (!_backing.TryWriteBacking(copies[index].Address, bytes)) + { + throw SubmissionScheduler.Fatal($"Could not write the required direct backing: addr=0x{copies[index].Address:X16} size=0x{copies[index].Size:X16}"); + } + + _gpuModifiedRanges.Remove(copies[index].Address, copies[index].Size); + } + + _tracker.ClearGpuDirtyPages(pending.WindowBegin, pending.WindowEnd - pending.WindowBegin); + if (pending.IsWrite) + { + _tracker.MarkCpuDirtyPages(pending.GuestAddress, pending.Size); + } + + RecordReadback(pending.WindowBegin, pending.WindowEnd, pending.IsWrite, copies, pending.Started, pending.Source); + return true; + } + finally + { + // The guest thread waited for the submission, so the GPU is done with the buffer. + buffer.Dispose(); + foreach (var copy in pending.Copies) + { + copy.Buffer.ReleaseForeignRead(); + } + } + } + private bool IsPendingDownloadCurrent(PendingDownload pending) { var total = 0UL; diff --git a/src/SharpEmu.Libs/Gpu/Scheduling/SubmissionScheduler.cs b/src/SharpEmu.Libs/Gpu/Scheduling/SubmissionScheduler.cs index 7999a4d..d7e6a8e 100644 --- a/src/SharpEmu.Libs/Gpu/Scheduling/SubmissionScheduler.cs +++ b/src/SharpEmu.Libs/Gpu/Scheduling/SubmissionScheduler.cs @@ -272,6 +272,18 @@ public sealed class SubmissionScheduler : IGpuTickScheduler, IDisposable _lastMemoryDrainTick = nextTick; } + // Waits for a tick that has already been submitted; safe from any thread, since it only + // waits on the timeline and records nothing. + public void WaitForSubmittedTick(ulong tick) + { + if (tick >= CurrentTick) + { + throw Fatal($"Cannot wait off the queue for a tick that is still recording: {tick} >= {CurrentTick}."); + } + + _timeline.Wait(tick); + } + public void Wait(ulong tick) { if (tick > CurrentTick) diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Buffers/GuestBufferAsyncReadbackTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Buffers/GuestBufferAsyncReadbackTests.cs index 015ec32..52aae00 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Buffers/GuestBufferAsyncReadbackTests.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Buffers/GuestBufferAsyncReadbackTests.cs @@ -48,6 +48,33 @@ public sealed class GuestBufferAsyncReadbackTests(HeadlessVulkanFixture fixture) harness.Shutdown(); } + // Without a readback queue the main queue carries the download and the guest thread waits + // for that submission; the worker records the copies and moves on. + [Fact] + public void AGuestReadWithoutAReadbackQueueStillWaitsOffTheQueueThread() + { + if (_vulkan is null || !GatePrerequisites.Ready(_vulkan)) return; + using var harness = new CacheHarness(_vulkan); + using var fatal = new FatalScope(); + Assert.Null(harness.Cache.AsyncReadback); + var address = harness.MapBacked(0x10000, ReadWrite); + GpuWrite(harness, address, 0x55667788); + Assert.True(harness.Cache.HasGpuDirtyBytes(address, 0x100)); + + Assert.True(harness.Cache.DownloadToCpu(address + 8, 4)); + + Assert.False(harness.Cache.HasGpuDirtyBytes(address, 0x100)); + Assert.Equal((1L, 0L), (harness.Cache.PendingReadbacksApplied, harness.Cache.PendingReadbacksRetried)); + Assert.Equal(0x55667788u, BitConverter.ToUInt32(harness.Read(address + 0xF0, 4))); + + // A second write after the first read is picked up by the next read, not served stale. + GpuWrite(harness, address, 0x99AABBCCu); + Assert.True(harness.Cache.DownloadToCpu(address, 4)); + Assert.Equal(0x99AABBCCu, BitConverter.ToUInt32(harness.Read(address + 0x40, 4))); + Assert.Equal((2L, 0L), (harness.Cache.PendingReadbacksApplied, harness.Cache.PendingReadbacksRetried)); + harness.Shutdown(); + } + private static void GpuWrite(CacheHarness harness, ulong address, uint value) => harness.Worker.Run(() => { var (buffer, offset) = harness.Cache.ObtainBuffer(address, 0x100, isWritten: true);