Revert "Fix/des render thread regression (#1086)" (#1091)

This reverts commit 28c922c7a5.

src/SharpEmu.Libs/Gpu/Buffers/GpuBuffer.cs

@@ -362,6 +362,7 @@ public unsafe class GpuBuffer : IDisposable
         var (required, preferred, avoided) = usage switch
         {
             GpuBufferUsage.DeviceLocal => (MemoryPropertyFlags.None, MemoryPropertyFlags.DeviceLocalBit, MemoryPropertyFlags.None),
+            GpuBufferUsage.Unified => (MemoryPropertyFlags.HostVisibleBit | MemoryPropertyFlags.HostCoherentBit | MemoryPropertyFlags.DeviceLocalBit, MemoryPropertyFlags.HostCachedBit, MemoryPropertyFlags.None),
             GpuBufferUsage.Upload => (MemoryPropertyFlags.HostVisibleBit, MemoryPropertyFlags.HostCoherentBit, MemoryPropertyFlags.DeviceLocalBit),
             GpuBufferUsage.Download => (MemoryPropertyFlags.HostVisibleBit, MemoryPropertyFlags.HostCoherentBit | MemoryPropertyFlags.HostCachedBit, MemoryPropertyFlags.DeviceLocalBit),
             _ => (MemoryPropertyFlags.HostVisibleBit, MemoryPropertyFlags.HostCoherentBit | MemoryPropertyFlags.DeviceLocalBit, MemoryPropertyFlags.None),

src/SharpEmu.Libs/Gpu/Buffers/GpuBufferUsage.cs

@@ -9,4 +9,6 @@ public enum GpuBufferUsage : byte
     Upload,
     Download,
     Stream,
+    // Device-local memory the host can map: guest buffers on unified-memory devices, read back without a copy.
+    Unified,
 }

src/SharpEmu.Libs/Gpu/Buffers/GuestBufferCache.cs

@@ -41,6 +41,7 @@ public sealed unsafe partial class GuestBufferCache : IGuestBufferStore, IDispos
     private readonly SubmissionScheduler _scheduler;
     private readonly IGpuQueueRelay _relay;
     private readonly GuestBufferUploader _uploader;
+    private readonly bool _unifiedBuffers;
     private readonly IGuestBackedSpace _backing;
     private readonly ICpuMemory _guest;
 
@@ -78,6 +79,7 @@ public sealed unsafe partial class GuestBufferCache : IGuestBufferStore, IDispos
         _backing = backing;
         _guest = guest;
         _faults = new BdaFaultProcessor(device, scheduler, this, CachingPageBits, CachingPageCount);
+        _unifiedBuffers = UnifiedBuffersEnabled && HasUnifiedMemoryType(device);
         _gds = new GpuBuffer(device, scheduler, GpuBufferUsage.Stream, 0, GpuBuffer.AllFlags, GdsBufferSize);
         _bdaPageTable = new GpuBuffer(device, scheduler, GpuBufferUsage.DeviceLocal, 0, GpuBuffer.AllFlags, BdaPageTableSize);
         _tracker = new GuestPageTracker(pages);
@@ -1619,6 +1621,12 @@ public sealed unsafe partial class GuestBufferCache : IGuestBufferStore, IDispos
     }
 
     private const ulong ReadbackWindowBytes = 512 * 1024;
+    // Uncached reads of device-local memory run near 300 MB/s, and most mapped reads want one
+    // descriptor word or packet dword: one tracker page is the smallest window that can turn clean.
+    // SHARPEMU_MAPPED_READBACK_KB overrides it (a power of two) for comparisons.
+    private static readonly ulong MappedReadbackWindowBytes =
+        ulong.TryParse(Environment.GetEnvironmentVariable("SHARPEMU_MAPPED_READBACK_KB"), out var mappedKb) && mappedKb != 0 &&
+        (mappedKb & (mappedKb - 1)) == 0 ? mappedKb * 1024 : TrackerLayout.PageBytes;
     private const long HotWindowLifetime = 512;
     private const int MaxEagerReadbacks = 4;
     private readonly Dictionary<ulong, HotWindow> _hotWindows = new();
@@ -1874,6 +1882,9 @@ public sealed unsafe partial class GuestBufferCache : IGuestBufferStore, IDispos
         public required bool IsWrite { get; init; }
         public required GuestMemoryProfile.ReadbackSource Source { get; init; }
         public required long Started { get; init; }
+
+        // Read from the buffers' own host mappings once the writer tick retires; no copy was recorded.
+        public bool Mapped { get; init; }
     }
 
     private PendingDownload? BeginReadMemoryOnGpu(ulong guestAddress, ulong size, bool isWrite, GuestMemoryProfile.ReadbackSource source,
@@ -1897,6 +1908,56 @@ public sealed unsafe partial class GuestBufferCache : IGuestBufferStore, IDispos
         {
             NoteHotReadback(guestAddress, size);
         }
+        if (copies.Count != 0 && _unifiedBuffers && AreMapped(copies, out var writer))
+        {
+            // The mapping is device-local memory the CPU reads uncached across the bus, so a
+            // mapped read copies only the pages around the request, not the whole window.
+            copies = CollectReadbackWindow(copies[0].Buffer, guestAddress, size, MappedReadbackWindowBytes, out windowBegin, out windowEnd);
+            if (writer == 0 || _scheduler.IsTickComplete(writer))
+            {
+                ApplyMappedRead(copies, windowBegin, windowEnd, guestAddress, size, isWrite, readbackStarted, source);
+                return null;
+            }
+
+            if (allowPending && writer < _scheduler.CurrentTick)
+            {
+                // A guest thread had to wait for this writer: submit later writes to the window right
+                // after their command, so the next wait ends with them instead of with their whole batch.
+                NoteHotReadback(guestAddress, size);
+
+                // The writer is already submitted: wait for it alone, not for the work queued after it.
+                var mappedTicks = new ulong[copies.Count];
+                for (var index = 0; index < copies.Count; index++)
+                {
+                    mappedTicks[index] = copies[index].Buffer.LastGpuWriteTick;
+                    copies[index].Buffer.RetainForeignRead();
+                }
+
+                return new PendingDownload
+                {
+                    Mapped = true,
+                    MainQueueTick = writer,
+                    Copies = copies,
+                    WriteTicks = mappedTicks,
+                    WindowBegin = windowBegin,
+                    WindowEnd = windowEnd,
+                    GuestAddress = guestAddress,
+                    Size = size,
+                    IsWrite = isWrite,
+                    Source = source,
+                    Started = readbackStarted,
+                };
+            }
+
+            if (!allowPending)
+            {
+                // On the command worker: submit the writer if it is still recording, then read the mapping.
+                _scheduler.Wait(writer);
+                ApplyMappedRead(copies, windowBegin, windowEnd, guestAddress, size, isWrite, readbackStarted, source);
+                return null;
+            }
+        }
+
         if (copies.Count != 0 && allowPending && TryBeginDownloadAsync(copies, out var ticket, out var writeTicks))
         {
             foreach (var copy in copies)
@@ -2018,6 +2079,11 @@ public sealed unsafe partial class GuestBufferCache : IGuestBufferStore, IDispos
     private bool CompleteReadMemoryOnGpu(PendingDownload pending, bool retrySynchronously)
     {
         using var readbackScope = GuestMemoryProfile.Measure(GuestMemoryProfile.Operation.BufferReadback);
+        if (pending.Mapped)
+        {
+            return CompleteMappedRead(pending, retrySynchronously);
+        }
+
         if (pending.Ticket is null)
         {
             return CompleteMainQueueRead(pending, retrySynchronously);
@@ -2073,6 +2139,103 @@ public sealed unsafe partial class GuestBufferCache : IGuestBufferStore, IDispos
         }
     }
 
+    // Guest buffers live in host-visible device memory on unified-memory devices (Apple GPUs), so a
+    // readback can copy straight out of the buffer once the last GPU write to it retired. That skips
+    // the copy command at the tail of the queue and the wait for every draw recorded after the writer.
+    // SHARPEMU_UNIFIED_BUFFERS=0 keeps device-local buffers and copy readbacks.
+    private static readonly bool UnifiedBuffersEnabled =
+        Environment.GetEnvironmentVariable("SHARPEMU_UNIFIED_BUFFERS") != "0";
+
+    private static bool HasUnifiedMemoryType(GpuDeviceInfo device)
+    {
+        const MemoryPropertyFlags unified = MemoryPropertyFlags.DeviceLocalBit | MemoryPropertyFlags.HostVisibleBit | MemoryPropertyFlags.HostCoherentBit;
+        for (uint index = 0; index < device.MemoryTypeCount; index++)
+        {
+            if ((device.GetMemoryTypeFlags(index) & unified) == unified)
+            {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    private static bool AreMapped(List<DownloadPiece> copies, out ulong writer)
+    {
+        writer = 0;
+        foreach (var copy in copies)
+        {
+            if (copy.Buffer.MappedPointer == null || !copy.Buffer.IsCoherent)
+            {
+                return false;
+            }
+
+            writer = Math.Max(writer, copy.Buffer.LastGpuWriteTick);
+        }
+
+        return true;
+    }
+
+    private void ApplyMappedRead(List<DownloadPiece> copies, ulong windowBegin, ulong windowEnd, ulong guestAddress, ulong size,
+        bool isWrite, long started, GuestMemoryProfile.ReadbackSource source)
+    {
+        foreach (var copy in copies)
+        {
+            var bytes = new ReadOnlySpan<byte>(copy.Buffer.MappedPointer + copy.SourceOffset, checked((int)copy.Size));
+            if (!WriteDownloaded(copy.Address, bytes))
+            {
+                throw SubmissionScheduler.Fatal($"Could not write the required direct backing: addr=0x{copy.Address:X16} size=0x{copy.Size:X16}");
+            }
+
+            _gpuModifiedRanges.Remove(copy.Address, copy.Size);
+        }
+
+        _tracker.ClearGpuDirtyPages(windowBegin, windowEnd - windowBegin);
+        if (isWrite)
+        {
+            _tracker.MarkCpuDirtyPages(guestAddress, size);
+        }
+
+        MappedReadbacks++;
+        RecordReadback(windowBegin, windowEnd, isWrite, copies, started, source);
+    }
+
+    // Readbacks served from a buffer's own mapping, without a copy command.
+    internal long MappedReadbacks { get; private set; }
+
+    internal bool UnifiedBuffers => _unifiedBuffers;
+
+    private bool CompleteMappedRead(PendingDownload pending, bool retrySynchronously)
+    {
+        try
+        {
+            if (!IsPendingDownloadCurrent(pending))
+            {
+                if (retrySynchronously)
+                {
+                    PendingReadbacksRetried++;
+                    Interlocked.Increment(ref _reportedRetried);
+                    ReadMemoryOnGpu(pending.GuestAddress, pending.Size, pending.IsWrite, pending.Source);
+                }
+
+                return false;
+            }
+
+            PendingReadbacksApplied++;
+            Interlocked.Increment(ref _reportedApplied);
+            ApplyMappedRead(pending.Copies, pending.WindowBegin, pending.WindowEnd, pending.GuestAddress, pending.Size,
+                pending.IsWrite, pending.Started, pending.Source);
+            return true;
+        }
+        finally
+        {
+            foreach (var copy in pending.Copies)
+            {
+                copy.Buffer.ReleaseForeignRead();
+            }
+        }
+    }
+
     private bool CompleteMainQueueRead(PendingDownload pending, bool retrySynchronously)
     {
         var buffer = pending.MainQueueBuffer ?? throw SubmissionScheduler.Fatal("The pending readback lost its buffer.");
@@ -2566,7 +2729,7 @@ public sealed unsafe partial class GuestBufferCache : IGuestBufferStore, IDispos
         }
 
         GpuBuffer CreateGpuBuffer() => new(
-            _device, _scheduler, GpuBufferUsage.DeviceLocal, overlap.Begin,
+            _device, _scheduler, _unifiedBuffers ? GpuBufferUsage.Unified : GpuBufferUsage.DeviceLocal, overlap.Begin,
             GpuBuffer.AllFlags | BufferUsageFlags.ShaderDeviceAddressBit, overlap.End - overlap.Begin, allowSlab: true);
         GpuBuffer created;
         try

src/SharpEmu.Libs/Gpu/Images/GpuTiler.cs

@@ -695,24 +695,8 @@ public sealed unsafe class GpuTiler : IDisposable
         var targetBase = tiledOffset & (StorageAlignment - 1);
         var dispatches = new List<TransferDispatch>();
         Prepare(true, tiledCapacity, linearCapacity, transfers, 0, targetBase, dispatches);
-        // The download comes first in the ring like an upload: a dropped image published every frame
-        // otherwise created and freed a pooled scratch the size of the image each time. A swapped
-        // download keeps its own scratch, so the swap output taken from the ring cannot wrap onto it.
-        var linear = swap == ColorChannelSwap.SwapBgra16
-            ? AllocateScratch((linearCapacity + 3) & ~3UL)
-            : AllocateUploadScratch(linearCapacity);
-        if (linear.Offset != 0)
-        {
-            var shifted = regions.ToArray();
-            for (var index = 0; index < shifted.Length; index++)
-            {
-                shifted[index].BufferOffset += linear.Offset;
-            }
-
-            regions = shifted;
-        }
-
-        image.DownloadToBuffer(regions, linear.Buffer, linear.Offset, linear.Size);
+        var linear = AllocateScratch((linearCapacity + 3) & ~3UL);
+        image.DownloadToBuffer(regions, linear.Buffer, 0, linear.Size);
         var source = linear;
         if (swap == ColorChannelSwap.SwapBgra16)
         {

tests/SharpEmu.Libs.Tests/Gpu/Buffers/GuestBufferAsyncReadbackTests.cs

@@ -75,6 +75,163 @@ public sealed class GuestBufferAsyncReadbackTests(HeadlessVulkanFixture fixture)
         harness.Shutdown();
     }
 
+    // On a unified-memory device a read of a retired GPU write copies straight out of the buffer's
+    // own mapping: no copy command, no pending readback.
+    [Fact]
+    public void AReadOfARetiredWriteComesFromTheUnifiedMapping()
+    {
+        if (_vulkan is null || !GatePrerequisites.Ready(_vulkan)) return;
+        using var harness = new CacheHarness(_vulkan);
+        using var fatal = new FatalScope();
+        if (!harness.Cache.UnifiedBuffers)
+        {
+            harness.Shutdown();
+            return;
+        }
+
+        var address = harness.MapBacked(0x10000, ReadWrite);
+        GpuWrite(harness, address, 0x0BADF00D);
+        harness.Worker.Run(() => harness.Scheduler.Wait(harness.Scheduler.Flush()));
+
+        Assert.True(harness.Cache.DownloadToCpu(address + 8, 4));
+
+        Assert.False(harness.Cache.HasGpuDirtyBytes(address, 0x100));
+        Assert.Equal(0x0BADF00Du, BitConverter.ToUInt32(harness.Read(address + 0x80, 4)));
+        Assert.Equal(1L, harness.Cache.MappedReadbacks);
+        Assert.Equal(0L, harness.Cache.PendingReadbacksApplied);
+        harness.Shutdown();
+    }
+
+    // A read while the writer is submitted but still running waits for that submission only, then
+    // reads the mapping; a write recorded after the read was issued is picked up by the next read.
+    [Fact]
+    public void AReadOfASubmittedWriteWaitsForItAndReadsTheUnifiedMapping()
+    {
+        if (_vulkan is null || !GatePrerequisites.Ready(_vulkan)) return;
+        using var harness = new CacheHarness(_vulkan);
+        using var fatal = new FatalScope();
+        if (!harness.Cache.UnifiedBuffers)
+        {
+            harness.Shutdown();
+            return;
+        }
+
+        var address = harness.MapBacked(0x10000, ReadWrite);
+        GpuWrite(harness, address, 0x12345678);
+        SubmitBatch(harness);
+
+        Assert.True(harness.Cache.DownloadToCpu(address, 4));
+
+        Assert.Equal(0x12345678u, BitConverter.ToUInt32(harness.Read(address + 0xFC, 4)));
+        Assert.Equal(1L, harness.Cache.MappedReadbacks);
+        Assert.False(harness.Cache.HasGpuDirtyBytes(address, 0x100));
+
+        GpuWrite(harness, address, 0x9ABCDEF0);
+        SubmitBatch(harness);
+        Assert.True(harness.Cache.DownloadToCpu(address, 4));
+        Assert.Equal(0x9ABCDEF0u, BitConverter.ToUInt32(harness.Read(address + 0x10, 4)));
+        harness.Shutdown();
+    }
+
+    // A shader store through a device address announces its page only after the fact. When the
+    // CPU dirtied the page meanwhile, the page stays CPU-owned: the shader's bytes reach guest
+    // memory by a merge, and the next upload copies only what the CPU changed, so neither the
+    // earlier nor the later shader bytes in the buffer are overwritten with stale guest bytes.
+    [Fact]
+    public void ShaderBytesOnAPageTheCpuAlsoWritesSurviveTheNextUpload()
+    {
+        if (_vulkan is null || !GatePrerequisites.Ready(_vulkan)) return;
+        using var harness = new CacheHarness(_vulkan);
+        using var fatal = new FatalScope();
+        if (!harness.Cache.UnifiedBuffers)
+        {
+            harness.Shutdown();
+            return;
+        }
+
+        var address = harness.MapBacked(0x10000, ReadWrite);
+        harness.Write(address, Enumerable.Repeat((byte)0x11, 0x1000).ToArray());
+        var (buffer, offset) = harness.Worker.Run(() => harness.Cache.ObtainBuffer(address, 0x1000, isWritten: false, requiresDeviceAddress: true));
+        void ShaderWrite(ulong at, uint value) => harness.Worker.Run(() =>
+        {
+            buffer.Fill(offset + at, 0x10, value);
+            harness.Scheduler.Wait(harness.Scheduler.Flush());
+        });
+
+        // The first shader write is reported on a clean page; the CPU write that follows downloads it.
+        ShaderWrite(0x100, 0x22222222);
+        harness.Worker.Run(() => harness.Cache.NoteDeviceAddressWrites(address, 0x1000));
+        Assert.True(harness.Store.MarkCpuWrite(address + 0x800, 4));
+        Assert.Equal(0x22222222u, BitConverter.ToUInt32(harness.Read(address + 0x100, 4)));
+        harness.Write(address + 0x800, [0xAA, 0xAA, 0xAA, 0xAA]);
+
+        // The second shader write is reported while the page is CPU-dirty, then the page uploads.
+        ShaderWrite(0x200, 0x33333333);
+        harness.Worker.Run(() => harness.Cache.NoteDeviceAddressWrites(address, 0x1000));
+        Assert.Equal(0x33333333u, BitConverter.ToUInt32(harness.Read(address + 0x200, 4)));
+        harness.Worker.Run(() =>
+        {
+            _ = harness.Cache.ObtainBuffer(address, 0x1000, isWritten: false, requiresDeviceAddress: true);
+            harness.Scheduler.Wait(harness.Scheduler.Flush());
+        });
+
+        var gpu = harness.ReadBack(buffer, offset, 0x1000);
+        Assert.Equal(0x22222222u, BitConverter.ToUInt32(gpu, 0x100));
+        Assert.Equal(0x33333333u, BitConverter.ToUInt32(gpu, 0x200));
+        Assert.Equal(0xAAAAAAAAu, BitConverter.ToUInt32(gpu, 0x800));
+        Assert.Equal(0x11111111u, BitConverter.ToUInt32(gpu, 0x900));
+        harness.Shutdown();
+    }
+
+    [Fact]
+    public void GrowingASharedBufferPreservesShaderBytesNotYetReported()
+    {
+        if (_vulkan is null || !GatePrerequisites.Ready(_vulkan)) return;
+        using var harness = new CacheHarness(_vulkan);
+        using var fatal = new FatalScope();
+        if (!harness.Cache.UnifiedBuffers)
+        {
+            harness.Shutdown();
+            return;
+        }
+
+        var address = harness.MapBacked(0x40000, ReadWrite);
+        harness.Write(address, Enumerable.Repeat((byte)0x11, 0x1000).ToArray());
+        var (old, offset) = harness.Worker.Run(() =>
+            harness.Cache.ObtainBuffer(address, 0x1000, isWritten: false, requiresDeviceAddress: true));
+        harness.Worker.Run(() =>
+        {
+            old.Fill(offset + 0x100, 0x10, 0x22222222);
+            harness.Scheduler.Wait(harness.Scheduler.Flush());
+            harness.Cache.NoteDeviceAddressWrites(address, 0x1000);
+        });
+        Assert.True(harness.Store.MarkCpuWrite(address + 0x800, 4));
+        harness.Write(address + 0x800, [0xAA, 0xAA, 0xAA, 0xAA]);
+        harness.Worker.Run(() =>
+        {
+            _ = harness.Cache.ObtainBuffer(address, 0x1000, isWritten: false, requiresDeviceAddress: true);
+            harness.Scheduler.Wait(harness.Scheduler.Flush());
+            // The device-address fault callback has not reported this new store yet.
+            old.Fill(offset + 0x200, 0x10, 0x33333333);
+            harness.Scheduler.Wait(harness.Scheduler.Flush());
+        });
+        Assert.Equal(0x11111111u, BitConverter.ToUInt32(harness.Read(address + 0x200, 4)));
+        Assert.True(harness.Store.MarkCpuWrite(address + 0x800, 4));
+        harness.Write(address + 0x800, [0xBB, 0xBB, 0xBB, 0xBB]);
+        var (grown, grownOffset) = harness.Worker.Run(() =>
+        {
+            var result = harness.Cache.ObtainBuffer(address, 0x20000, isWritten: false, requiresDeviceAddress: true);
+            harness.Scheduler.Wait(harness.Scheduler.Flush());
+            return result;
+        });
+        Assert.NotSame(old, grown);
+        var gpu = harness.ReadBack(grown, grownOffset, 0x1000);
+        Assert.Equal(0x22222222u, BitConverter.ToUInt32(gpu, 0x100));
+        Assert.Equal(0x33333333u, BitConverter.ToUInt32(gpu, 0x200));
+        Assert.Equal(0xBBBBBBBBu, BitConverter.ToUInt32(gpu, 0x800));
+        harness.Shutdown();
+    }
+
     private static void GpuWrite(CacheHarness harness, ulong address, uint value) => harness.Worker.Run(() =>
     {
         var (buffer, offset) = harness.Cache.ObtainBuffer(address, 0x100, isWritten: true);

tests/SharpEmu.Libs.Tests/Gpu/Images/GuestImageCacheTests.Readback.cs

@@ -374,9 +374,7 @@ public sealed partial class GuestImageCacheTests
     }
 
     [Fact]
-    // Tiled readbacks of dropped images download through the tiler's ring, so a collection under
-    // pressure holds no pooled scratch for them and need not end the tick to bound it.
-    public void Pressure_TiledReadbacksOfDroppedImagesHoldNoScratch()
+    public void Pressure_CompletesTiledReadbackBatchesBeforeAllocatingMoreScratch()
     {
         if (!GatePrerequisites.Ready(_vulkan)) return;
         using var harness = new CacheHarness(_vulkan);
@@ -402,11 +400,14 @@ public sealed partial class GuestImageCacheTests
             harness.Images.TilerForTest.OutstandingScratchBudget = 1;
             harness.Images.SetCollectionThresholds(0, 0, ulong.MaxValue, 2);
             harness.Images.ResetRecency(System.Runtime.InteropServices.CollectionsMarshal.AsSpan(images), 2);
+            var tick = harness.Scheduler.CurrentTick;
             harness.Images.RunGarbageCollector();
+            Assert.True(harness.Scheduler.CurrentTick > tick);
             Assert.All(images, image => Assert.False(harness.Images.Contains(image)));
-            Assert.Equal(0UL, harness.Images.TilerForTest.OutstandingScratchBytes);
             Assert.Equal(0UL, harness.Images.TilerForTest.PooledScratchBytes);
         });
+        // Earlier batches have already published; the final batch may be pending.
+        Assert.Contains(clear, harness.Read(address, (int)size).Chunk(4).Select(bytes => BitConverter.ToUInt32(bytes)));
         harness.Finish();
         for (var index = 0; index < 3; index++)
         {