diff --git a/src/SharpEmu.Libs/Gpu/Buffers/GpuBuffer.cs b/src/SharpEmu.Libs/Gpu/Buffers/GpuBuffer.cs index fec9cf7..bf14ce8 100644 --- a/src/SharpEmu.Libs/Gpu/Buffers/GpuBuffer.cs +++ b/src/SharpEmu.Libs/Gpu/Buffers/GpuBuffer.cs @@ -149,6 +149,9 @@ public unsafe class GpuBuffer : IDisposable public bool IsCoherent { get; } + // The host mapping of the whole buffer, or null for memory the host cannot map. + public byte* MappedPointer => _mapped; + public GpuBufferUsage Usage { get; } public ulong CpuAddress { get; } @@ -339,6 +342,7 @@ public unsafe class GpuBuffer : IDisposable var (required, preferred, avoided) = usage switch { GpuBufferUsage.DeviceLocal => (MemoryPropertyFlags.None, MemoryPropertyFlags.DeviceLocalBit, MemoryPropertyFlags.None), + GpuBufferUsage.Unified => (MemoryPropertyFlags.HostVisibleBit | MemoryPropertyFlags.HostCoherentBit | MemoryPropertyFlags.DeviceLocalBit, MemoryPropertyFlags.HostCachedBit, MemoryPropertyFlags.None), GpuBufferUsage.Upload => (MemoryPropertyFlags.HostVisibleBit, MemoryPropertyFlags.HostCoherentBit, MemoryPropertyFlags.DeviceLocalBit), GpuBufferUsage.Download => (MemoryPropertyFlags.HostVisibleBit, MemoryPropertyFlags.HostCoherentBit | MemoryPropertyFlags.HostCachedBit, MemoryPropertyFlags.DeviceLocalBit), _ => (MemoryPropertyFlags.HostVisibleBit, MemoryPropertyFlags.HostCoherentBit | MemoryPropertyFlags.DeviceLocalBit, MemoryPropertyFlags.None), diff --git a/src/SharpEmu.Libs/Gpu/Buffers/GpuBufferUsage.cs b/src/SharpEmu.Libs/Gpu/Buffers/GpuBufferUsage.cs index 09fc7c3..63dde7c 100644 --- a/src/SharpEmu.Libs/Gpu/Buffers/GpuBufferUsage.cs +++ b/src/SharpEmu.Libs/Gpu/Buffers/GpuBufferUsage.cs @@ -9,4 +9,6 @@ public enum GpuBufferUsage : byte Upload, Download, Stream, + // Device-local memory the host can map: guest buffers on unified-memory devices, read back without a copy. + Unified, } diff --git a/src/SharpEmu.Libs/Gpu/Buffers/GuestBufferCache.cs b/src/SharpEmu.Libs/Gpu/Buffers/GuestBufferCache.cs index aa2c6fb..75d2e7a 100644 --- a/src/SharpEmu.Libs/Gpu/Buffers/GuestBufferCache.cs +++ b/src/SharpEmu.Libs/Gpu/Buffers/GuestBufferCache.cs @@ -41,6 +41,7 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable private readonly SubmissionScheduler _scheduler; private readonly IGpuQueueRelay _relay; private readonly GuestBufferUploader _uploader; + private readonly bool _unifiedBuffers; private readonly IGuestBackedSpace _backing; private readonly BdaFaultProcessor _faults; private readonly GpuBuffer _gds; @@ -72,6 +73,7 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable _relay = relay; _backing = backing; _faults = new BdaFaultProcessor(device, scheduler, this, CachingPageBits, CachingPageCount); + _unifiedBuffers = UnifiedBuffersEnabled && HasUnifiedMemoryType(device); _gds = new GpuBuffer(device, scheduler, GpuBufferUsage.Stream, 0, GpuBuffer.AllFlags, GdsBufferSize); _bdaPageTable = new GpuBuffer(device, scheduler, GpuBufferUsage.DeviceLocal, 0, GpuBuffer.AllFlags, BdaPageTableSize); _tracker = new GuestPageTracker(pages); @@ -1182,6 +1184,9 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable public required bool IsWrite { get; init; } public required GuestMemoryProfile.ReadbackSource Source { get; init; } public required long Started { get; init; } + + // Read from the buffers' own host mappings once the writer tick retires; no copy was recorded. + public bool Mapped { get; init; } } private PendingDownload? BeginReadMemoryOnGpu(ulong guestAddress, ulong size, bool isWrite, GuestMemoryProfile.ReadbackSource source, @@ -1205,6 +1210,53 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable { NoteHotReadback(guestAddress, size); } + if (copies.Count != 0 && _unifiedBuffers && AreMapped(copies, out var writer)) + { + if (writer == 0 || _scheduler.IsTickComplete(writer)) + { + ApplyMappedRead(copies, windowBegin, windowEnd, guestAddress, size, isWrite, readbackStarted, source); + return null; + } + + if (allowPending && writer < _scheduler.CurrentTick) + { + // A guest thread had to wait for this writer: submit later writes to the window right + // after their command, so the next wait ends with them instead of with their whole batch. + NoteHotReadback(guestAddress, size); + + // The writer is already submitted: wait for it alone, not for the work queued after it. + var mappedTicks = new ulong[copies.Count]; + for (var index = 0; index < copies.Count; index++) + { + mappedTicks[index] = copies[index].Buffer.LastGpuWriteTick; + copies[index].Buffer.RetainForeignRead(); + } + + return new PendingDownload + { + Mapped = true, + MainQueueTick = writer, + Copies = copies, + WriteTicks = mappedTicks, + WindowBegin = windowBegin, + WindowEnd = windowEnd, + GuestAddress = guestAddress, + Size = size, + IsWrite = isWrite, + Source = source, + Started = readbackStarted, + }; + } + + if (!allowPending) + { + // On the command worker: submit the writer if it is still recording, then read the mapping. + _scheduler.Wait(writer); + ApplyMappedRead(copies, windowBegin, windowEnd, guestAddress, size, isWrite, readbackStarted, source); + return null; + } + } + if (copies.Count != 0 && allowPending && TryBeginDownloadAsync(copies, out var ticket, out var writeTicks)) { foreach (var copy in copies) @@ -1326,6 +1378,11 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable private bool CompleteReadMemoryOnGpu(PendingDownload pending, bool retrySynchronously) { using var readbackScope = GuestMemoryProfile.Measure(GuestMemoryProfile.Operation.BufferReadback); + if (pending.Mapped) + { + return CompleteMappedRead(pending, retrySynchronously); + } + if (pending.Ticket is null) { return CompleteMainQueueRead(pending, retrySynchronously); @@ -1381,6 +1438,103 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable } } + // Guest buffers live in host-visible device memory on unified-memory devices (Apple GPUs), so a + // readback can copy straight out of the buffer once the last GPU write to it retired. That skips + // the copy command at the tail of the queue and the wait for every draw recorded after the writer. + // SHARPEMU_UNIFIED_BUFFERS=0 keeps device-local buffers and copy readbacks. + private static readonly bool UnifiedBuffersEnabled = + Environment.GetEnvironmentVariable("SHARPEMU_UNIFIED_BUFFERS") != "0"; + + private static bool HasUnifiedMemoryType(GpuDeviceInfo device) + { + const MemoryPropertyFlags unified = MemoryPropertyFlags.DeviceLocalBit | MemoryPropertyFlags.HostVisibleBit | MemoryPropertyFlags.HostCoherentBit; + for (uint index = 0; index < device.MemoryTypeCount; index++) + { + if ((device.GetMemoryTypeFlags(index) & unified) == unified) + { + return true; + } + } + + return false; + } + + private static bool AreMapped(List copies, out ulong writer) + { + writer = 0; + foreach (var copy in copies) + { + if (copy.Buffer.MappedPointer == null || !copy.Buffer.IsCoherent) + { + return false; + } + + writer = Math.Max(writer, copy.Buffer.LastGpuWriteTick); + } + + return true; + } + + private void ApplyMappedRead(List copies, ulong windowBegin, ulong windowEnd, ulong guestAddress, ulong size, + bool isWrite, long started, GuestMemoryProfile.ReadbackSource source) + { + foreach (var copy in copies) + { + var bytes = new ReadOnlySpan(copy.Buffer.MappedPointer + copy.SourceOffset, checked((int)copy.Size)); + if (!_backing.TryWriteBacking(copy.Address, bytes)) + { + throw SubmissionScheduler.Fatal($"Could not write the required direct backing: addr=0x{copy.Address:X16} size=0x{copy.Size:X16}"); + } + + _gpuModifiedRanges.Remove(copy.Address, copy.Size); + } + + _tracker.ClearGpuDirtyPages(windowBegin, windowEnd - windowBegin); + if (isWrite) + { + _tracker.MarkCpuDirtyPages(guestAddress, size); + } + + MappedReadbacks++; + RecordReadback(windowBegin, windowEnd, isWrite, copies, started, source); + } + + // Readbacks served from a buffer's own mapping, without a copy command. + internal long MappedReadbacks { get; private set; } + + internal bool UnifiedBuffers => _unifiedBuffers; + + private bool CompleteMappedRead(PendingDownload pending, bool retrySynchronously) + { + try + { + if (!IsPendingDownloadCurrent(pending)) + { + if (retrySynchronously) + { + PendingReadbacksRetried++; + Interlocked.Increment(ref _reportedRetried); + ReadMemoryOnGpu(pending.GuestAddress, pending.Size, pending.IsWrite, pending.Source); + } + + return false; + } + + PendingReadbacksApplied++; + Interlocked.Increment(ref _reportedApplied); + ApplyMappedRead(pending.Copies, pending.WindowBegin, pending.WindowEnd, pending.GuestAddress, pending.Size, + pending.IsWrite, pending.Started, pending.Source); + return true; + } + finally + { + foreach (var copy in pending.Copies) + { + copy.Buffer.ReleaseForeignRead(); + } + } + } + private bool CompleteMainQueueRead(PendingDownload pending, bool retrySynchronously) { var buffer = pending.MainQueueBuffer ?? throw SubmissionScheduler.Fatal("The pending readback lost its buffer."); @@ -1850,7 +2004,7 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable } var bufferIdentifier = _registry.AllocateBuffer(new GpuBuffer( - _device, _scheduler, GpuBufferUsage.DeviceLocal, overlap.Begin, + _device, _scheduler, _unifiedBuffers ? GpuBufferUsage.Unified : GpuBufferUsage.DeviceLocal, overlap.Begin, GpuBuffer.AllFlags | BufferUsageFlags.ShaderDeviceAddressBit, overlap.End - overlap.Begin, allowSlab: true), overlap.Begin, overlap.End - overlap.Begin); foreach (var oldId in overlapping) { diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Buffers/GuestBufferAsyncReadbackTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Buffers/GuestBufferAsyncReadbackTests.cs index 52aae00..95f94fa 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Buffers/GuestBufferAsyncReadbackTests.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Buffers/GuestBufferAsyncReadbackTests.cs @@ -75,6 +75,64 @@ public sealed class GuestBufferAsyncReadbackTests(HeadlessVulkanFixture fixture) harness.Shutdown(); } + // On a unified-memory device a read of a retired GPU write copies straight out of the buffer's + // own mapping: no copy command, no pending readback. + [Fact] + public void AReadOfARetiredWriteComesFromTheUnifiedMapping() + { + if (_vulkan is null || !GatePrerequisites.Ready(_vulkan)) return; + using var harness = new CacheHarness(_vulkan); + using var fatal = new FatalScope(); + if (!harness.Cache.UnifiedBuffers) + { + harness.Shutdown(); + return; + } + + var address = harness.MapBacked(0x10000, ReadWrite); + GpuWrite(harness, address, 0x0BADF00D); + harness.Worker.Run(() => harness.Scheduler.Wait(harness.Scheduler.Flush())); + + Assert.True(harness.Cache.DownloadToCpu(address + 8, 4)); + + Assert.False(harness.Cache.HasGpuDirtyBytes(address, 0x100)); + Assert.Equal(0x0BADF00Du, BitConverter.ToUInt32(harness.Read(address + 0x80, 4))); + Assert.Equal(1L, harness.Cache.MappedReadbacks); + Assert.Equal(0L, harness.Cache.PendingReadbacksApplied); + harness.Shutdown(); + } + + // A read while the writer is submitted but still running waits for that submission only, then + // reads the mapping; a write recorded after the read was issued is picked up by the next read. + [Fact] + public void AReadOfASubmittedWriteWaitsForItAndReadsTheUnifiedMapping() + { + if (_vulkan is null || !GatePrerequisites.Ready(_vulkan)) return; + using var harness = new CacheHarness(_vulkan); + using var fatal = new FatalScope(); + if (!harness.Cache.UnifiedBuffers) + { + harness.Shutdown(); + return; + } + + var address = harness.MapBacked(0x10000, ReadWrite); + GpuWrite(harness, address, 0x12345678); + SubmitBatch(harness); + + Assert.True(harness.Cache.DownloadToCpu(address, 4)); + + Assert.Equal(0x12345678u, BitConverter.ToUInt32(harness.Read(address + 0xFC, 4))); + Assert.Equal(1L, harness.Cache.MappedReadbacks); + Assert.False(harness.Cache.HasGpuDirtyBytes(address, 0x100)); + + GpuWrite(harness, address, 0x9ABCDEF0); + SubmitBatch(harness); + Assert.True(harness.Cache.DownloadToCpu(address, 4)); + Assert.Equal(0x9ABCDEF0u, BitConverter.ToUInt32(harness.Read(address + 0x10, 4))); + harness.Shutdown(); + } + private static void GpuWrite(CacheHarness harness, ulong address, uint value) => harness.Worker.Run(() => { var (buffer, offset) = harness.Cache.ObtainBuffer(address, 0x100, isWritten: true);