diff --git a/src/SharpEmu.Libs/Gpu/Vulkan/VulkanTickDevice.cs b/src/SharpEmu.Libs/Gpu/Vulkan/VulkanTickDevice.cs index 9233797..5a42bb8 100644 --- a/src/SharpEmu.Libs/Gpu/Vulkan/VulkanTickDevice.cs +++ b/src/SharpEmu.Libs/Gpu/Vulkan/VulkanTickDevice.cs @@ -48,6 +48,149 @@ internal sealed unsafe class VulkanTickDevice : IGpuTickDevice public VulkanCommandProfile? CommandProfile { get; } + // MoltenVK records Metal command buffers with unretained references and signals the timeline + // semaphore with a Metal event at the end of each one, which can fire before Metal has finished + // with the command buffer. Destroying a resource once its tick signals is valid Vulkan but breaks + // Metal's rule that unretained resources outlive the command buffer. On macOS each submission + // also signals a fence, which MoltenVK signals from the command buffer's completion handler, and a + // tick counts as complete to the host only once its fence has signaled too. + // SHARPEMU_FENCE_RETIREMENT=0 reports the timeline value alone. + private static readonly bool FenceRetirement = OperatingSystem.IsMacOS() && + Environment.GetEnvironmentVariable("SHARPEMU_FENCE_RETIREMENT") != "0"; + + private readonly object _fenceGate = new(); + private sealed class InFlightFence(ulong tick, Fence fence) + { + public readonly ulong Tick = tick; + public readonly Fence Fence = fence; + // Host threads blocked on the fence; it is reset and pooled only once none are. + public int Waiters; + } + + private readonly Queue _inFlightFences = new(); + private readonly List _retiredWithWaiters = new(); + private readonly Stack _freeFences = new(); + + private Fence TakeFenceLocked() + { + if (_freeFences.TryPop(out var fence)) + { + return fence; + } + + var info = new FenceCreateInfo { SType = StructureType.FenceCreateInfo }; + RequireSuccess(_vk.CreateFence(_device, &info, null, out fence), "vkCreateFence(scheduler retirement)"); + return fence; + } + + private void RecycleLocked(InFlightFence entry) + { + var fence = entry.Fence; + RequireSuccess(_vk.ResetFences(_device, 1, &fence), "vkResetFences(scheduler retirement)"); + _freeFences.Push(fence); + } + + // Retires the leading submissions whose fences signaled; returns the first tick still in flight. + private ulong RetireSignaledFencesLocked() + { + for (var index = _retiredWithWaiters.Count - 1; index >= 0; index--) + { + if (_retiredWithWaiters[index].Waiters == 0) + { + RecycleLocked(_retiredWithWaiters[index]); + _retiredWithWaiters.RemoveAt(index); + } + } + + while (_inFlightFences.TryPeek(out var entry)) + { + if (_vk.GetFenceStatus(_device, entry.Fence) != Result.Success) + { + return entry.Tick; + } + + _inFlightFences.Dequeue(); + if (entry.Waiters == 0) + { + RecycleLocked(entry); + } + else + { + _retiredWithWaiters.Add(entry); + } + } + + return ulong.MaxValue; + } + + // The timeline value the host may treat as complete. + private ulong RetiredValue(ulong timelineValue) + { + if (!FenceRetirement) + { + return timelineValue; + } + + lock (_fenceGate) + { + // Below the oldest in-flight tick the timeline value is already retired; poll fences only + // for ticks the timeline reports done. + if (!_inFlightFences.TryPeek(out var oldest) || timelineValue < oldest.Tick) + { + return timelineValue; + } + + var firstInFlight = RetireSignaledFencesLocked(); + return firstInFlight == ulong.MaxValue ? timelineValue : Math.Min(timelineValue, firstInFlight - 1); + } + } + + // Waits until every submission up to the tick has retired. The waited fence is counted so no + // other thread resets and reuses it while this one blocks on it. + private bool WaitRetired(ulong tick) + { + if (!FenceRetirement) + { + return true; + } + + InFlightFence? waited = null; + lock (_fenceGate) + { + if (RetireSignaledFencesLocked() > tick) + { + return true; + } + + foreach (var entry in _inFlightFences) + { + if (entry.Tick > tick) + { + break; + } + + waited = entry; + } + + if (waited is null) + { + return true; + } + + waited.Waiters++; + } + + var fence = waited.Fence; + var result = _vk.WaitForFences(_device, 1, &fence, true, ulong.MaxValue); + lock (_fenceGate) + { + waited.Waiters--; + _ = RetireSignaledFencesLocked(); + } + + return result == Result.Success; + } + public object QueueGate { get; } public ulong TimelineHandle => _timeline.Handle; @@ -56,7 +199,7 @@ internal sealed unsafe class VulkanTickDevice : IGpuTickDevice { ulong value; RequireSuccess(_vk.GetSemaphoreCounterValue(_device, _timeline, &value), "vkGetSemaphoreCounterValue"); - return value; + return RetiredValue(value); } public bool TryWaitTimeline(ulong tick, out string failure) @@ -75,7 +218,7 @@ internal sealed unsafe class VulkanTickDevice : IGpuTickDevice result = _vk.WaitSemaphores(_device, &waitInfo, ulong.MaxValue); } failure = result.ToString(); - return result == Result.Success; + return result == Result.Success && WaitRetired(tick); } public nint[] AllocateBuffers(int count) @@ -168,7 +311,46 @@ internal sealed unsafe class VulkanTickDevice : IGpuTickDevice SignalSemaphoreInfoCount = (uint)bundle.SignalCount, PSignalSemaphoreInfos = signalInfos, }; - var result = _vk.QueueSubmit2(_queue, 1, &submitInfo, default); + Fence fence = default; + ulong fenceTick = 0; + if (FenceRetirement && bundle.SignalCount != 0) + { + for (var index = 0; index < bundle.SignalCount; index++) + { + if (signalSemaphores[index] == _timeline.Handle) + { + fenceTick = Math.Max(fenceTick, signalTicks[index]); + } + } + + if (fenceTick != 0) + { + // Tracked before the submit so no reader sees the tick signal without its fence. + lock (_fenceGate) + { + fence = TakeFenceLocked(); + _inFlightFences.Enqueue(new InFlightFence(fenceTick, fence)); + } + } + } + + var result = _vk.QueueSubmit2(_queue, 1, &submitInfo, fence); + if (fence.Handle != 0 && result != Result.Success) + { + lock (_fenceGate) + { + // Submissions are serialized by the queue gate, so the failed one is the newest. + var kept = _inFlightFences.Where(entry => entry.Fence.Handle != fence.Handle).ToArray(); + _inFlightFences.Clear(); + foreach (var entry in kept) + { + _inFlightFences.Enqueue(entry); + } + + _freeFences.Push(fence); + } + } + if (result == Result.Success) CommandProfile?.MarkSubmitted(buffer); failure = result.ToString(); @@ -179,6 +361,23 @@ internal sealed unsafe class VulkanTickDevice : IGpuTickDevice public void Dispose() { CommandProfile?.Dispose(); + lock (_fenceGate) + { + foreach (var entry in _inFlightFences.Concat(_retiredWithWaiters)) + { + var fence = entry.Fence; + _ = _vk.WaitForFences(_device, 1, &fence, true, ulong.MaxValue); + _vk.DestroyFence(_device, fence, null); + } + + _inFlightFences.Clear(); + _retiredWithWaiters.Clear(); + while (_freeFences.TryPop(out var fence)) + { + _vk.DestroyFence(_device, fence, null); + } + } + _vk.DestroySemaphore(_device, _timeline, null); _vk.DestroyCommandPool(_device, _pool, null); }