shifathosn
/
sharpemu
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[Vk] On macOS, retire a GPU tick only once its submission's fence signals 

MoltenVK records Metal command buffers with unretained resource references
and signals a Vulkan timeline semaphore from a Metal event scheduled at the
end of the command buffer. That event can fire before Metal has finished
executing the command buffer, so a host thread that treats the timeline
value as authoritative can free resources the GPU is still reading. Metal
API validation reports this as "destroyed while still required by a command
buffer", and the resulting use-after-free showed up as intermittent GPU
tick stalls.

Every submission that signals the timeline now also signals a VkFence, which
MoltenVK signals from the Metal command buffer's completion handler. The
timeline value a host thread is allowed to act on is clamped to just below
the oldest tick whose fence has not signaled yet, and a timeline wait is
followed by a blocking wait on the last fence covering that tick. Fences are
pooled, tracked before the submit so no reader can observe a signaled tick
without its fence, and reset only once no thread is blocked on them. Fences
are polled at all only when the timeline already claims the tick is done, so
the common path costs one queue peek. The behaviour is macOS-only and can be
turned off with SHARPEMU_FENCE_RETIREMENT=0.
This commit is contained in:
kuba 2026-09-29 23:06:58 +02:00
parent 4d65ff6d51
commit bd840c422b
1 changed files with 202 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

205
src/SharpEmu.Libs/Gpu/Vulkan/VulkanTickDevice.cs
View File

@ -48,6 +48,149 @@ internal sealed unsafe class VulkanTickDevice : IGpuTickDevice
public VulkanCommandProfile? CommandProfile { get; }
// MoltenVK records Metal command buffers with unretained references and signals the timeline
// semaphore with a Metal event at the end of each one, which can fire before Metal has finished
// with the command buffer. Destroying a resource once its tick signals is valid Vulkan but breaks
// Metal's rule that unretained resources outlive the command buffer. On macOS each submission
// also signals a fence, which MoltenVK signals from the command buffer's completion handler, and a
// tick counts as complete to the host only once its fence has signaled too.
// SHARPEMU_FENCE_RETIREMENT=0 reports the timeline value alone.
private static readonly bool FenceRetirement = OperatingSystem.IsMacOS() &&
Environment.GetEnvironmentVariable("SHARPEMU_FENCE_RETIREMENT") != "0";
private readonly object _fenceGate = new();
private sealed class InFlightFence(ulong tick, Fence fence)
{
public readonly ulong Tick = tick;
public readonly Fence Fence = fence;
// Host threads blocked on the fence; it is reset and pooled only once none are.
public int Waiters;
}
private readonly Queue<InFlightFence> _inFlightFences = new();
private readonly List<InFlightFence> _retiredWithWaiters = new();
private readonly Stack<Fence> _freeFences = new();
private Fence TakeFenceLocked()
{
if (_freeFences.TryPop(out var fence))
{
return fence;
}
var info = new FenceCreateInfo { SType = StructureType.FenceCreateInfo };
RequireSuccess(_vk.CreateFence(_device, &info, null, out fence), "vkCreateFence(scheduler retirement)");
return fence;
}
private void RecycleLocked(InFlightFence entry)
{
var fence = entry.Fence;
RequireSuccess(_vk.ResetFences(_device, 1, &fence), "vkResetFences(scheduler retirement)");
_freeFences.Push(fence);
}
// Retires the leading submissions whose fences signaled; returns the first tick still in flight.
private ulong RetireSignaledFencesLocked()
{
for (var index = _retiredWithWaiters.Count - 1; index >= 0; index--)
{
if (_retiredWithWaiters[index].Waiters == 0)
{
RecycleLocked(_retiredWithWaiters[index]);
_retiredWithWaiters.RemoveAt(index);
}
}
while (_inFlightFences.TryPeek(out var entry))
{
if (_vk.GetFenceStatus(_device, entry.Fence) != Result.Success)
{
return entry.Tick;
}
_inFlightFences.Dequeue();
if (entry.Waiters == 0)
{
RecycleLocked(entry);
}
else
{
_retiredWithWaiters.Add(entry);
}
}
return ulong.MaxValue;
}
// The timeline value the host may treat as complete.
private ulong RetiredValue(ulong timelineValue)
{
if (!FenceRetirement)
{
return timelineValue;
}
lock (_fenceGate)
{
// Below the oldest in-flight tick the timeline value is already retired; poll fences only
// for ticks the timeline reports done.
if (!_inFlightFences.TryPeek(out var oldest) || timelineValue < oldest.Tick)
{
return timelineValue;
}
var firstInFlight = RetireSignaledFencesLocked();
return firstInFlight == ulong.MaxValue ? timelineValue : Math.Min(timelineValue, firstInFlight - 1);
}
}
// Waits until every submission up to the tick has retired. The waited fence is counted so no
// other thread resets and reuses it while this one blocks on it.
private bool WaitRetired(ulong tick)
{
if (!FenceRetirement)
{
return true;
}
InFlightFence? waited = null;
lock (_fenceGate)
{
if (RetireSignaledFencesLocked() > tick)
{
return true;
}
foreach (var entry in _inFlightFences)
{
if (entry.Tick > tick)
{
break;
}
waited = entry;
}
if (waited is null)
{
return true;
}
waited.Waiters++;
}
var fence = waited.Fence;
var result = _vk.WaitForFences(_device, 1, &fence, true, ulong.MaxValue);
lock (_fenceGate)
{
waited.Waiters--;
_ = RetireSignaledFencesLocked();
}
return result == Result.Success;
}
public object QueueGate { get; }
public ulong TimelineHandle => _timeline.Handle;
@ -56,7 +199,7 @@ internal sealed unsafe class VulkanTickDevice : IGpuTickDevice
{
ulong value;
RequireSuccess(_vk.GetSemaphoreCounterValue(_device, _timeline, &value), "vkGetSemaphoreCounterValue");
return value;
return RetiredValue(value);
}
public bool TryWaitTimeline(ulong tick, out string failure)
@ -75,7 +218,7 @@ internal sealed unsafe class VulkanTickDevice : IGpuTickDevice
result = _vk.WaitSemaphores(_device, &waitInfo, ulong.MaxValue);
}
failure = result.ToString();
return result == Result.Success;
return result == Result.Success && WaitRetired(tick);
}
public nint[] AllocateBuffers(int count)
@ -168,7 +311,46 @@ internal sealed unsafe class VulkanTickDevice : IGpuTickDevice
SignalSemaphoreInfoCount = (uint)bundle.SignalCount,
PSignalSemaphoreInfos = signalInfos,
};
var result = _vk.QueueSubmit2(_queue, 1, &submitInfo, default);
Fence fence = default;
ulong fenceTick = 0;
if (FenceRetirement && bundle.SignalCount != 0)
{
for (var index = 0; index < bundle.SignalCount; index++)
{
if (signalSemaphores[index] == _timeline.Handle)
{
fenceTick = Math.Max(fenceTick, signalTicks[index]);
}
}
if (fenceTick != 0)
{
// Tracked before the submit so no reader sees the tick signal without its fence.
lock (_fenceGate)
{
fence = TakeFenceLocked();
_inFlightFences.Enqueue(new InFlightFence(fenceTick, fence));
}
}
}
var result = _vk.QueueSubmit2(_queue, 1, &submitInfo, fence);
if (fence.Handle != 0 && result != Result.Success)
{
lock (_fenceGate)
{
// Submissions are serialized by the queue gate, so the failed one is the newest.
var kept = _inFlightFences.Where(entry => entry.Fence.Handle != fence.Handle).ToArray();
_inFlightFences.Clear();
foreach (var entry in kept)
{
_inFlightFences.Enqueue(entry);
}
_freeFences.Push(fence);
}
}
if (result == Result.Success)
CommandProfile?.MarkSubmitted(buffer);
failure = result.ToString();
@ -179,6 +361,23 @@ internal sealed unsafe class VulkanTickDevice : IGpuTickDevice
public void Dispose()
{
CommandProfile?.Dispose();
lock (_fenceGate)
{
foreach (var entry in _inFlightFences.Concat(_retiredWithWaiters))
{
var fence = entry.Fence;
_ = _vk.WaitForFences(_device, 1, &fence, true, ulong.MaxValue);
_vk.DestroyFence(_device, fence, null);
}
_inFlightFences.Clear();
_retiredWithWaiters.Clear();
while (_freeFences.TryPop(out var fence))
{
_vk.DestroyFence(_device, fence, null);
}
}
_vk.DestroySemaphore(_device, _timeline, null);
_vk.DestroyCommandPool(_device, _pool, null);
}