[Gpu] Compile compute pipelines off the command-stream thread

A host shader compiler can spend tens of seconds on a single large translated
compute program, and the command stream could not advance while it did, because
the pipeline cache holds its lock across the creation call. Every other queue
that wanted any pipeline waited behind it too.

The pipeline host can now report that a compute pipeline is not ready yet. The
Vulkan presenter answers that way the first time it sees a program: it prepares
everything the command stream owns (descriptor set layout, pipeline layout,
shader module) up front and runs only vkCreateComputePipelines on a worker
thread, so the task touches nothing else. The number of compiles in flight is
bounded, since each one blocks its thread inside the host compiler and a frame
can touch many programs at once. Asking again returns the finished pipeline
without starting a second compile.

The dispatch waits for its pipeline rather than being dropped: a guest that
does not replay a one-shot dispatch deadlocks, because the dispatch can feed a
label a later packet waits on. Asking again from the executor keeps that wait
outside the pipeline cache's lock. Hosts that compile cheaply keep the blocking
behaviour through the interface default, and
SHARPEMU_ASYNC_COMPUTE_PIPELINES=0 compiles inline.

src/SharpEmu.Libs/Gpu/Pipelines/IShaderPipelineHost.cs

@@ -67,4 +67,14 @@ internal interface IShaderPipelineHost
     PipelineHandle CreateGraphicsPipeline(GraphicsPipelineDescription description);
 
     PipelineHandle CreateComputePipeline(ComputePipelineDescription description);
+
+    // Creates the compute pipeline without blocking the command stream on the host
+    // shader compiler: false means the compile is still running and the caller should ask
+    // again, true hands over the finished pipeline. A backend that compiles cheaply keeps
+    // the blocking behaviour.
+    bool TryCreateComputePipeline(ComputePipelineDescription description, out PipelineHandle handle)
+    {
+        handle = CreateComputePipeline(description);
+        return true;
+    }
 }

src/SharpEmu.Libs/Gpu/Pipelines/ShaderPipelineCache.cs

@@ -595,4 +595,37 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider
             return created;
         }
     }
+
+    public bool TryCreateComputePipeline(ComputeInputInfo input, ShaderProgram program, out PipelineHandle handle)
+    {
+        using var profileScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.PipelineCreation);
+        handle = default;
+        if (!program.IsValid)
+        {
+            throw SubmissionScheduler.Fatal("The dispatch has no compute program.");
+        }
+
+        var stage = input.Stage.Program ?? throw SubmissionScheduler.Fatal("The compute stage has no program.");
+        var key = new ComputePipelineKey(program.Id);
+        lock (_gate)
+        {
+            if (_computePipelines.TryGetValue(key, out var cached))
+            {
+                handle = cached;
+                return true;
+            }
+
+            if (!_host.TryCreateComputePipeline(
+                    new ComputePipelineDescription { Input = input, Program = program, Stage = stage },
+                    out var created))
+            {
+                return false;
+            }
+
+            _computePipelines.Add(key, created);
+            ShaderCacheCounters.CountComputePipeline();
+            handle = created;
+            return true;
+        }
+    }
 }

src/SharpEmu.Libs/Gpu/Rendering/IShaderPipelineProvider.cs

@@ -85,4 +85,12 @@ public interface IShaderPipelineProvider
         uint dimensionZ);
 
     PipelineHandle CreateComputePipeline(ComputeInputInfo input, ShaderProgram program);
+
+    // False means the host is still compiling this pipeline; asking again returns it once
+    // the compile finishes, without starting a second one.
+    bool TryCreateComputePipeline(ComputeInputInfo input, ShaderProgram program, out PipelineHandle handle)
+    {
+        handle = CreateComputePipeline(input, program);
+        return true;
+    }
 }

src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Compute.cs

@@ -128,7 +128,18 @@ public sealed partial class RenderExecutor
         _host.EndRendering();
         using (_host.BeginPreparation())
         {
-            var pipeline = _pipelines.CreateComputePipeline(input, computeProgram.Program);
+            // The host may compile this program off the command-stream thread and report that
+            // the pipeline is not ready yet. The dispatch waits for it rather than being
+            // dropped: a guest that does not replay a one-shot dispatch deadlocks, because
+            // the dispatch can feed a label a later packet waits on. Asking again keeps the
+            // wait outside the pipeline cache's lock, so other queues can create their own
+            // pipelines while this program compiles.
+            PipelineHandle pipeline;
+            while (!_pipelines.TryCreateComputePipeline(input, computeProgram.Program, out pipeline))
+            {
+                Thread.Sleep(1);
+            }
+
             var bindings = _host.PrepareBindings(input.Stage);
             if (program.UsesDeviceAddresses)
             {

src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Pipelines.cs

@@ -852,6 +852,166 @@ internal static unsafe partial class VulkanVideoPresenter
             return RegisterPipeline(entry);
         }
 
+        // One compute pipeline whose vkCreateComputePipelines call runs on a worker
+        // thread. Everything the command stream owns (descriptor and pipeline layout,
+        // the module handle) is prepared before the task starts, so the task touches
+        // nothing but the Vulkan pipeline creation itself.
+        private sealed class PendingComputePipeline
+        {
+            public required DescriptorSetLayout SetLayout;
+            public required PipelineLayout Layout;
+            public required DescriptorSetDemand Demand;
+            public required bool UsesPushDescriptors;
+            public required ulong Hash;
+            public required int SpirvBytes;
+            public Task<Pipeline> Compile = Task.FromResult(default(Pipeline));
+            public long StartTimestamp;
+            public long CompileMilliseconds;
+        }
+
+        private readonly Dictionary<ulong, PendingComputePipeline> _pendingComputePipelines = new();
+        private static readonly SemaphoreSlim _computeCompileSlots =
+            new(Math.Max(2, Environment.ProcessorCount / 2));
+
+        // A host shader compiler can spend tens of seconds on one large translated
+        // program. Running the compile on a worker thread keeps it out of the pipeline
+        // cache's lock, so other queues can still create their own pipelines meanwhile;
+        // SHARPEMU_ASYNC_COMPUTE_PIPELINES=0 compiles inline instead.
+        private static readonly bool _asyncComputePipelines =
+            !string.Equals(
+                Environment.GetEnvironmentVariable("SHARPEMU_ASYNC_COMPUTE_PIPELINES"),
+                "0",
+                StringComparison.Ordinal);
+
+        public bool TryCreateComputePipeline(ComputePipelineDescription description, out PipelineHandle handle)
+        {
+            if (!_asyncComputePipelines)
+            {
+                handle = CreateComputePipeline(description);
+                return true;
+            }
+
+            handle = default;
+            var key = description.Program.Id;
+            if (_pendingComputePipelines.TryGetValue(key, out var pending))
+            {
+                if (!pending.Compile.IsCompleted)
+                {
+                    return false;
+                }
+
+                _pendingComputePipelines.Remove(key);
+                var compiled = pending.Compile.GetAwaiter().GetResult();
+                ReportPipelineCreation(
+                    Interlocked.Read(ref pending.CompileMilliseconds),
+                    "compute-async",
+                    $"cs=0x{pending.Hash:X16} waited_ms={(long)Stopwatch.GetElapsedTime(pending.StartTimestamp).TotalMilliseconds}",
+                    pending.SpirvBytes.ToString());
+                MarkPipelineCacheDirty();
+                Interlocked.Increment(ref _perfPipelineCreations);
+                SetDebugName(ObjectType.Pipeline, compiled.Handle, $"SharpEmu compute cs=0x{pending.Hash:X16}");
+                handle = RegisterPipeline(new RenderPipelineEntry
+                {
+                    Pipeline = compiled,
+                    Layout = pending.Layout,
+                    SetLayout = pending.SetLayout,
+                    Demand = pending.Demand,
+                    UsesPushDescriptors = pending.UsesPushDescriptors,
+                    ProfileComputeHash = pending.Hash,
+                });
+                return true;
+            }
+
+            using var profileScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.PipelineSetup);
+            var bindings = new List<DescriptorSetLayoutBinding>();
+            CollectLayoutBindings(bindings, description.Stage, ShaderStage.Compute);
+            var setLayout = CreateDescriptorSetLayout(bindings, out var usesPushDescriptors, out var demand);
+            var layout = CreatePipelineLayout(setLayout, ShaderStageFlags.ComputeBit);
+            var computeModule = new ShaderModule(description.Program.Module);
+            if (computeModule.Handle == 0)
+            {
+                throw SubmissionScheduler.Fatal($"The compute pipeline has no module: hash=0x{description.Stage.Hash:X16}.");
+            }
+
+            var device = _device;
+            var cache = _pipelineCache;
+            var vk = _vk;
+            var started = new PendingComputePipeline
+            {
+                SetLayout = setLayout,
+                Layout = layout,
+                Demand = demand,
+                UsesPushDescriptors = usesPushDescriptors,
+                Hash = description.Stage.Hash,
+                SpirvBytes = SpirvBytesOf(computeModule.Handle),
+                StartTimestamp = Stopwatch.GetTimestamp(),
+            };
+            started.Compile = Task.Factory.StartNew(
+                () =>
+                {
+                    // Each compile blocks its thread inside the Metal compiler service,
+                    // so the number in flight is bounded instead of one thread per
+                    // program the frame happens to touch.
+                    _computeCompileSlots.Wait();
+                    var compileStart = Stopwatch.GetTimestamp();
+                    try
+                    {
+                        return CompileComputePipeline(vk, device, cache, computeModule, layout);
+                    }
+                    finally
+                    {
+                        Interlocked.Exchange(
+                            ref started.CompileMilliseconds,
+                            (long)Stopwatch.GetElapsedTime(compileStart).TotalMilliseconds);
+                        _computeCompileSlots.Release();
+                    }
+                },
+                CancellationToken.None,
+                TaskCreationOptions.LongRunning,
+                TaskScheduler.Default);
+            _pendingComputePipelines.Add(key, started);
+            return false;
+        }
+
+        // vkCreateComputePipelines is the only call here; Vulkan synchronises host
+        // access to the pipeline cache internally, so several may run at once.
+        private static Pipeline CompileComputePipeline(
+            Vk vk,
+            Device device,
+            PipelineCache cache,
+            ShaderModule module,
+            PipelineLayout layout)
+        {
+            var entryPoint = (byte*)SilkMarshal.StringToPtr("main");
+            try
+            {
+                var stageInfo = new PipelineShaderStageCreateInfo
+                {
+                    SType = StructureType.PipelineShaderStageCreateInfo,
+                    Stage = ShaderStageFlags.ComputeBit,
+                    Module = module,
+                    PName = entryPoint,
+                };
+                var pipelineInfo = new ComputePipelineCreateInfo
+                {
+                    SType = StructureType.ComputePipelineCreateInfo,
+                    Stage = stageInfo,
+                    Layout = layout,
+                };
+                var result = vk.CreateComputePipelines(device, cache, 1, &pipelineInfo, null, out var pipeline);
+                if (result != Result.Success)
+                {
+                    throw SubmissionScheduler.Fatal($"vkCreateComputePipelines(async) failed: {result}.");
+                }
+
+                return pipeline;
+            }
+            finally
+            {
+                SilkMarshal.Free((nint)entryPoint);
+            }
+        }
+
         private void DestroyRenderPipelines()
         {
             foreach (var entry in _pipelineEntries.Values)

tests/SharpEmu.Libs.Tests/Gpu/Pipelines/PipelineTestSupport.cs

@@ -89,6 +89,23 @@ internal sealed class FakePipelineHost(ICpuMemory memory) : IShaderPipelineHost
         var handle = _nextHandle++;
         return new PipelineHandle(handle, handle, UsesPushDescriptors: true);
     }
+
+    // How many times each program reports "still compiling" before it is handed over,
+    // standing in for the host shader compiler running on a worker thread.
+    public Dictionary<ulong, int> PendingComputeCompiles { get; } = new();
+
+    public bool TryCreateComputePipeline(ComputePipelineDescription description, out PipelineHandle handle)
+    {
+        handle = default;
+        if (PendingComputeCompiles.TryGetValue(description.Program.Id, out var remaining) && remaining > 0)
+        {
+            PendingComputeCompiles[description.Program.Id] = remaining - 1;
+            return false;
+        }
+
+        handle = CreateComputePipeline(description);
+        return true;
+    }
 }
 
 // Remembers every request; compiles nothing unless a module factory was given.

tests/SharpEmu.Libs.Tests/Gpu/Pipelines/ShaderPipelineCacheTests.cs

@@ -231,6 +231,30 @@ public sealed class ShaderPipelineCacheTests : IDisposable
         Assert.Equal(2, cache.ComputePipelineCount);
     }
 
+    [Fact]
+    public void TryCreateComputePipeline_ReportsTheHostCompileAndCachesTheResult()
+    {
+        var guest = new PipelineTestGuest();
+        var cache = new ShaderPipelineCache(guest.Context, guest.Host, guest.Compiler, guest.Registry);
+        var input = ComputeProgram().Input;
+        var program = new ShaderProgram(7, 1);
+        guest.Host.PendingComputeCompiles[program.Id] = 2;
+
+        Assert.False(cache.TryCreateComputePipeline(input, program, out _));
+        Assert.False(cache.TryCreateComputePipeline(input, program, out _));
+        Assert.Empty(guest.Host.ComputePipelines);
+        Assert.Equal(0, cache.ComputePipelineCount);
+
+        Assert.True(cache.TryCreateComputePipeline(input, program, out var ready));
+        Assert.Single(guest.Host.ComputePipelines);
+
+        // The cached pipeline is served without asking the host again.
+        Assert.True(cache.TryCreateComputePipeline(input, program, out var cached));
+        Assert.Equal(ready, cached);
+        Assert.Single(guest.Host.ComputePipelines);
+        Assert.Equal(1, cache.ComputePipelineCount);
+    }
+
     [Fact]
     public void GraphicsPipelines_AreCachedByTheirKey()
     {