[ShaderPrewarm] Hold the game until the prewarm finishes and resume interrupted runs

The prewarm compiled thousands of compute pipelines on background threads
while the now faster boot ran beside it, so logos, Bink movies and the first
scenes raced the compile and dropped frames.

- Hold the guest at its first splash hide, GPU submission or CPU flip until
  the prewarm is decided and done; the render thread keeps presenting the
  splash. SHARPEMU_SHADER_PREWARM_WAIT=0 opts out.
- Stamp only the shader translators, the compiler type and the driver, so a
  Libs-only build no longer triggers a full prewarm.
- Record finished pipelines in shader-prewarm.progress so an interrupted
  prewarm resumes with the remaining ones.

src/SharpEmu.Libs/Gpu/Pipelines/ShaderPrewarmList.cs

@@ -1,6 +1,7 @@
 // Copyright (C) 2026 SharpEmu Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
+using System.Globalization;
 using System.IO.Hashing;
 using System.Text;
 using SharpEmu.HLE;
@@ -141,6 +142,7 @@ internal sealed class ShaderPrewarmList : IDisposable
 {
     public const string FileName = "shader-prewarm.bin";
     public const string StampFileName = "shader-prewarm.stamp";
+    public const string ProgressFileName = "shader-prewarm.progress";
 
     private const uint Magic = 0x57504553;
     private const uint FormatVersion = 1;
@@ -151,16 +153,19 @@ internal sealed class ShaderPrewarmList : IDisposable
     private readonly object _gate = new();
     private readonly FileStream _file;
     private readonly string _stampPath;
+    private readonly string _progressPath;
+    private readonly object _progressGate = new();
     private readonly HashSet<(ulong Hash, uint CodeSize, ulong Address)> _codes = [];
     private readonly HashSet<ulong> _computes = [];
     private readonly Dictionary<(ulong Hash, uint CodeSize, ulong Address), ShaderCodeCapture> _loadedCodes = new();
     private readonly List<ComputePrewarmRecord> _loadedComputes = [];
     private bool _failed;
 
-    private ShaderPrewarmList(FileStream file, string stampPath)
+    private ShaderPrewarmList(FileStream file, string stampPath, string progressPath)
     {
         _file = file;
         _stampPath = stampPath;
+        _progressPath = progressPath;
     }
 
     public int LoadedComputeCount => _loadedComputes.Count;
@@ -172,7 +177,7 @@ internal sealed class ShaderPrewarmList : IDisposable
             Directory.CreateDirectory(directory);
             var path = Path.Combine(directory, FileName);
             var file = new FileStream(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read);
-            var list = new ShaderPrewarmList(file, Path.Combine(directory, StampFileName));
+            var list = new ShaderPrewarmList(file, Path.Combine(directory, StampFileName), Path.Combine(directory, ProgressFileName));
             try
             {
                 list.Load();
@@ -230,6 +235,93 @@ internal sealed class ShaderPrewarmList : IDisposable
         }
     }
 
+    public static ulong Identity(ComputePrewarmRecord record) => XxHash3.HashToUInt64(SerializeCompute(record));
+
+    public HashSet<ulong> ReadProgress(string stamp)
+    {
+        var completed = new HashSet<ulong>();
+        lock (_progressGate)
+        {
+            try
+            {
+                if (!File.Exists(_progressPath))
+                {
+                    return completed;
+                }
+
+                var lines = File.ReadAllLines(_progressPath);
+                if (lines.Length == 0 || lines[0] != ProgressKey(stamp))
+                {
+                    File.Delete(_progressPath);
+                    return completed;
+                }
+
+                for (var index = 1; index < lines.Length; index++)
+                {
+                    if (ulong.TryParse(lines[index], NumberStyles.HexNumber, CultureInfo.InvariantCulture, out var identity))
+                    {
+                        completed.Add(identity);
+                    }
+                }
+            }
+            catch (Exception exception) when (exception is IOException or UnauthorizedAccessException)
+            {
+                completed.Clear();
+            }
+        }
+
+        return completed;
+    }
+
+    public void AppendProgress(string stamp, IReadOnlyCollection<ulong> identities)
+    {
+        if (identities.Count == 0)
+        {
+            return;
+        }
+
+        lock (_progressGate)
+        {
+            try
+            {
+                var lines = new List<string>(identities.Count + 1);
+                if (!File.Exists(_progressPath))
+                {
+                    lines.Add(ProgressKey(stamp));
+                }
+
+                foreach (var identity in identities)
+                {
+                    lines.Add(identity.ToString("X16", CultureInfo.InvariantCulture));
+                }
+
+                File.AppendAllLines(_progressPath, lines);
+            }
+            catch (Exception exception) when (exception is IOException or UnauthorizedAccessException)
+            {
+                Console.Error.WriteLine($"[LOADER][WARN] Shader prewarm progress write failed: {exception.Message}");
+            }
+        }
+    }
+
+    public void ClearProgress()
+    {
+        lock (_progressGate)
+        {
+            try
+            {
+                File.Delete(_progressPath);
+            }
+            catch (Exception exception) when (exception is IOException or UnauthorizedAccessException)
+            {
+                Console.Error.WriteLine($"[LOADER][WARN] Shader prewarm progress clear failed: {exception.Message}");
+            }
+        }
+    }
+
+    private static string ProgressKey(string stamp) =>
+        XxHash3.HashToUInt64(Encoding.UTF8.GetBytes(stamp)).ToString("X16", CultureInfo.InvariantCulture);
+
     public void RecordCompute(ShaderCodeCapture code, ComputePrewarmRecord record)
     {
         var payload = SerializeCompute(record);

src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CommandStream.cs

@@ -40,6 +40,7 @@ internal static unsafe partial class VulkanVideoPresenter
         }
 
         EnsureStarted(1280, 720);
+        WaitForShaderPrewarm("first GPU submission");
         // Queue publication precedes device initialization; only the render thread consumes it.
         lock (_gate)
         {
@@ -117,6 +118,7 @@ internal static unsafe partial class VulkanVideoPresenter
         _ = width;
         _ = height;
         _ = pitchInPixel;
+        WaitForShaderPrewarm("first flip");
         if (!IsKnownDisplayBuffer(address) || !TryGetActivePresenter(out var presenter))
         {
             return false;

src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Present.cs

@@ -157,8 +157,73 @@ internal static unsafe partial class VulkanVideoPresenter
         _guestFlipVersionSequence = 0;
     }
 
+    private static readonly bool _shaderPrewarmWaitDisabled =
+        Environment.GetEnvironmentVariable("SHARPEMU_SHADER_PREWARM_WAIT") == "0";
+    private static bool _shaderPrewarmDecided;
+    private static bool _shaderPrewarmRunning;
+    private static bool _shaderPrewarmReleased;
+    private static int _shaderPrewarmHoldReported;
+    private static int _shaderPrewarmProgress;
+    private static int _shaderPrewarmTotal;
+
+    private static void SetShaderPrewarmState(bool running)
+    {
+        lock (_gate)
+        {
+            _shaderPrewarmDecided = true;
+            _shaderPrewarmRunning = running;
+            Volatile.Write(ref _shaderPrewarmReleased, !running);
+            System.Threading.Monitor.PulseAll(_gate);
+        }
+    }
+
+    internal static void WaitForShaderPrewarm(string holder)
+    {
+        if (_shaderPrewarmWaitDisabled || Volatile.Read(ref _shaderPrewarmReleased))
+        {
+            return;
+        }
+
+        var started = System.Diagnostics.Stopwatch.GetTimestamp();
+        var reported = started;
+        var reporter = false;
+        lock (_gate)
+        {
+            while (_thread is not null && (!_shaderPrewarmDecided || _shaderPrewarmRunning) &&
+                   !_closed && _presenterStartupFailure is null &&
+                   !HostSessionControl.IsShutdownRequested && !Volatile.Read(ref _presenterCloseRequested))
+            {
+                if (!reporter && _shaderPrewarmRunning &&
+                    Interlocked.Exchange(ref _shaderPrewarmHoldReported, 1) == 0)
+                {
+                    reporter = true;
+                    Console.Error.WriteLine(
+                        $"[LOADER][INFO] Shader prewarm: holding the game at its {holder} until it finishes.");
+                }
+
+                System.Threading.Monitor.Wait(_gate, 1000);
+                if (reporter && System.Diagnostics.Stopwatch.GetElapsedTime(reported) >= TimeSpan.FromSeconds(5))
+                {
+                    reported = System.Diagnostics.Stopwatch.GetTimestamp();
+                    var total = Volatile.Read(ref _shaderPrewarmTotal);
+                    var done = Volatile.Read(ref _shaderPrewarmProgress);
+                    Console.Error.WriteLine(
+                        $"[LOADER][INFO] Shader prewarm: {done}/{total} ({(total == 0 ? 0 : done * 100 / total)}%)");
+                }
+            }
+        }
+
+        if (reporter)
+        {
+            Console.Error.WriteLine(string.Create(
+                System.Globalization.CultureInfo.InvariantCulture,
+                $"[LOADER][INFO] Shader prewarm: game released after {System.Diagnostics.Stopwatch.GetElapsedTime(started).TotalSeconds:F1}s."));
+        }
+    }
+
     public static void HideSplashScreen()
     {
+        WaitForShaderPrewarm("splash screen");
         lock (_gate)
         {
             _splashHidden = true;

src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.ShaderPrewarm.cs

@@ -17,6 +17,7 @@ internal static unsafe partial class VulkanVideoPresenter
     private sealed partial class Presenter
     {
         private const int MaxLoggedPrewarmFailures = 8;
+        private const int PrewarmProgressBatch = 256;
 
         private ShaderPrewarmList? _shaderPrewarmList;
         private ConcurrentDictionary<string, byte>? _prewarmedShaderIdentities;
@@ -27,16 +28,25 @@ internal static unsafe partial class VulkanVideoPresenter
         private int _shaderPrewarmCompiled;
         private int _shaderPrewarmFailed;
         private int _shaderPrewarmRuntimeHits;
+        private readonly ConcurrentQueue<ulong> _shaderPrewarmCompleted = new();
 
         ShaderPrewarmList? IShaderPipelineHost.ShaderPrewarm => Volatile.Read(ref _shaderPrewarmList);
 
         private void StartShaderPrewarm()
+        {
+            if (!StartShaderPrewarmCore())
+            {
+                SetShaderPrewarmState(false);
+            }
+        }
+
+        private bool StartShaderPrewarmCore()
         {
             if (_shaderPrewarmList is not null ||
                 _pipelineCacheShardDirectory is not null ||
                 Environment.GetEnvironmentVariable("SHARPEMU_SHADER_PREWARM") == "0")
             {
-                return;
+                return false;
             }
 
             var directory = Path.GetDirectoryName(VulkanPipelineCacheStorage.ResolvePath(
@@ -44,7 +54,7 @@ internal static unsafe partial class VulkanVideoPresenter
                 Environment.GetEnvironmentVariable("SHARPEMU_VK_PIPELINE_CACHE_PATH")));
             if (string.IsNullOrEmpty(directory) || ShaderPrewarmList.Open(directory) is not { } list)
             {
-                return;
+                return false;
             }
 
             _prewarmedShaderIdentities = new ConcurrentDictionary<string, byte>(StringComparer.Ordinal);
@@ -57,7 +67,7 @@ internal static unsafe partial class VulkanVideoPresenter
             {
                 list.WriteStamp(stamp);
                 _shaderPrewarmFinished = true;
-                return;
+                return false;
             }
 
             if (list.IsStampCurrent(stamp))
@@ -65,18 +75,31 @@ internal static unsafe partial class VulkanVideoPresenter
                 _shaderPrewarmFinished = true;
                 Console.Error.WriteLine(
                     $"[LOADER][INFO] Shader prewarm skipped: compute={list.LoadedComputeCount} reason=same-build-and-driver");
-                return;
+                return false;
             }
 
-            var work = new ConcurrentQueue<(ComputePrewarmRecord Record, ShaderCodeCapture Code)>(list.LoadedComputes());
+            var completed = list.ReadProgress(stamp);
+            var pending = list.LoadedComputes()
+                .Where(item => !completed.Contains(ShaderPrewarmList.Identity(item.Record)))
+                .ToList();
+            var started = Stopwatch.GetTimestamp();
+            if (pending.Count == 0)
+            {
+                FinishShaderPrewarm(list, stamp, started);
+                return false;
+            }
+
+            var work = new ConcurrentQueue<(ComputePrewarmRecord Record, ShaderCodeCapture Code)>(pending);
             var workerCount = Math.Clamp(Environment.ProcessorCount / 4, 1, 4);
             var remaining = workerCount;
-            var started = Stopwatch.GetTimestamp();
             var host = (IShaderPipelineHost)this;
             var sharedInt64Atomics = host.SharedInt64AtomicsEnabled;
             var execGuardElision = host.ExecGuardElisionEnabled;
+            Volatile.Write(ref _shaderPrewarmTotal, pending.Count);
+            Volatile.Write(ref _shaderPrewarmProgress, 0);
             Console.Error.WriteLine(
-                $"[LOADER][INFO] Shader prewarm started: compute={work.Count} workers={workerCount}");
+                $"[LOADER][INFO] Shader prewarm started: compute={pending.Count} resumed={completed.Count} workers={workerCount}");
+            SetShaderPrewarmState(true);
             _shaderPrewarmThreads = new Thread[workerCount];
             for (var index = 0; index < workerCount; index++)
             {
@@ -85,8 +108,14 @@ internal static unsafe partial class VulkanVideoPresenter
                     while (!_shaderPrewarmStopping && work.TryDequeue(out var item))
                     {
                         PrewarmComputePipeline(item.Record, item.Code, compiler, sharedInt64Atomics, execGuardElision);
+                        _shaderPrewarmCompleted.Enqueue(ShaderPrewarmList.Identity(item.Record));
+                        if (Interlocked.Increment(ref _shaderPrewarmProgress) % PrewarmProgressBatch == 0)
+                        {
+                            FlushShaderPrewarmProgress(list, stamp);
+                        }
                     }
 
+                    FlushShaderPrewarmProgress(list, stamp);
                     if (Interlocked.Decrement(ref remaining) == 0)
                     {
                         FinishShaderPrewarm(list, stamp, started);
@@ -99,6 +128,19 @@ internal static unsafe partial class VulkanVideoPresenter
                 };
                 _shaderPrewarmThreads[index].Start();
             }
+
+            return true;
+        }
+
+        private void FlushShaderPrewarmProgress(ShaderPrewarmList list, string stamp)
+        {
+            var batch = new List<ulong>();
+            while (_shaderPrewarmCompleted.TryDequeue(out var identity))
+            {
+                batch.Add(identity);
+            }
+
+            list.AppendProgress(stamp, batch);
         }
 
         private static string ShaderPrewarmStampPart(Type type) =>
@@ -109,8 +151,7 @@ internal static unsafe partial class VulkanVideoPresenter
                 '|',
                 ShaderPrewarmStampPart(typeof(Gen5ShaderTranslator)),
                 ShaderPrewarmStampPart(typeof(Gen5SpirvTranslator)),
-                ShaderPrewarmStampPart(compiler.GetType()),
-                ShaderPrewarmStampPart(typeof(ShaderProgramCache)),
+                compiler.GetType().FullName,
                 DriverCacheSignature().TrimEnd('\n'));
 
         private void PrewarmComputePipeline(
@@ -182,11 +223,14 @@ internal static unsafe partial class VulkanVideoPresenter
             var matched = runtime.Keys.Count(prewarmed.ContainsKey);
             if (_shaderPrewarmStopping)
             {
+                SetShaderPrewarmState(false);
                 return;
             }
 
             list.WriteStamp(stamp);
+            list.ClearProgress();
             _shaderPrewarmFinished = true;
+            SetShaderPrewarmState(false);
             _pipelineCacheDirty = true;
             Console.Error.WriteLine(
                 $"[LOADER][INFO] Shader prewarm done: compiled={Volatile.Read(ref _shaderPrewarmCompiled)} " +
@@ -218,6 +262,7 @@ internal static unsafe partial class VulkanVideoPresenter
         private void StopShaderPrewarm()
         {
             _shaderPrewarmStopping = true;
+            SetShaderPrewarmState(false);
             foreach (var thread in _shaderPrewarmThreads)
             {
                 thread.Join(TimeSpan.FromSeconds(10));

tests/SharpEmu.Libs.Tests/Gpu/Pipelines/ShaderPrewarmListTests.cs

@@ -119,6 +119,52 @@ public sealed class ShaderPrewarmListTests : IDisposable
         Assert.False(list.IsStampCurrent("build-b"));
     }
 
+    [Fact]
+    public void ProgressResumesUnderTheSameStampAndClears()
+    {
+        using (var list = Open())
+        {
+            Assert.Empty(list.ReadProgress("build-a"));
+            list.AppendProgress("build-a", [1UL, 2UL]);
+            list.AppendProgress("build-a", [0xFEDC_BA98_7654_3210UL]);
+        }
+
+        using (var reopened = Open())
+        {
+            Assert.Equal(new HashSet<ulong> { 1UL, 2UL, 0xFEDC_BA98_7654_3210UL }, reopened.ReadProgress("build-a"));
+            reopened.ClearProgress();
+            Assert.Empty(reopened.ReadProgress("build-a"));
+        }
+    }
+
+    [Fact]
+    public void ProgressOfAnotherStampIsDiscarded()
+    {
+        using var list = Open();
+        list.AppendProgress("build-a", [7UL]);
+
+        Assert.Empty(list.ReadProgress("build-b"));
+        Assert.Empty(list.ReadProgress("build-a"));
+        list.AppendProgress("build-b", [8UL]);
+        Assert.Equal(new HashSet<ulong> { 8UL }, list.ReadProgress("build-b"));
+    }
+
+    [Fact]
+    public void ARecordIdentityDependsOnItsSpecialization()
+    {
+        using (var list = Open())
+        {
+            _ = CompileAtRuntime(list, BufferDescriptorWords.Format32UInt);
+            _ = CompileAtRuntime(list, BufferDescriptorWords.Format32x4UInt);
+        }
+
+        using var reloaded = Open();
+        var identities = reloaded.LoadedComputes().Select(item => ShaderPrewarmList.Identity(item.Record)).ToArray();
+        Assert.Equal(2, identities.Length);
+        Assert.NotEqual(identities[0], identities[1]);
+        Assert.Equal(identities[0], ShaderPrewarmList.Identity(reloaded.LoadedComputes()[0].Record));
+    }
+
     [Fact]
     public void RecordedReadsReplayAsMergedRanges()
     {

tests/SharpEmu.Libs.Tests/VideoOut/PresenterStartupTests.cs

@@ -30,6 +30,10 @@ public sealed class PresenterStartupTests
             Set("_presenterStartupFailure", null);
             Set("_activePresenter", null);
             Set("_thread", Thread.CurrentThread);
+            Set("_shaderPrewarmDecided", true);
+            Set("_shaderPrewarmRunning", false);
+            Set("_shaderPrewarmReleased", false);
+            Set("_shaderPrewarmHoldReported", 0);
         }
 
         private static FieldInfo Field(string name) => PresenterType.GetField(name, BindingFlags.Static | BindingFlags.NonPublic)!;
@@ -106,6 +110,45 @@ public sealed class PresenterStartupTests
         }
     }
 
+    [Theory]
+    [InlineData(false)]
+    [InlineData(true)]
+    public void ASubmissionWaitsUntilTheShaderPrewarmIsDecidedAndDone(bool undecided)
+    {
+        var runner = new StreamRunner();
+        var queue = new CommandStreamQueue(runner.Host);
+        Assert.True(runner.Host.Memory.TryWrite(StreamRunner.CommandAddress, new byte[] { 0, 16, 0, 192, 0, 0, 0, 0 }));
+        using var state = new StartupState();
+        state.Set(undecided ? "_shaderPrewarmDecided" : "_shaderPrewarmRunning", !undecided);
+        state.Publish(queue);
+        Exception? failure = null;
+        var thread = new Thread(() =>
+        {
+            try { VulkanVideoPresenter.SubmitCommandStream(runner.Host.Memory, 0, StreamRunner.CommandAddress, 2, 1, null); }
+            catch (Exception exception) { failure = exception; }
+        });
+        thread.Start();
+        try
+        {
+            Assert.True(SpinWait.SpinUntil(() => (thread.ThreadState & ThreadState.WaitSleepJoin) != 0, 5000));
+            Assert.False(thread.Join(200));
+            Assert.Equal(0, queue.PendingSubmissionCount);
+            state.Set("_shaderPrewarmDecided", true);
+            state.Set("_shaderPrewarmRunning", false);
+            Assert.True(thread.Join(5000));
+            Assert.Null(failure);
+            Assert.Equal(1, queue.PendingSubmissionCount);
+        }
+        finally
+        {
+            if (thread.IsAlive)
+            {
+                VulkanVideoPresenter.RequestClose();
+                Assert.True(thread.Join(5000));
+            }
+        }
+    }
+
     [Fact]
     public void StartupFailureRejectsTheSubmission()
     {