shifathosn
/
sharpemu
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge content-addressed Vulkan driver caches

This commit is contained in:
kuba 2026-10-01 18:31:19 +02:00
parent b7a0ecadbe 68f31747c6
commit 01745089ee
7 changed files with 237 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/SharpEmu.Libs/VideoOut/VulkanPipelineCacheStorage.cs
View File

@ -7,6 +7,12 @@ internal static class VulkanPipelineCacheStorage
{
private const string CacheFileName = "vulkan-pipeline-cache.bin";
// A guest shader can produce many different native modules as translation
// and specialization change. Importing their shared driver blob recompiles
// obsolete MSL on MoltenVK, so native cache buckets identify emitted code.
internal static string CompiledShaderIdentity(ReadOnlySpan<byte> spirv) =>
Convert.ToHexString(System.Security.Cryptography.SHA256.HashData(spirv));
internal static string ResolvePath(string? titleId, string? configuredPath)
{
if (!string.IsNullOrWhiteSpace(configuredPath))

43
src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Device.Setup.cs
View File

@ -1200,6 +1200,17 @@ internal static unsafe partial class VulkanVideoPresenter
var persistentCacheEnabled =
!string.Equals(cacheMode, "0", StringComparison.Ordinal);
_pipelineCachePath = persistentCacheEnabled ? GetPipelineCachePath() : null;
// MoltenVK imports compile every cached MSL library. Load shader
// groups on demand on macOS so a mature cache cannot hold startup
// behind unrelated scenes. Keep explicit overrides for comparison.
var shardMode = Environment.GetEnvironmentVariable("SHARPEMU_VK_PIPELINE_CACHE_SHARDS");
var useShards = shardMode == "1" || (shardMode is null && OperatingSystem.IsMacOS());
if (_pipelineCachePath is not null &&
useShards)
{
_pipelineCacheShardDirectory = _pipelineCachePath + ".shards";
_pipelineCachePath = Path.Combine(_pipelineCacheShardDirectory, "present.bin");
}
byte[] initialData = [];
try
{
@ -1335,6 +1346,9 @@ internal static unsafe partial class VulkanVideoPresenter
private void SavePipelineCache(bool force)
{
// Cache export must not race asynchronous pipeline creation.
if (_pendingComputePipelines.Values.Any(pending => !pending.Compile.IsCompleted)) return;
SaveGuestPipelineCaches();
if (_pipelineCache.Handle == 0 || string.IsNullOrWhiteSpace(_pipelineCachePath))
{
return;
@ -1345,19 +1359,28 @@ internal static unsafe partial class VulkanVideoPresenter
return;
}
if (SaveDriverPipelineCache(_pipelineCache, _pipelineCachePath))
{
_pipelineCacheDirty = false;
_lastPipelineCacheSaveTick = Environment.TickCount64;
}
}
private bool SaveDriverPipelineCache(PipelineCache cache, string path)
{
try
{
nuint size = 0;
var result = _vk.GetPipelineCacheData(
_device,
_pipelineCache,
cache,
&size,
null);
if (result != Result.Success || size == 0 || size > 256u * 1024u * 1024u)
{
Console.Error.WriteLine(
$"[LOADER][WARN] Vulkan pipeline cache query failed: result={result} size={size}");
return;
return false;
}
var data = new byte[checked((int)size)];
@ -1365,7 +1388,7 @@ internal static unsafe partial class VulkanVideoPresenter
{
result = _vk.GetPipelineCacheData(
_device,
_pipelineCache,
cache,
&size,
dataPointer);
}
@ -1374,7 +1397,7 @@ internal static unsafe partial class VulkanVideoPresenter
{
Console.Error.WriteLine(
$"[LOADER][WARN] Vulkan pipeline cache export failed: {result}");
return;
return false;
}
if (size != (nuint)data.Length)
@ -1382,24 +1405,24 @@ internal static unsafe partial class VulkanVideoPresenter
Array.Resize(ref data, checked((int)size));
}
var directory = Path.GetDirectoryName(_pipelineCachePath);
var directory = Path.GetDirectoryName(path);
if (!string.IsNullOrWhiteSpace(directory))
{
Directory.CreateDirectory(directory);
}
var temporaryPath = _pipelineCachePath + $".{Environment.ProcessId}.tmp";
var temporaryPath = path + $".{Environment.ProcessId}.tmp";
File.WriteAllBytes(temporaryPath, PipelineCacheSignature.Wrap(DriverCacheSignature(), data));
File.Move(temporaryPath, _pipelineCachePath, overwrite: true);
_pipelineCacheDirty = false;
_lastPipelineCacheSaveTick = Environment.TickCount64;
File.Move(temporaryPath, path, overwrite: true);
Console.Error.WriteLine(
$"[LOADER][INFO] Vulkan pipeline cache saved: path={_pipelineCachePath} bytes={data.Length}");
$"[LOADER][INFO] Vulkan pipeline cache saved: path={path} bytes={data.Length}");
return true;
}
catch (Exception exception)
{
Console.Error.WriteLine(
$"[LOADER][WARN] Vulkan pipeline cache save failed: {exception.Message}");
return false;
}
}

2
src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Device.Teardown.cs
View File

@ -26,6 +26,7 @@ internal static unsafe partial class VulkanVideoPresenter
_vk.DeviceWaitIdle(_device);
}
DrainPendingComputePipelines();
SavePipelineCache(force: true);
DrainFrameSlots();
CollectCompletedGuestSubmissions(waitForOldest: false);
@ -58,6 +59,7 @@ internal static unsafe partial class VulkanVideoPresenter
{
_scheduler.Dispose();
_deviceInfo.Slabs.Destroy();
DestroyGuestPipelineCaches();
if (_pipelineCache.Handle != 0)
{
_vk.DestroyPipelineCache(_device, _pipelineCache, null);

118
src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.PipelineCacheShards.cs Normal file
View File

@ -0,0 +1,118 @@
// Copyright (C) 2026 SharpEmu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
namespace SharpEmu.Libs.VideoOut;
using System.Diagnostics;
using SharpEmu.Libs.Gpu.Pipelines;
using Silk.NET.Vulkan;
internal static unsafe partial class VulkanVideoPresenter
{
private sealed partial class Presenter
{
private sealed class DriverCacheShard
{
public required Lazy<PipelineCache> Cache;
public required string Path;
public bool Dirty;
}
private string? _pipelineCacheShardDirectory;
private readonly Dictionary<string, DriverCacheShard> _pipelineCacheShards = new();
private static readonly bool LegacyShaderGroups =
Environment.GetEnvironmentVariable("SHARPEMU_VK_PIPELINE_CACHE_LEGACY_SHARDS") == "1";
private string ComputeCacheKey(ulong guestHash, ulong module) =>
!LegacyShaderGroups && _shaderModuleCacheIdentities.TryGetValue(module, out var identity)
? $"c2-{identity}" : $"c-{guestHash:X16}";
private string GraphicsCacheKey(ulong vertexHash, ulong pixelHash, ulong vertexModule, ulong pixelModule)
{
if (!LegacyShaderGroups && _shaderModuleCacheIdentities.TryGetValue(vertexModule, out var vertex) &&
(pixelModule == 0 || _shaderModuleCacheIdentities.ContainsKey(pixelModule)))
return $"g2-{vertex}-{(pixelModule == 0 ? "none" : _shaderModuleCacheIdentities[pixelModule])}";
return $"g-{vertexHash:X16}-{pixelHash:X16}";
}
// MoltenVK recompiles cached MSL libraries during vkCreatePipelineCache.
// Loading only requested translated modules prevents a mature cache from
// compiling every previously visited scene before the first frame.
// Vulkan still validates the complete shader/layout key inside each blob;
// content hashes only select a storage bucket, never a pipeline to reuse.
private PipelineCache GetGuestPipelineCache(string key)
=> ResolveGuestPipelineCache(GetGuestPipelineCacheSource(key));
// The dictionary belongs to the render thread. Only the lazy native
// creation runs on compiler workers; variants share one initialization.
private Lazy<PipelineCache>? GetGuestPipelineCacheSource(string key)
{
if (_pipelineCacheShardDirectory is null) return null;
if (_pipelineCacheShards.TryGetValue(key, out var existing))
{
existing.Dirty = true;
return existing.Cache;
}
var path = Path.Combine(_pipelineCacheShardDirectory, key + ".bin");
var source = new Lazy<PipelineCache>(() => LoadGuestPipelineCache(key, path));
_pipelineCacheShards.Add(key, new DriverCacheShard { Cache = source, Path = path, Dirty = true });
return source;
}
private PipelineCache ResolveGuestPipelineCache(Lazy<PipelineCache>? source)
{
var cache = source?.Value ?? default;
return cache.Handle != 0 ? cache : _pipelineCache;
}
private PipelineCache LoadGuestPipelineCache(string key, string path)
{
byte[] data = [];
try
{
if (File.Exists(path) &&
!PipelineCacheSignature.TryUnwrap(DriverCacheSignature(), File.ReadAllBytes(path), out data))
Console.Error.WriteLine($"[LOADER][INFO] Vulkan cache shard invalidated: path={path}");
}
catch (Exception exception) when (exception is IOException or UnauthorizedAccessException)
{
Console.Error.WriteLine($"[LOADER][WARN] Vulkan cache shard read failed: {exception.Message}");
}
var started = Stopwatch.GetTimestamp();
var result = TryCreatePipelineCache(data, out var cache);
if (result != Result.Success && data.Length != 0)
result = TryCreatePipelineCache([], out cache);
if (result != Result.Success)
{
Console.Error.WriteLine($"[LOADER][WARN] Vulkan cache shard unavailable: key={key} result={result}");
return default;
}
if (data.Length != 0)
Console.Error.WriteLine($"[LOADER][INFO] Vulkan cache shard loaded: key={key} bytes={data.Length} ms={Stopwatch.GetElapsedTime(started).TotalMilliseconds:F1}");
return cache;
}
private void SaveGuestPipelineCaches()
{
foreach (var shard in _pipelineCacheShards.Values)
{
if (shard.Dirty && shard.Cache.IsValueCreated && shard.Cache.Value.Handle != 0 &&
SaveDriverPipelineCache(shard.Cache.Value, shard.Path))
shard.Dirty = false;
}
}
private void DestroyGuestPipelineCaches()
{
foreach (var shard in _pipelineCacheShards.Values)
{
if (shard.Cache.IsValueCreated && shard.Cache.Value.Handle != 0)
_vk.DestroyPipelineCache(_device, shard.Cache.Value, null);
}
_pipelineCacheShards.Clear();
}
}
}

37
src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Pipelines.cs
View File

@ -56,6 +56,7 @@ internal static unsafe partial class VulkanVideoPresenter
private readonly Dictionary<ulong, ShaderModule> _shaderModules = new();
private readonly Dictionary<ulong, int> _shaderModuleSpirvBytes = new();
private readonly Dictionary<ulong, string> _shaderModuleCacheIdentities = new();
private long _pipelineCreationMilliseconds;
private KhrPushDescriptor _pushDescriptorApi = null!;
private uint _maxPushDescriptors;
@ -171,6 +172,7 @@ internal static unsafe partial class VulkanVideoPresenter
SetDebugName(ObjectType.ShaderModule, module.Handle, $"SharpEmu {stage} 0x{hash:X16}");
_shaderModules.Add(programId, module);
_shaderModuleSpirvBytes[module.Handle] = shader.Payload.Length;
_shaderModuleCacheIdentities[module.Handle] = VulkanPipelineCacheStorage.CompiledShaderIdentity(shader.Payload);
return module.Handle;
}
@ -786,7 +788,9 @@ internal static unsafe partial class VulkanVideoPresenter
Layout = layout,
};
var graphicsStart = Stopwatch.GetTimestamp();
Check(_vk.CreateGraphicsPipelines(_device, _pipelineCache, 1, &pipelineInfo, null, out var pipeline),
var cache = GetGuestPipelineCache(GraphicsCacheKey(description.VertexStage.Hash,
description.PixelStage?.Hash ?? 0, vertexModule.Handle, pixelModule.Handle));
Check(_vk.CreateGraphicsPipelines(_device, cache, 1, &pipelineInfo, null, out var pipeline),
$"vkCreateGraphicsPipelines(rendering vs=0x{description.VertexStage.Hash:X16} ps=0x{description.PixelStage?.Hash ?? 0:X16})");
ReportPipelineCreation(
(long)Stopwatch.GetElapsedTime(graphicsStart).TotalMilliseconds,
@ -849,7 +853,8 @@ internal static unsafe partial class VulkanVideoPresenter
Layout = layout,
};
var computeStart = Stopwatch.GetTimestamp();
Check(_vk.CreateComputePipelines(_device, _pipelineCache, 1, &pipelineInfo, null, out pipeline), $"vkCreateComputePipelines(rendering) hash=0x{description.Stage.Hash:X16}");
var cache = GetGuestPipelineCache(ComputeCacheKey(description.Stage.Hash, computeModule.Handle));
Check(_vk.CreateComputePipelines(_device, cache, 1, &pipelineInfo, null, out pipeline), $"vkCreateComputePipelines(rendering) hash=0x{description.Stage.Hash:X16}");
ReportPipelineCreation(
(long)Stopwatch.GetElapsedTime(computeStart).TotalMilliseconds,
"compute",
@ -878,8 +883,8 @@ internal static unsafe partial class VulkanVideoPresenter
// One compute pipeline whose vkCreateComputePipelines call runs on a worker
// thread. Everything the command stream owns (descriptor and pipeline layout,
// the module handle) is prepared before the task starts, so the task touches
// nothing but the Vulkan pipeline creation itself.
// the module handle) is prepared before the task starts. The worker loads
// its optional driver-cache shard and creates the native pipeline.
private sealed class PendingComputePipeline
{
public required DescriptorSetLayout SetLayout;
@ -958,7 +963,7 @@ internal static unsafe partial class VulkanVideoPresenter
}
var device = _device;
var cache = _pipelineCache;
var cacheSource = GetGuestPipelineCacheSource(ComputeCacheKey(description.Stage.Hash, computeModule.Handle));
var vk = _vk;
var started = new PendingComputePipeline
{
@ -980,6 +985,9 @@ internal static unsafe partial class VulkanVideoPresenter
var compileStart = Stopwatch.GetTimestamp();
try
{
// Importing a MoltenVK cache compiles its MSL libraries.
// Keep that work inside the same bounded compiler slot.
var cache = ResolveGuestPipelineCache(cacheSource);
return CompileComputePipeline(vk, device, cache, computeModule, layout);
}
finally
@ -1036,6 +1044,25 @@ internal static unsafe partial class VulkanVideoPresenter
}
}
private void DrainPendingComputePipelines()
{
foreach (var pending in _pendingComputePipelines.Values)
{
try
{
var pipeline = pending.Compile.GetAwaiter().GetResult();
if (pipeline.Handle != 0) _vk.DestroyPipeline(_device, pipeline, null);
}
catch (Exception exception)
{
Console.Error.WriteLine($"[LOADER][WARN] Pending compute pipeline failed during shutdown: {exception.Message}");
}
_vk.DestroyPipelineLayout(_device, pending.Layout, null);
_vk.DestroyDescriptorSetLayout(_device, pending.SetLayout, null);
}
_pendingComputePipelines.Clear();
}
private void DestroyRenderPipelines()
{
foreach (var entry in _pipelineEntries.Values)

3
tests/SharpEmu.Libs.Tests/VideoOut/PresenterUnderTest.cs
View File

@ -84,6 +84,9 @@ internal sealed class PresenterUnderTest : IDisposable
field.SetValue(Instance, Activator.CreateInstance(field.FieldType, nonPublic: true));
}
SetField("_shaderModuleCacheIdentities", new Dictionary<ulong, string>());
var cacheShards = PresenterType.GetField("_pipelineCacheShards", InstanceMembers)!;
cacheShards.SetValue(Instance, Activator.CreateInstance(cacheShards.FieldType, nonPublic: true));
forwarder.Target = this;
}

43
tests/SharpEmu.Libs.Tests/VideoOut/VulkanPipelineCacheShardTests.cs Normal file
View File

@ -0,0 +1,43 @@
// Copyright (C) 2026 SharpEmu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
using SharpEmu.Libs.Tests.Gpu.Images;
using SharpEmu.Libs.Tests.Gpu.Vulkan;
using Silk.NET.Vulkan;
using Xunit;
namespace SharpEmu.Libs.Tests.VideoOut;
public sealed class VulkanPipelineCacheShardTests(HeadlessVulkanFixture fixture) : IClassFixture<HeadlessVulkanFixture>
{
[Fact]
public void ShardsLoadLazilyShareInitializationAndRecoverFromInvalidData()
{
var vulkan = fixture.Vulkan;
if (!GatePrerequisites.Ready(vulkan)) return;
var directory = Path.Combine(Path.GetTempPath(), "SharpEmuTests", Guid.NewGuid().ToString("N"));
using var presenter = new PresenterUnderTest(vulkan);
presenter.SetField("_pipelineCacheShardDirectory", directory);
try
{
Directory.CreateDirectory(directory);
var path = Path.Combine(directory, "test.bin");
File.WriteAllBytes(path, [1, 2, 3]);
var source = (Lazy<PipelineCache>)presenter.InvokeMethod("GetGuestPipelineCacheSource", "test")!;
Assert.False(source.IsValueCreated);
Assert.Same(source, presenter.InvokeMethod("GetGuestPipelineCacheSource", "test"));
Assert.NotEqual(0ul, source.Value.Handle);
presenter.InvokeMethod("SaveGuestPipelineCaches");
Assert.True(new FileInfo(path).Length > 3);
presenter.InvokeMethod("DestroyGuestPipelineCaches");
var reloaded = (Lazy<PipelineCache>)presenter.InvokeMethod("GetGuestPipelineCacheSource", "test")!;
Assert.NotSame(source, reloaded);
Assert.NotEqual(0ul, reloaded.Value.Handle);
}
finally
{
presenter.InvokeMethod("DestroyGuestPipelineCaches");
Directory.Delete(directory, recursive: true);
}
}
}