diff --git a/src/SharpEmu.ShaderCompiler/Resources/ScalarGraphDiskCache.cs b/src/SharpEmu.ShaderCompiler/Resources/ScalarGraphDiskCache.cs new file mode 100644 index 0000000..5c876e4 --- /dev/null +++ b/src/SharpEmu.ShaderCompiler/Resources/ScalarGraphDiskCache.cs @@ -0,0 +1,120 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; + +namespace SharpEmu.ShaderCompiler.Resources; + +internal static class ScalarGraphDiskCache +{ + private const int Version = 1; + private const int MaxEntryBytes = 32 * 1024 * 1024; + private const long MaxCacheBytes = 256L * 1024 * 1024; + private static readonly string CachePath = Environment.GetEnvironmentVariable("SHARPEMU_SHADER_ANALYSIS_CACHE_PATH") ?? + Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "SharpEmu", "shader-analysis"); + private static readonly bool Trace = Environment.GetEnvironmentVariable("SHARPEMU_PROFILE_SHADER_PLANNING") == "1"; + private static readonly object WriteLock = new(); + + internal static ScalarValueGraph Build(Gen5ShaderProgram program, uint userDataBase, uint userDataCount, + IReadOnlySet? fixedFunctionVertexLoads, uint waveSize, string? cachePath = null) + { + var directory = cachePath ?? CachePath; + if (directory == "0") return ScalarValueGraph.Build(program, userDataBase, userDataCount, fixedFunctionVertexLoads, waveSize); + var key = Key(program, userDataBase, userDataCount, fixedFunctionVertexLoads, waveSize); + var path = Path.Combine(directory, key + ".graph"); + try + { + using var file = File.OpenRead(path); + if (file.Length is > 32 and <= MaxEntryBytes) + { + var digest = new byte[32]; + file.ReadExactly(digest); + var bytes = new byte[checked((int)file.Length - 32)]; + file.ReadExactly(bytes); + if (CryptographicOperations.FixedTimeEquals(digest, SHA256.HashData(bytes))) + { + using var reader = new BinaryReader(new MemoryStream(bytes, false)); + var cached = ScalarValueGraph.ReadSnapshot(reader, program, userDataBase, userDataCount, fixedFunctionVertexLoads, waveSize); + if (Trace) Console.Error.WriteLine($"[SHADER_ANALYSIS_CACHE] hit key={key} bytes={bytes.Length}"); + return cached; + } + } + } + catch (Exception error) when (IsCacheFailure(error)) { } + + var graph = ScalarValueGraph.Build(program, userDataBase, userDataCount, fixedFunctionVertexLoads, waveSize); + try + { + using var stream = new MemoryStream(); + using (var writer = new BinaryWriter(stream, Encoding.UTF8, true)) graph.WriteSnapshot(writer); + if (stream.Length <= MaxEntryBytes - 32) + { + var bytes = stream.GetBuffer().AsSpan(0, (int)stream.Length); + lock (WriteLock) + { + Directory.CreateDirectory(directory); + // Keep a bounded cache across compiler versions too. No live + // analysis objects or guest memory are retained by the cache. + var entries = new DirectoryInfo(directory).GetFiles("*.graph").OrderBy(f => f.LastWriteTimeUtc).ToArray(); + var total = entries.Sum(f => f.Length) + stream.Length + 32; + foreach (var entry in entries) + { + if (total <= MaxCacheBytes) break; + total -= entry.Length; + entry.Delete(); + } + var temporary = path + "." + Guid.NewGuid().ToString("N") + ".tmp"; + try + { + using (var file = new FileStream(temporary, FileMode.CreateNew, FileAccess.Write, FileShare.None)) + { + file.Write(SHA256.HashData(bytes)); + file.Write(bytes); + } + File.Move(temporary, path, true); + } + finally { File.Delete(temporary); } + } + if (Trace) Console.Error.WriteLine($"[SHADER_ANALYSIS_CACHE] miss key={key} bytes={bytes.Length}"); + } + } + catch (Exception error) when (IsCacheFailure(error)) { } + return graph; + } + + internal static string Key(Gen5ShaderProgram program, uint userDataBase, uint userDataCount, + IReadOnlySet? fixedFunctionVertexLoads, uint waveSize) + { + using var stream = new MemoryStream(); + using (var writer = new BinaryWriter(stream, Encoding.UTF8, true)) + { + writer.Write(Version); + writer.Write(typeof(ScalarValueGraph).Module.ModuleVersionId.ToByteArray()); + writer.Write(userDataBase); writer.Write(userDataCount); writer.Write(waveSize); + var loads = fixedFunctionVertexLoads?.Order().ToArray() ?? []; + writer.Write(loads.Length); + foreach (var load in loads) writer.Write(load); + writer.Write(program.Instructions.Count); + foreach (var instruction in program.Instructions) + { + writer.Write(instruction.Pc); writer.Write((int)instruction.Encoding); writer.Write(instruction.Opcode); + writer.Write(instruction.Words.Count); + foreach (var word in instruction.Words) writer.Write(word); + writer.Write(instruction.Sources.Count); + foreach (var operand in instruction.Sources) { writer.Write((int)operand.Kind); writer.Write(operand.Value); } + writer.Write(instruction.Destinations.Count); + foreach (var operand in instruction.Destinations) { writer.Write((int)operand.Kind); writer.Write(operand.Value); } + // Controls are a closed set of decoded record types. Include the + // runtime type and every property, including variable NSA arrays. + writer.Write(instruction.Control?.GetType().FullName ?? ""); + writer.Write(instruction.Control is { } control ? JsonSerializer.Serialize(control, control.GetType()) : ""); + } + } + return Convert.ToHexString(SHA256.HashData(stream.GetBuffer().AsSpan(0, (int)stream.Length))); + } + + private static bool IsCacheFailure(Exception error) => error is IOException or UnauthorizedAccessException or + InvalidDataException or ArgumentException or OverflowException; +} diff --git a/src/SharpEmu.ShaderCompiler/Resources/ScalarValueGraph.Snapshot.cs b/src/SharpEmu.ShaderCompiler/Resources/ScalarValueGraph.Snapshot.cs new file mode 100644 index 0000000..9d2b146 --- /dev/null +++ b/src/SharpEmu.ShaderCompiler/Resources/ScalarValueGraph.Snapshot.cs @@ -0,0 +1,134 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.ShaderCompiler.Ir; + +namespace SharpEmu.ShaderCompiler.Resources; + +public sealed partial class ScalarValueGraph +{ + // Snapshots are taken immediately after Builder.Run, before planning mutates + // memory entries or substitutes nodes. References use snapshot-local indices, + // never process-local IDs. Two-pass restoration preserves loop phi cycles. + internal void WriteSnapshot(BinaryWriter writer) + { + var indices = new Dictionary(_values.Count); + for (var i = 0; i < _values.Count; i++) indices.Add(_values[i], i); + void Reference(ScalarValue? value) => writer.Write(value is null ? -1 : indices[value]); + writer.Write(_values.Count); + foreach (var value in _values) + { + writer.Write((byte)value.Kind); + writer.Write((byte)value.Type); + writer.Write((byte)value.Operation); + writer.Write(value.Payload); + writer.Write(value.Operands.Length); + foreach (var operand in value.Operands) Reference(operand); + writer.Write(value.PhiPredecessors.Length); + foreach (var predecessor in value.PhiPredecessors) writer.Write(predecessor); + } + writer.Write(_internedValues.Count); + foreach (var entry in _internedValues) + { + Reference(entry.Value); + writer.Write(entry.Payload); + writer.Write(entry.Identity); + } + writer.Write(Accesses.Length); + foreach (var access in Accesses) + { + writer.Write(access is not null); + if (access is null) continue; + Reference(access.Handle); + Reference(access.SamplerHandle); + Reference(access.Read); + Reference(access.Offset); + Reference(access.Active); + } + writer.Write(BranchConditions.Count); + foreach (var (pc, value) in BranchConditions) { writer.Write(pc); Reference(value); } + writer.Write(_undefinedOrigins.Count); + foreach (var (value, origin) in _undefinedOrigins) + { + Reference(value); writer.Write(origin.Pc); writer.Write(origin.Opcode); + } + writer.Write(_bitScanInstructions.Count); + foreach (var (value, pc) in _bitScanInstructions) { Reference(value); writer.Write(pc); } + writer.Write(BuilderInstruction.Pc); + writer.Write(BuilderInstruction.Opcode ?? ""); + } + + internal static ScalarValueGraph ReadSnapshot(BinaryReader reader, Gen5ShaderProgram program, + uint userDataBase, uint userDataCount, IReadOnlySet? fixedFunctionVertexLoads, uint waveSize) + { + var graph = new ScalarValueGraph(program, + IrControlFlowGraph.Build(program.Instructions, Gen5IrBranchResolver.Instance), + MemoryAccessTable.Build(program, fixedFunctionVertexLoads), userDataBase, userDataCount, waveSize); + int Count(int maximum = 1_000_000) + { + var count = reader.ReadInt32(); + if (count < 0 || count > maximum || count > reader.BaseStream.Length - reader.BaseStream.Position) + throw new InvalidDataException("Invalid graph snapshot count."); + return count; + } + var count = Count(); + var operands = new int[count][]; + for (var i = 0; i < count; i++) + { + var kind = (ScalarValueKind)reader.ReadByte(); + var type = (ScalarValueType)reader.ReadByte(); + var operation = (ScalarOperation)reader.ReadByte(); + if (!Enum.IsDefined(kind) || !Enum.IsDefined(type) || !Enum.IsDefined(operation)) + throw new InvalidDataException("Invalid graph snapshot node."); + var payload = reader.ReadUInt64(); + var refs = new int[Count(65536)]; + for (var j = 0; j < refs.Length; j++) refs[j] = reader.ReadInt32(); + operands[i] = refs; + var predecessors = new int[Count(65536)]; + for (var j = 0; j < predecessors.Length; j++) predecessors[j] = reader.ReadInt32(); + if (kind == ScalarValueKind.Phi ? predecessors.Length != refs.Length : predecessors.Length != 0) + throw new InvalidDataException("Invalid graph snapshot phi."); + var value = ScalarValue.CreateInterned(kind, type, operation, payload, new ScalarValue[refs.Length]); + if (kind == ScalarValueKind.Phi) value.SetPhiOperands(predecessors, value.Operands); + graph._values.Add(value); + } + ScalarValue Node(int index) => (uint)index < (uint)count ? graph._values[index] + : throw new InvalidDataException("Invalid graph snapshot reference."); + ScalarValue? Reference() + { + var index = reader.ReadInt32(); + return index == -1 ? null : Node(index); + } + ScalarValue Required() => Reference() ?? throw new InvalidDataException("Missing graph snapshot node."); + for (var i = 0; i < count; i++) + for (var j = 0; j < operands[i].Length; j++) graph._values[i].Operands[j] = Node(operands[i][j]); + var internedCount = Count(count); + for (var i = 0; i < internedCount; i++) + { + var value = Required(); + var payload = reader.ReadUInt64(); + var identity = reader.ReadUInt64(); + var hash = InternHash(value.Kind, value.Type, payload, value.Operation, value.Operands, identity); + var head = graph._interned.TryGetValue(hash, out var first) ? first : -1; + graph._interned[hash] = graph._internedValues.Count; + graph._internedValues.Add(new(value, payload, identity, head)); + } + var accessCount = Count(); + if (accessCount != graph.Memory.Count) throw new InvalidDataException("Graph memory table changed."); + graph.Accesses = new MemoryAccessBinding?[accessCount]; + for (var i = 0; i < accessCount; i++) + if (reader.ReadBoolean()) graph.Accesses[i] = new(Reference(), Reference(), Reference(), Reference(), Reference()); + var branches = Count(); + for (var i = 0; i < branches; i++) graph.BranchConditions.Add(reader.ReadUInt32(), Required()); + var origins = Count(count); + for (var i = 0; i < origins; i++) graph._undefinedOrigins.Add(Required(), (reader.ReadUInt32(), reader.ReadString())); + var bitScans = Count(count); + for (var i = 0; i < bitScans; i++) graph._bitScanInstructions.Add(Required(), reader.ReadUInt32()); + var pc = reader.ReadUInt32(); + var opcode = reader.ReadString(); + graph.BuilderInstruction = (pc, opcode.Length == 0 ? null! : opcode); + if (reader.BaseStream.Position != reader.BaseStream.Length) + throw new InvalidDataException("Trailing graph snapshot data."); + return graph; + } +} diff --git a/src/SharpEmu.ShaderCompiler/Resources/ScalarValueGraph.cs b/src/SharpEmu.ShaderCompiler/Resources/ScalarValueGraph.cs index d0aa865..48841ec 100644 --- a/src/SharpEmu.ShaderCompiler/Resources/ScalarValueGraph.cs +++ b/src/SharpEmu.ShaderCompiler/Resources/ScalarValueGraph.cs @@ -386,14 +386,7 @@ public sealed partial class ScalarValueGraph private ScalarValue Intern(ScalarValueKind kind, ScalarValueType type, ulong payload = 0, ScalarOperation operation = ScalarOperation.None, ReadOnlySpan operands = default, ulong identity = 0) { - var hash = new HashCode(); - hash.Add(kind); - hash.Add(type); - hash.Add(payload); - hash.Add(operation); - hash.Add(identity); - foreach (var operand in operands) hash.Add(operand.Id); - var key = hash.ToHashCode(); + var key = InternHash(kind, type, payload, operation, operands, identity); var head = _interned.TryGetValue(key, out var first) ? first : -1; for (var index = head; index >= 0; index = _internedValues[index].Next) { @@ -412,6 +405,19 @@ public sealed partial class ScalarValueGraph return created; } + private static int InternHash(ScalarValueKind kind, ScalarValueType type, ulong payload, + ScalarOperation operation, ReadOnlySpan operands, ulong identity) + { + var hash = new HashCode(); + hash.Add(kind); + hash.Add(type); + hash.Add(payload); + hash.Add(operation); + hash.Add(identity); + foreach (var operand in operands) hash.Add(operand.Id); + return hash.ToHashCode(); + } + private ScalarValue Track(ScalarValue value) { _values.Add(value); diff --git a/src/SharpEmu.ShaderCompiler/Resources/ShaderResourcePlan.cs b/src/SharpEmu.ShaderCompiler/Resources/ShaderResourcePlan.cs index 8089357..f9bb8ba 100644 --- a/src/SharpEmu.ShaderCompiler/Resources/ShaderResourcePlan.cs +++ b/src/SharpEmu.ShaderCompiler/Resources/ShaderResourcePlan.cs @@ -48,7 +48,7 @@ public sealed class ShaderResourcePlan public static ShaderResourcePlan Extract(Gen5ShaderProgram program, ShaderStage stage, ulong hash, uint userDataBase, uint userDataCount, IReadOnlySet? fixedFunctionVertexLoads = null, Action? beforeResourceTracking = null, uint waveSize = 64) { - var graph = ScalarValueGraph.Build(program, userDataBase, userDataCount, fixedFunctionVertexLoads, waveSize); + var graph = ScalarGraphDiskCache.Build(program, userDataBase, userDataCount, fixedFunctionVertexLoads, waveSize); var plan = new ShaderResourcePlan(graph, stage, hash); var reads = ResourceTableReadPlanner.Plan(graph, stage, hash); var memo = new Dictionary(); diff --git a/tests/SharpEmu.ShaderCompiler.Tests/Resources/ScalarGraphDiskCacheTests.cs b/tests/SharpEmu.ShaderCompiler.Tests/Resources/ScalarGraphDiskCacheTests.cs new file mode 100644 index 0000000..cd9479e --- /dev/null +++ b/tests/SharpEmu.ShaderCompiler.Tests/Resources/ScalarGraphDiskCacheTests.cs @@ -0,0 +1,84 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.ShaderCompiler.Resources; +using Xunit; +using static SharpEmu.ShaderCompiler.Tests.Resources.ResourceTestProgram; + +namespace SharpEmu.ShaderCompiler.Tests.Resources; + +public sealed class ScalarGraphDiskCacheTests +{ + [Fact] + public void SnapshotPreservesCyclesInterningAndInstructionProvenance() + { + var program = Program(ScalarLoad(0, 0, 4), EndProgram(8)); + var graph = ScalarValueGraph.Build(program, 0, 4); + var phi = graph.Phi(0, ScalarValueType.U32); + phi.SetPhiOperands([1, 2], [graph.UserData(0), phi]); + graph.BranchConditions[0] = phi; + graph.BuilderInstruction = (4, "unsupported"); + var undefined = graph.Undefined(ScalarValueType.U32); + var scan = graph.FindLowestSetBit(graph.UserData(1), 12); + graph.BranchConditions[4] = undefined; + graph.BranchConditions[12] = scan; + using var stream = new MemoryStream(); + using (var writer = new BinaryWriter(stream, System.Text.Encoding.UTF8, true)) graph.WriteSnapshot(writer); + stream.Position = 0; + using var reader = new BinaryReader(stream); + var restored = ScalarValueGraph.ReadSnapshot(reader, program, 0, 4, null, 64); + var loop = restored.BranchConditions[0]; + Assert.Same(loop, loop.Operands[1]); + Assert.Equal(new[] { 1, 2 }, loop.PhiPredecessors); + Assert.Same(restored.UserData(0), loop.Operands[0]); + Assert.Same(restored.BranchConditions[4], restored.Undefined(ScalarValueType.U32)); + Assert.True(restored.TryGetUndefinedOrigin(restored.BranchConditions[4], out var origin)); + Assert.Equal((4u, "unsupported"), origin); + Assert.Same(restored.BranchConditions[12], restored.FindLowestSetBit(restored.UserData(1), 12)); + Assert.Equal(graph.Accesses.Length, restored.Accesses.Length); + Assert.NotSame(graph.Accesses[0]!.Read, restored.Accesses[0]!.Read); + Assert.True(restored.Equivalent(graph.Accesses[0]!.Read!, restored.Accesses[0]!.Read!)); + } + + [Fact] + public void CacheReusesSnapshotAndRebuildsTruncatedOrCorruptEntries() + { + var directory = Path.Combine(Path.GetTempPath(), "sharpemu-graph-" + Guid.NewGuid().ToString("N")); + try + { + var program = Program(ScalarLoad(0, 0, 4), EndProgram(8)); + var first = ScalarGraphDiskCache.Build(program, 0, 4, null, 64, directory); + var file = Assert.Single(Directory.GetFiles(directory, "*.graph")); + var bytes = File.ReadAllBytes(file); + var timestamp = File.GetLastWriteTimeUtc(file); + var second = ScalarGraphDiskCache.Build(program, 0, 4, null, 64, directory); + Assert.Equal(timestamp, File.GetLastWriteTimeUtc(file)); + Assert.NotSame(first, second); + Assert.Equal(first.Values.Count, second.Values.Count); + foreach (var corrupt in new[] { bytes[..20], bytes.Select((b, i) => i == 40 ? (byte)(b ^ 1) : b).ToArray() }) + { + File.WriteAllBytes(file, corrupt); + var rebuilt = ScalarGraphDiskCache.Build(program, 0, 4, null, 64, directory); + Assert.Equal(first.Values.Count, rebuilt.Values.Count); + Assert.Equal(bytes, File.ReadAllBytes(file)); + } + } + finally { if (Directory.Exists(directory)) Directory.Delete(directory, true); } + } + + [Fact] + public void KeysIncludeEffectiveInstructionsControlsWaveAndFixedFunctionLoads() + { + var instruction = ScalarLoad(0, 0, 4); + var program = Program(instruction, EndProgram(8)); + string Key(Gen5ShaderProgram p, uint wave = 64, IReadOnlySet? loads = null, uint count = 4) => + ScalarGraphDiskCache.Key(p, 0, count, loads, wave); + var key = Key(program); + Assert.NotEqual(key, Key(program, 32)); + Assert.NotEqual(key, Key(program, count: 8)); + Assert.NotEqual(key, Key(program, loads: new HashSet { 0 })); + Assert.NotEqual(key, Key(Program(instruction with { Opcode = "SBufferLoadDword" }, EndProgram(8)))); + Assert.NotEqual(key, Key(Program(instruction with { Control = new Gen5ScalarMemoryControl(1, 32, null) }, EndProgram(8)))); + Assert.Equal(key, Key(program with { Address = program.Address + 4096 })); + } +}