From 593cec957d937e12aa8013139bfa27fde34b34df Mon Sep 17 00:00:00 2001 From: Astellou Date: Thu, 8 Oct 2026 23:50:28 +0200 Subject: [PATCH] [GPU] Run the command stream on its own thread and speed up draw execution ASTRO BOT crowd scene: ~25 FPS -> ~55-58 FPS. - The PM4 command stream is interpreted on its own thread. Draws, dispatches, barriers, end-of-pipe writes and the interpreter's stores are handed to the render thread in stream order through a short queue (64 items); callbacks that return a result wait for the work before them. - Graphics program resolution runs on a worker thread ahead of the draws, from copy-on-write register bank snapshots, and is validated against the GPU ownership state before use. - The resource materialization cache rebases buffer base addresses instead of missing on every moved descriptor. - DCC metadata fills that cannot be absorbed run on the host, so binding the surface no longer waits for a GPU readback. - Cheaper per-draw work: cached pipeline lookups no longer wait on program preparation, redundant dynamic state is skipped, image bindings and target requests are memoized, fewer allocations. - Guest thread stack/TLS slots are chosen and mapped under one lock, so two threads can no longer share a stack. - Wave lowering split into native wave32, native wave64 and emulated wave64. --- .../Cpu/Native/DirectExecutionBackend.cs | 30 + .../Agc/AgcExports.CommandStream.cs | 244 ++- .../Gpu/Buffers/GuestBufferCache.cs | 185 +- .../GpuCommandInterpreter.Adaptations.cs | 6 +- .../GpuCommandInterpreter.Registers.cs | 16 +- .../GpuCommandInterpreter.Waits.cs | 2 +- .../Gpu/GpuCommands/GpuCommandInterpreter.cs | 25 +- .../Gpu/GpuCommands/ICommandStreamHost.cs | 8 + .../GpuCommands/Registers/ContextRegisters.cs | 12 +- .../GpuCommands/Registers/RegisterBanks.cs | 311 +++- .../Registers/RegisterWriteTable.cs | 18 +- .../Registers/RegisterWriters.Context.cs | 118 +- .../Registers/ShaderProgramRegisters.cs | 10 + .../Gpu/Images/CachedImage.Transfers.cs | 13 +- .../Gpu/Images/ColorToMultisampleDepthBlit.cs | 1 + .../Gpu/Images/GuestImageCache.Lookup.cs | 19 + .../Gpu/Images/GuestImageCache.Metadata.cs | 11 +- .../Gpu/Images/GuestImageCache.Transfers.cs | 1 + .../ImageRequestBuilders.ColorTarget.cs | 51 +- .../ImageRequestBuilders.DepthTarget.cs | 24 +- .../Images/ImageRequestBuilders.Texture.cs | 3 +- src/SharpEmu.Libs/Gpu/Images/SamplerStore.cs | 12 +- .../Gpu/Images/TextureTransferLayout.cs | 3 +- src/SharpEmu.Libs/Gpu/Images/TileGeometry.cs | 18 + .../Gpu/Pipelines/IShaderPipelineHost.cs | 3 + .../Gpu/Pipelines/PipelineKeys.cs | 30 +- .../Gpu/Pipelines/ShaderPipelineCache.cs | 210 ++- .../Gpu/Pipelines/ShaderProgramCache.cs | 62 +- .../Gpu/Rendering/ParallelProgramPrefetch.cs | 282 ++++ .../Gpu/Rendering/RenderExecutor.Compute.cs | 10 +- .../Gpu/Rendering/RenderExecutor.Prefetch.cs | 115 ++ .../Gpu/Rendering/RenderExecutor.Resources.cs | 5 +- .../Gpu/Rendering/RenderExecutor.cs | 27 +- .../Gpu/Rendering/RenderingState.cs | 24 +- .../Gpu/Scheduling/GpuWritebackEpoch.cs | 16 + .../Gpu/Scheduling/SubmissionScheduler.cs | 12 + .../Gpu/Vulkan/GraphicsDynamicStateEpoch.cs | 16 + .../VulkanVideoPresenter.CachedImages.cs | 31 + .../VulkanVideoPresenter.CommandStream.cs | 416 ++++- .../VulkanVideoPresenter.CommandThread.cs | 511 ++++++ .../VulkanVideoPresenter.ConcurrentResolve.cs | 134 ++ .../VulkanVideoPresenter.Descriptors.cs | 136 +- .../VulkanVideoPresenter.Device.Setup.cs | 7 +- .../VulkanVideoPresenter.Draws.Recording.cs | 4 + .../VulkanVideoPresenter.GuestSubmissions.cs | 16 +- .../VulkanVideoPresenter.Pipelines.cs | 61 +- .../VulkanVideoPresenter.Present.Hdr.cs | 1 + .../VulkanVideoPresenter.RenderLoop.cs | 80 +- .../VulkanVideoPresenter.Rendering.cs | 106 +- .../VulkanVideoPresenter.Scheduler.cs | 2 + .../VulkanVideoPresenter.ShaderPrewarm.cs | 5 +- .../Gen5SpirvTranslator.Alu.cs | 102 +- .../Gen5SpirvTranslator.CommonWave.cs | 27 + ...5SpirvTranslator.EmulatedWave64Lowering.cs | 578 +++++++ ...en5SpirvTranslator.NativeWave32Lowering.cs | 156 ++ ...en5SpirvTranslator.NativeWave64Lowering.cs | 160 ++ .../Gen5SpirvTranslator.Resources.cs | 34 +- .../Gen5SpirvTranslator.WaveLowering.cs | 34 + .../Gen5SpirvTranslator.cs | 465 +---- .../Gen5PixelInputMapping.cs | 14 +- .../Resources/BindingLayout.cs | 15 +- .../Resources/ResourceMaterializationCache.cs | 1504 ++++++++++++++++- .../ShaderCompileRequest.cs | 2 + .../Registers/RegisterBanksSnapshotTests.cs | 236 +++ .../Images/GuestImageCacheTests.Metadata.cs | 47 + .../Gpu/Pipelines/ShaderPipelineCacheTests.cs | 3 + .../Rendering/RenderExecutorComputeTests.cs | 16 + .../Gpu/Rendering/RenderExecutorDrawTests.cs | 16 +- .../Rendering/RenderExecutorPrefetchTests.cs | 87 + .../Rendering/RenderExecutorTestSupport.cs | 32 +- .../VideoOut/PresenterImageBindingTests.cs | 35 + .../Gen5WaveLoweringTests.cs | 250 +++ .../ResourceMaterializationCacheTests.cs | 290 ++++ 73 files changed, 6651 insertions(+), 915 deletions(-) create mode 100644 src/SharpEmu.Libs/Gpu/Rendering/ParallelProgramPrefetch.cs create mode 100644 src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Prefetch.cs create mode 100644 src/SharpEmu.Libs/Gpu/Scheduling/GpuWritebackEpoch.cs create mode 100644 src/SharpEmu.Libs/Gpu/Vulkan/GraphicsDynamicStateEpoch.cs create mode 100644 src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CommandThread.cs create mode 100644 src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.ConcurrentResolve.cs create mode 100644 src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.CommonWave.cs create mode 100644 src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.EmulatedWave64Lowering.cs create mode 100644 src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.NativeWave32Lowering.cs create mode 100644 src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.NativeWave64Lowering.cs create mode 100644 src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.WaveLowering.cs create mode 100644 tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/Registers/RegisterBanksSnapshotTests.cs create mode 100644 tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorPrefetchTests.cs create mode 100644 tests/SharpEmu.ShaderCompiler.Tests/Gen5WaveLoweringTests.cs diff --git a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.cs b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.cs index 331b974..c807f76 100644 --- a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.cs +++ b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.cs @@ -5604,6 +5604,11 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I return false; } + // Map does not reject a range that is already mapped: it reuses and zero-fills it. Two threads + // creating guest threads at once could both find the same slot free and share one stack, so + // choosing and mapping a slot happen under one lock. + private static readonly object GuestThreadRegionGate = new(); + private static bool TryMapGuestThreadRegion( IVirtualMemory virtualMemory, ulong baseAddress, @@ -5611,6 +5616,20 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I ProgramHeaderFlags protection, out ulong mappedBase, out string? error) + { + lock (GuestThreadRegionGate) + { + return TryMapGuestThreadRegionLocked(virtualMemory, baseAddress, size, protection, out mappedBase, out error); + } + } + + private static bool TryMapGuestThreadRegionLocked( + IVirtualMemory virtualMemory, + ulong baseAddress, + ulong size, + ProgramHeaderFlags protection, + out ulong mappedBase, + out string? error) { for (int i = 0; i < GuestThreadRegionSlots; i++) { @@ -5645,6 +5664,17 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I IVirtualMemory virtualMemory, out ulong tlsBase, out string? error) + { + lock (GuestThreadRegionGate) + { + return TryMapGuestThreadTlsRegionLocked(virtualMemory, out tlsBase, out error); + } + } + + private static bool TryMapGuestThreadTlsRegionLocked( + IVirtualMemory virtualMemory, + out ulong tlsBase, + out string? error) { for (int i = 0; i < GuestThreadRegionSlots; i++) { diff --git a/src/SharpEmu.Libs/Agc/AgcExports.CommandStream.cs b/src/SharpEmu.Libs/Agc/AgcExports.CommandStream.cs index 73abc74..ec78bd1 100644 --- a/src/SharpEmu.Libs/Agc/AgcExports.CommandStream.cs +++ b/src/SharpEmu.Libs/Agc/AgcExports.CommandStream.cs @@ -23,6 +23,21 @@ public static partial class AgcExports internal sealed record RetainedTargetlessDraw(RegisterBanks Banks, TargetlessDrawArguments Arguments); + // A draw or dispatch the translation deferred, with the banks it was issued under. + internal abstract record DeferredWork; + + // Takes the deferred work of a command stream interpreted on its own thread. + internal interface IDeferredWorkSink + { + // True on the thread that interprets the stream; elsewhere the translation runs work itself. + bool IsProducerThread { get; } + + // The non-draw items handed over so far; a draw's resolution waits until they ran. + long Gate { get; } + + void EnqueueDeferred(DeferredWork work, bool isDraw); + } + // The snapshots the submit-time prepass captured for one submission. private sealed record SubmittedGeometrySnapshots( Dictionary? IndexSnapshots, @@ -160,9 +175,18 @@ public static partial class AgcExports { _pipelines = new ShaderPipelineCache(_context, pipelineHost, GuestGpu.Current, CreateShaderHeaderRegistry(_context)); _executor = new RenderExecutor(renderHost, _pipelines); + if (host is IConcurrentResolveHost concurrentHost) + { + _parallel = new ParallelProgramPrefetch(_executor, renderHost, concurrentHost); + _executor.Prefetch = _parallel; + } } } + // Resolves the programs of deferred draws ahead of them, for a host that answers reads from + // another thread; draws are deferred whenever it exists. + private readonly ParallelProgramPrefetch? _parallel; + // The context bank of the queue whose slice runs now; the Metal host rebuilds its records from it. internal ContextRegisters? CurrentContextRegisters => _current?.TypedRegisters?.Context; @@ -196,17 +220,194 @@ public static partial class AgcExports } } + private sealed record PendingDraw( + ulong SubmitId, + SubmittedDcbState State, + RegisterBanks Banks, + bool Indexed, + DrawIndexedArguments IndexedArguments, + DrawAutoArguments AutoArguments, + SubmittedVertexSnapshot? VertexSnapshot, + SubmittedIndexSnapshot? IndexSnapshot, + ParallelProgramPrefetch.Job? Job) : DeferredWork; + + private sealed record PendingDispatch( + ulong SubmitId, + SubmittedDcbState State, + RegisterBanks Banks, + uint GroupsX, + uint GroupsY, + uint GroupsZ, + uint DispatchInitiator, + ulong IndirectArgumentsAddress) : DeferredWork; + + // With the command stream on its own thread, its draws and dispatches reach the render + // thread through the sink, which runs them with RunDeferred in stream order. + internal IDeferredWorkSink? Sink { get; set; } + + private bool Producing => Sink is { IsProducerThread: true }; + + // The queue state of the work the render thread runs now; _current belongs to the thread + // interpreting the stream. + private SubmittedDcbState? _executing; + + public void RunDeferred(DeferredWork work) + { + switch (work) + { + case PendingDraw draw: + { + var executing = _executing; + _executing = draw.State; + RecordKnownColorTargets(draw.State, draw.Banks); + if (_parallel is not null) + { + _parallel.Current = draw.Job; + } + + try + { + RunExecutorDraw(draw.SubmitId, draw.State, draw.Banks, draw.Indexed, draw.IndexedArguments, draw.AutoArguments, + clearSnapshots: false); + } + finally + { + _executing = executing; + if (_parallel is not null) + { + _parallel.Current = null; + } + } + + break; + } + case PendingDispatch dispatch: + { + var executing = _executing; + _executing = dispatch.State; + try + { + ExecuteDispatch(_executor!, dispatch.SubmitId, dispatch.Banks, dispatch.GroupsX, dispatch.GroupsY, dispatch.GroupsZ, + dispatch.DispatchInitiator, dispatch.IndirectArgumentsAddress); + } + finally + { + _executing = executing; + } + + break; + } + default: + throw _host.Fatal($"The deferred work is unknown: {work.GetType().Name}."); + } + } + + // The resolution worker, so the render thread can order it with the stream's other work. + internal ParallelProgramPrefetch? Prefetch => _parallel; + + private readonly Queue _pendingDraws = new(); + + public bool HasPendingDraws => _pendingDraws.Count != 0; + + private bool _draining; + + // Runs every deferred draw in issue order with the banks and snapshots it was issued with. + // A draw calls back into the host, whose entry points drain too: those calls run inside + // the draw and must not start the next one. + public void DrainPendingDraws() + { + if (_draining) + { + return; + } + + _draining = true; + try + { + DrainPendingDrawsCore(); + } + finally + { + _draining = false; + } + } + + private void DrainPendingDrawsCore() + { + while (_pendingDraws.TryDequeue(out var pending)) + { + var current = _current; + var state = pending.State; + _current = state; + state.CurrentVertexSnapshot = pending.VertexSnapshot; + state.CurrentIndexSnapshot = pending.IndexSnapshot; + if (_parallel is not null) + { + _parallel.Current = pending.Job; + } + + try + { + RunExecutorDraw(pending.SubmitId, state, pending.Banks, pending.Indexed, pending.IndexedArguments, pending.AutoArguments); + } + finally + { + _current = current; + if (_parallel is not null) + { + _parallel.Current = null; + } + } + } + + // What follows a drain may change guest memory and cache state the worker reads. + _parallel?.WaitIdle(); + } + private bool TryRunExecutorDraw(ulong submitId, SubmittedDcbState state, bool indexed, in DrawIndexedArguments indexedArguments, in DrawAutoArguments autoArguments) { - if (_executor is not { } executor) + if (_executor is null) { return false; } var banks = RequireTypedRegisters(state); - RecordKnownColorTargets(state, banks); + var producing = Producing; + if (!producing) + { + // A draw run by the render thread records its targets there. + RecordKnownColorTargets(state, banks); + } + state.FrameDrawCount++; state.SawIndexedDraw |= indexed; + if (_parallel is not null || producing) + { + var snapshot = banks.Snapshot(); + var pending = new PendingDraw(submitId, state, snapshot, indexed, indexedArguments, autoArguments, + state.CurrentVertexSnapshot, state.CurrentIndexSnapshot, _parallel?.Submit(snapshot, producing ? Sink!.Gate : 0)); + if (producing) + { + Sink!.EnqueueDeferred(pending, isDraw: true); + } + else + { + _pendingDraws.Enqueue(pending); + } + + state.CurrentIndexSnapshot = null; + state.CurrentVertexSnapshot = null; + return true; + } + + RunExecutorDraw(submitId, state, banks, indexed, indexedArguments, autoArguments); + return true; + } + + private void RunExecutorDraw(ulong submitId, SubmittedDcbState state, RegisterBanks banks, bool indexed, in DrawIndexedArguments indexedArguments, in DrawAutoArguments autoArguments, + bool clearSnapshots = true) + { + var executor = _executor!; var drawStarted = DcbParseProfile.Begin(); try { @@ -222,17 +423,19 @@ public static partial class AgcExports finally { DcbParseProfile.RecordDraw(drawStarted); - state.CurrentIndexSnapshot = null; - state.CurrentVertexSnapshot = null; + // The snapshot fields belong to the interpreting thread when it is another one. + if (clearSnapshots) + { + state.CurrentIndexSnapshot = null; + state.CurrentVertexSnapshot = null; + } } - - return true; } // Keeps the banks the draw was issued under; the flip replays it into the display buffer. internal void RetainTargetlessDraw(RegisterBanks banks, in TargetlessDrawArguments arguments) { - var state = RequireCurrent(); + var state = _executing ?? RequireCurrent(); if (RenderTrace.Enabled) { RenderTrace.Write($"TargetlessRetention submit={arguments.SubmitId} replaced={state.RetainedTargetlessDraw is not null} previousSubmit={state.RetainedTargetlessDraw?.Arguments.SubmitId} export=0x{banks.Shader.Vertex.ExportAddress:X16} pixel=0x{banks.Shader.Pixel.Address:X16}"); @@ -363,22 +566,35 @@ public static partial class AgcExports { var banks = RequireTypedRegisters(state); state.FrameDispatchCount++; - var executorStarted = DcbParseProfile.Begin(); - try + if (Producing) { - executor.Dispatch(submitId, banks, endX, endY, endZ, dispatchInitiator, indirectArgumentsAddress); - } - finally - { - DcbParseProfile.RecordDispatch(executorStarted); + var snapshot = banks.Snapshot(); + Sink!.EnqueueDeferred(new PendingDispatch(submitId, state, snapshot, endX, endY, endZ, dispatchInitiator, indirectArgumentsAddress), + isDraw: false); + return; } + ExecuteDispatch(executor, submitId, banks, endX, endY, endZ, dispatchInitiator, indirectArgumentsAddress); return; } throw _host.Fatal("The command stream has no render executor."); } + private static void ExecuteDispatch(RenderExecutor executor, ulong submitId, RegisterBanks banks, uint endX, uint endY, uint endZ, + uint dispatchInitiator, ulong indirectArgumentsAddress) + { + var executorStarted = DcbParseProfile.Begin(); + try + { + executor.Dispatch(submitId, banks, endX, endY, endZ, dispatchInitiator, indirectArgumentsAddress); + } + finally + { + DcbParseProfile.RecordDispatch(executorStarted); + } + } + // The retained draw runs into the display buffer with the target words it was last bound with. private void ReplayRetainedTargetlessDraw(RenderExecutor executor, SubmittedDcbState state, int handle, int displayBufferIndex) { diff --git a/src/SharpEmu.Libs/Gpu/Buffers/GuestBufferCache.cs b/src/SharpEmu.Libs/Gpu/Buffers/GuestBufferCache.cs index 183f607..69b9cdb 100644 --- a/src/SharpEmu.Libs/Gpu/Buffers/GuestBufferCache.cs +++ b/src/SharpEmu.Libs/Gpu/Buffers/GuestBufferCache.cs @@ -333,6 +333,7 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable throw SubmissionScheduler.Fatal("The host DMA write range is invalid."); } + GpuWritebackEpoch.Advance(); if (!_backing.TryWriteBacking(guestAddress, data)) { throw SubmissionScheduler.Fatal($"Could not write the required direct backing: addr=0x{guestAddress:X16} size=0x{data.Length:X16}"); @@ -413,26 +414,71 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable if (images.OverlapsDccMetadata(guestAddress, size) && TryWriteFillToBacking(guestAddress, size, value)) { _gpuModifiedRanges.Remove(guestAddress, size); - var firstPage = guestAddress & ~(TrackerLayout.PageBytes - 1); - for (var page = firstPage; page < guestAddress + size; page += TrackerLayout.PageBytes) - { - if (!_gpuModifiedRanges.Overlaps(page, TrackerLayout.PageBytes)) - { - _tracker.ClearGpuDirtyPages(page, TrackerLayout.PageBytes); - } - } + Interlocked.Increment(ref _gpuModifiedVersion); + ClearGpuDirtyPagesOfRemovedRange(guestAddress, size); + } + } + + // After a range left the GPU-modified set: the pages inside it are clean in one tracker walk, + // and only a page the range shares at either end can still hold another GPU-modified range. + private void ClearGpuDirtyPagesOfRemovedRange(ulong guestAddress, ulong size) + { + const ulong pageBytes = TrackerLayout.PageBytes; + var end = guestAddress + size; + var firstPage = guestAddress & ~(pageBytes - 1); + var lastPage = (end - 1) & ~(pageBytes - 1); + var startPartial = guestAddress != firstPage; + var endPartial = end != lastPage + pageBytes; + var innerBegin = startPartial ? firstPage + pageBytes : firstPage; + var innerEnd = endPartial ? lastPage : end; + if (innerBegin < innerEnd) + { + _tracker.ClearGpuDirtyPages(innerBegin, innerEnd - innerBegin); + } + + if (startPartial) + { + ClearGpuDirtyPageIfUnmodified(firstPage); + } + + if (endPartial && (lastPage != firstPage || !startPartial)) + { + ClearGpuDirtyPageIfUnmodified(lastPage); + } + } + + private void ClearGpuDirtyPageIfUnmodified(ulong page) + { + if (!_gpuModifiedRanges.Overlaps(page, TrackerLayout.PageBytes)) + { + _tracker.ClearGpuDirtyPages(page, TrackerLayout.PageBytes); } } public void FillDccMetadata(ulong guestAddress, ulong size, uint value) { if (guestAddress == 0 || (guestAddress & 3) != 0 || size == 0 || (size & 3) != 0 || size > ulong.MaxValue - guestAddress || - HasGpuDirtyBytes(guestAddress, size) || RequireImageCache().QueryRegion(guestAddress, size).ImageBytes) + HasGpuDirtyBytes(guestAddress, size)) { FillBuffer(guestAddress, size, value, false); return; } + // Images over the range read it again; only GPU-owned image bytes need the general fill. + // The buffer copy is filled on the GPU rather than uploaded from the written bytes. + var images = RequireImageCache(); + var region = images.QueryRegion(guestAddress, size); + if (region.GpuImageBytes) + { + FillBuffer(guestAddress, size, value, false); + return; + } + + if (region.ImageBytes) + { + images.InvalidateMemory(guestAddress, size); + } + var (destination, destinationOffset) = ObtainBuffer(guestAddress, size, false, false, FindBuffer(guestAddress, size)); destination.Fill(destinationOffset, size, value); if (!TryWriteFillToBacking(guestAddress, size, value)) @@ -443,21 +489,31 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable private bool TryWriteFillToBacking(ulong guestAddress, ulong size, uint value) { - var values = new uint[(int)Math.Min(size / sizeof(uint), 4096)]; - Array.Fill(values, value); - var bytes = MemoryMarshal.AsBytes(values); - for (ulong offset = 0; offset < size;) + // Metadata fills repeat every frame: the pattern buffer is pooled. + var count = (int)Math.Min(size / sizeof(uint), 4096); + var values = System.Buffers.ArrayPool.Shared.Rent(count); + try { - var chunk = (int)Math.Min(size - offset, (ulong)bytes.Length); - if (!_backing.TryWriteBacking(guestAddress + offset, bytes[..chunk])) + values.AsSpan(0, count).Fill(value); + var bytes = MemoryMarshal.AsBytes(values.AsSpan(0, count)); + for (ulong offset = 0; offset < size;) { - return false; + var chunk = (int)Math.Min(size - offset, (ulong)bytes.Length); + GpuWritebackEpoch.Advance(); + if (!_backing.TryWriteBacking(guestAddress + offset, bytes[..chunk])) + { + return false; + } + + offset += (ulong)chunk; } - offset += (ulong)chunk; + return true; + } + finally + { + System.Buffers.ArrayPool.Shared.Return(values); } - - return true; } public const ulong MaxHostCopyWords = 64 * 1024; @@ -1241,6 +1297,7 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable var copies = pending.Copies; readback.Complete(pending.Ticket, (index, bytes) => { + GpuWritebackEpoch.Advance(); if (!_backing.TryWriteBacking(copies[index].Address, bytes)) { throw SubmissionScheduler.Fatal($"Could not write the required direct backing: addr=0x{copies[index].Address:X16} size=0x{(ulong)bytes.Length:X16}"); @@ -1249,6 +1306,7 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable foreach (var copy in copies) { _gpuModifiedRanges.Remove(copy.Address, copy.Size); + Interlocked.Increment(ref _gpuModifiedVersion); } _tracker.ClearGpuDirtyPages(pending.WindowBegin, pending.WindowEnd - pending.WindowBegin); @@ -1456,6 +1514,7 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable foreach (var copy in copies) { _gpuModifiedRanges.Remove(copy.Address, copy.Size); + Interlocked.Increment(ref _gpuModifiedVersion); } return; @@ -1492,6 +1551,7 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable foreach (var copy in copies) { _gpuModifiedRanges.Remove(copy.Address, copy.Size); + Interlocked.Increment(ref _gpuModifiedVersion); } } @@ -1590,6 +1650,7 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable readback.Read(pieces, waitTick, (index, bytes) => { + GpuWritebackEpoch.Advance(); if (!_backing.TryWriteBacking(copies[index].Address, bytes)) { throw SubmissionScheduler.Fatal($"Could not write the required direct backing: addr=0x{copies[index].Address:X16} size=0x{(ulong)bytes.Length:X16}"); @@ -1622,6 +1683,7 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable var placement = copy.Placement; var offset = baseOffset + placement.DataOffset; _download.Invalidate(offset, placement.DataSize); + GpuWritebackEpoch.Advance(); if (!_backing.TryWriteBacking(copy.Address, _download.Mapped.Slice((int)offset, (int)placement.DataSize))) { throw SubmissionScheduler.Fatal($"Could not write the required direct backing: addr=0x{copy.Address:X16} size=0x{placement.DataSize:X16}"); @@ -1713,20 +1775,79 @@ public sealed unsafe class GuestBufferCache : IGuestBufferStore, IDisposable profileScope.SwitchPhase(isWritten ? RenderPhaseProfile.Phase.BufferDirtySyncWritten : isTexelBuffer ? RenderPhaseProfile.Phase.BufferDirtySyncTexel : RenderPhaseProfile.Phase.BufferDirtySyncUpload); - var copies = new List(); - var totalSize = 0UL; - GpuBuffer? source = null; - _tracker.ForEachUploadRange( - guestAddress, - size, - isWritten, - (address, bytes) => + var collector = _uploadCollector; + if (collector is null || collector.InUse) + { + // The first call on a thread keeps its collector; a nested one gets its own. + collector = new UploadCollector(); + _uploadCollector ??= collector; + } + + collector.Begin(this, buffer, guestAddress, size); + try + { + return SynchronizeBufferCore(collector, buffer, guestAddress, size, isWritten, isTexelBuffer, preserveCpuWriteHotPages, startedAt); + } + finally + { + collector.End(); + } + } + + // The ranges of one synchronization. The delegates are bound once, so a call that has dirty + // pages to upload allocates no closure and no list. + private sealed class UploadCollector + { + private GuestBufferCache _cache = null!; + private GpuBuffer _buffer = null!; + private ulong _guestAddress; + private ulong _size; + public readonly List Copies = new(); + public ulong TotalSize; + public GpuBuffer? Source; + public bool InUse; + public readonly Action Collect; + public readonly Action Upload; + + public UploadCollector() + { + Collect = (address, bytes) => { - copies.Add(new BufferCopy(totalSize, buffer.Offset(address), bytes)); - totalSize += bytes; - }, - () => source = _uploader.PrepareSource(buffer.CpuAddress, CollectionsMarshal.AsSpan(copies), totalSize, guestAddress, size), - preserveCpuWriteHotPages); + Copies.Add(new BufferCopy(TotalSize, _buffer.Offset(address), bytes)); + TotalSize += bytes; + }; + Upload = () => Source = _cache._uploader.PrepareSource(_buffer.CpuAddress, CollectionsMarshal.AsSpan(Copies), TotalSize, _guestAddress, _size); + } + + public void Begin(GuestBufferCache cache, GpuBuffer buffer, ulong guestAddress, ulong size) + { + InUse = true; + _cache = cache; + _buffer = buffer; + _guestAddress = guestAddress; + _size = size; + } + + public void End() + { + InUse = false; + _cache = null!; + _buffer = null!; + Copies.Clear(); + TotalSize = 0; + Source = null; + } + } + + [ThreadStatic] private static UploadCollector? _uploadCollector; + + private bool SynchronizeBufferCore(UploadCollector collector, GpuBuffer buffer, ulong guestAddress, ulong size, bool isWritten, + bool isTexelBuffer, bool preserveCpuWriteHotPages, long startedAt) + { + var copies = collector.Copies; + _tracker.ForEachUploadRange(guestAddress, size, isWritten, collector.Collect, collector.Upload, preserveCpuWriteHotPages); + var source = collector.Source; + var totalSize = collector.TotalSize; if (source != null) { buffer.NoteGpuWrite(); diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Adaptations.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Adaptations.cs index 268275b..23adb2a 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Adaptations.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Adaptations.cs @@ -26,7 +26,7 @@ public sealed partial class GpuCommandInterpreter } // A compare-swap loop applies once per try and suspends while the compare fails. - if (!atomic.TryApply(_host.TryReadGuest, _host.Memory, out var priorValue, out _, out var comparePassed)) + if (!atomic.TryApply(_host.TryReadGuestOperand, _host.Memory, out var priorValue, out _, out var comparePassed)) { throw _host.Fatal($"The atomic address cannot be accessed: address=0x{atomic.Address:X16} size={atomic.ByteCount}."); } @@ -68,7 +68,7 @@ public sealed partial class GpuCommandInterpreter if (semaphore.IsSignal) { - if (!Packets.MemorySemaphorePacket.TrySignal(_host.TryReadGuest, _host.Memory, semaphore.Address, semaphore.WriteSignal, out _)) + if (!Packets.MemorySemaphorePacket.TrySignal(_host.TryReadGuestOperand, _host.Memory, semaphore.Address, semaphore.WriteSignal, out _)) { throw _host.Fatal($"The semaphore cannot be signaled: address=0x{semaphore.Address:X16}."); } @@ -77,7 +77,7 @@ public sealed partial class GpuCommandInterpreter } // A wait takes one token, or suspends until a signal adds one. - if (!Packets.MemorySemaphorePacket.TryConsume(_host.TryReadGuest, _host.Memory, semaphore.Address, out var priorValue)) + if (!Packets.MemorySemaphorePacket.TryConsume(_host.TryReadGuestOperand, _host.Memory, semaphore.Address, out var priorValue)) { throw _host.Fatal($"The semaphore cannot be read: address=0x{semaphore.Address:X16}."); } diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Registers.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Registers.cs index a168128..9b42efc 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Registers.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Registers.cs @@ -82,8 +82,8 @@ public sealed partial class GpuCommandInterpreter var next = packet.PacketAddress + (ulong)packet.Length * sizeof(uint); var expectedHeader = PacketHeader.Make(centroidPacketDwords, PacketOpcode.SetContextRegister); if (packet.Remaining < packet.Length + centroidPacketDwords || - ReadDword(next) != expectedHeader || - RegisterBankLayout.Normalize(ReadDword(next + sizeof(uint))) != ContextRegisterOffset.PaScCentroidPriority0) + ReadDword(next, RenderPhaseProfile.CommandReadKind.Payload) != expectedHeader || + RegisterBankLayout.Normalize(ReadDword(next + sizeof(uint), RenderPhaseProfile.CommandReadKind.Payload)) != ContextRegisterOffset.PaScCentroidPriority0) { throw _host.Fatal($"The sample location packet is not followed by the centroid priority packet: header=0x{packet.Header:X8} address=0x{packet.PacketAddress:X16} next=0x{next:X16}."); } @@ -102,16 +102,16 @@ public sealed partial class GpuCommandInterpreter var address = packet.PacketAddress; var registerHeader = PacketHeader.Make(3, PacketOpcode.SetContextRegister); - if (ReadDword(address + 40) != registerHeader || ReadDword(address + 44) != RegisterBankLayout.DepthInfo || - ReadDword(address + 52) != registerHeader || ReadDword(address + 56) != RegisterBankLayout.DepthView || - ReadDword(address + 64) != registerHeader || ReadDword(address + 68) != RegisterBankLayout.HtileDataBase || - ReadDword(address + 76) != registerHeader || ReadDword(address + 80) != RegisterBankLayout.HtileSurface || - ReadDword(address + 88) != PacketHeader.Make(2, PacketOpcode.Nop)) + if (ReadDword(address + 40, RenderPhaseProfile.CommandReadKind.Payload) != registerHeader || ReadDword(address + 44, RenderPhaseProfile.CommandReadKind.Payload) != RegisterBankLayout.DepthInfo || + ReadDword(address + 52, RenderPhaseProfile.CommandReadKind.Payload) != registerHeader || ReadDword(address + 56, RenderPhaseProfile.CommandReadKind.Payload) != RegisterBankLayout.DepthView || + ReadDword(address + 64, RenderPhaseProfile.CommandReadKind.Payload) != registerHeader || ReadDword(address + 68, RenderPhaseProfile.CommandReadKind.Payload) != RegisterBankLayout.HtileDataBase || + ReadDword(address + 76, RenderPhaseProfile.CommandReadKind.Payload) != registerHeader || ReadDword(address + 80, RenderPhaseProfile.CommandReadKind.Payload) != RegisterBankLayout.HtileSurface || + ReadDword(address + 88, RenderPhaseProfile.CommandReadKind.Payload) != PacketHeader.Make(2, PacketOpcode.Nop)) { return; } - var sizeXy = ReadDword(address + 92); + var sizeXy = ReadDword(address + 92, RenderPhaseProfile.CommandReadKind.Payload); if (sizeXy != 0) { TypedRegisters.CompositeDepthSizeXy = sizeXy; diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Waits.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Waits.cs index 7e6f8fd..97b7928 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Waits.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.Waits.cs @@ -28,7 +28,7 @@ public sealed partial class GpuCommandInterpreter return; } - var value = is64Bit ? ReadQword(address) : ReadDword(address); + var value = ReadWaitOperand(address, is64Bit); if (!WaitOperation.TryCompare(value, reference, mask, compareFunction, out var satisfied)) { throw _host.Fatal($"The wait compare function is unknown: function={compareFunction} address=0x{address:X16}."); diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.cs index ef91e56..d10d372 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/GpuCommandInterpreter.cs @@ -394,7 +394,12 @@ public sealed partial class GpuCommandInterpreter { RenderPhaseProfile.RecordCommandRead(readKind, sizeof(uint)); Span bytes = stackalloc byte[sizeof(uint)]; - if (!_host.TryReadGuest(address, bytes)) + // The packets themselves are command data; anything else is an operand earlier work may write. + var read = readKind is RenderPhaseProfile.CommandReadKind.Header or RenderPhaseProfile.CommandReadKind.Payload or + RenderPhaseProfile.CommandReadKind.RegisterTable + ? _host.TryReadGuest(address, bytes) + : _host.TryReadGuestOperand(address, bytes); + if (!read) { throw _host.Fatal($"The command stream cannot read guest memory: address=0x{address:X16} size=4."); } @@ -406,7 +411,7 @@ public sealed partial class GpuCommandInterpreter { RenderPhaseProfile.RecordCommandRead(RenderPhaseProfile.CommandReadKind.Operand64, sizeof(ulong)); Span bytes = stackalloc byte[sizeof(ulong)]; - if (!_host.TryReadGuest(address, bytes)) + if (!_host.TryReadGuestOperand(address, bytes)) { throw _host.Fatal($"The command stream cannot read guest memory: address=0x{address:X16} size=8."); } @@ -414,10 +419,24 @@ public sealed partial class GpuCommandInterpreter return BinaryPrimitives.ReadUInt64LittleEndian(bytes); } + private ulong ReadWaitOperand(ulong address, bool is64Bit) + { + RenderPhaseProfile.RecordCommandRead(is64Bit ? RenderPhaseProfile.CommandReadKind.Operand64 : RenderPhaseProfile.CommandReadKind.Operand32, + is64Bit ? sizeof(ulong) : sizeof(uint)); + Span bytes = stackalloc byte[sizeof(ulong)]; + var size = is64Bit ? sizeof(ulong) : sizeof(uint); + if (!_host.TryReadGuestWaitOperand(address, bytes[..size])) + { + throw _host.Fatal($"The command stream cannot read guest memory: address=0x{address:X16} size={size}."); + } + + return is64Bit ? BinaryPrimitives.ReadUInt64LittleEndian(bytes) : BinaryPrimitives.ReadUInt32LittleEndian(bytes); + } + internal void ReadBytes(ulong address, Span destination) { RenderPhaseProfile.RecordCommandRead(RenderPhaseProfile.CommandReadKind.Other, destination.Length); - if (!_host.TryReadGuest(address, destination)) + if (!_host.TryReadGuestOperand(address, destination)) { throw _host.Fatal($"The command stream cannot read guest memory: address=0x{address:X16} size={destination.Length}."); } diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/ICommandStreamHost.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/ICommandStreamHost.cs index 6196265..74eda4d 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/ICommandStreamHost.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/ICommandStreamHost.cs @@ -84,6 +84,14 @@ public interface ICommandStreamHost // Reads guest memory the GPU may have written; the host synchronizes GPU-owned pages first. bool TryReadGuest(ulong address, Span destination); + // Reads a packet operand (a label, a predicate, indirect arguments): memory earlier work may + // write, unlike the packets themselves. A host that defers work completes it first. + bool TryReadGuestOperand(ulong address, Span destination) => TryReadGuest(address, destination); + + // Reads the label a wait polls. A stale value only suspends the wait until a retry, so a host + // that runs work behind the stream need not complete it first. + bool TryReadGuestWaitOperand(ulong address, Span destination) => TryReadGuestOperand(address, destination); + // Runs commands other threads posted to this worker. Called before every packet. void RunPendingCommands(); diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/ContextRegisters.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/ContextRegisters.cs index 5ad7830..1e6f6a5 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/ContextRegisters.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/ContextRegisters.cs @@ -338,8 +338,15 @@ public sealed class ScreenViewportRegisters public ScreenViewportRegisters Copy() { - var copy = (ScreenViewportRegisters)MemberwiseClone(); + var copy = CopySharingViewports(); copy.Viewports = (ViewportRegisters[])Viewports.Clone(); + return copy; + } + + // A copy that shares the viewport array, the largest part, until RegisterBanks copies it. + public ScreenViewportRegisters CopySharingViewports() + { + var copy = (ScreenViewportRegisters)MemberwiseClone(); copy.ClipRectangleLeft = (int[])ClipRectangleLeft.Clone(); copy.ClipRectangleTop = (int[])ClipRectangleTop.Clone(); copy.ClipRectangleRight = (int[])ClipRectangleRight.Clone(); @@ -388,6 +395,9 @@ public sealed class ContextRegisters public EnhancedQualityAntialiasingRegisters EnhancedQualityAntialiasing; public ShaderInterfaceRegisters ShaderInterface = new(); + // A copy that shares every nested part; RegisterBanks copies a part before writing it. + public ContextRegisters ShallowCopy() => (ContextRegisters)MemberwiseClone(); + public uint RenderTargetMaskForSlot(uint slot) => (RenderTargetMask >> (int)(slot * 4)) & 0xFu; public ContextRegisters Copy() diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterBanks.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterBanks.cs index 8715653..7c20201 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterBanks.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterBanks.cs @@ -1,6 +1,8 @@ // Copyright (C) 2026 SharpEmu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +using SharpEmu.Libs.Gpu.Images; + namespace SharpEmu.Libs.Gpu.GpuCommands.Registers; // The three typed register banks of one queue, with the state the register writers carry between packets. @@ -9,15 +11,31 @@ public sealed class RegisterBanks private ContextRegisters? _savedContext; private uint? _savedCompositeDepthSizeXy; - public RegisterBanks(Func fatal) => Fatal = fatal; + public RegisterBanks(Func fatal) + : this(fatal, new ContextRegisters(), new ShaderProgramRegisters(), new UserConfigRegisters()) + { + } + + // Banks over given parts, so a copy or a snapshot does not first build empty ones. + private RegisterBanks( + Func fatal, + ContextRegisters context, + ShaderProgramRegisters shader, + UserConfigRegisters userConfig) + { + Fatal = fatal; + Context = context; + Shader = shader; + UserConfig = userConfig; + } public Func Fatal { get; } - public ContextRegisters Context { get; private set; } = new(); + public ContextRegisters Context { get; private set; } - public ShaderProgramRegisters Shader { get; private set; } = new(); + public ShaderProgramRegisters Shader { get; private set; } - public UserConfigRegisters UserConfig { get; private set; } = new(); + public UserConfigRegisters UserConfig { get; private set; } public bool ContextPushed => _savedContext is not null; @@ -31,11 +49,8 @@ public sealed class RegisterBanks // A copy of the live banks; the saved context and the draw-time markers are not part of it. public RegisterBanks Clone() { - var clone = new RegisterBanks(Fatal) + var clone = new RegisterBanks(Fatal, Context.Copy(), Shader.Copy(), UserConfig.Copy()) { - Context = Context.Copy(), - Shader = Shader.Copy(), - UserConfig = UserConfig.Copy(), CompositeDepthSizeXy = CompositeDepthSizeXy, UserDataMarker = UserDataMarker, IndexTypeAndSize = IndexTypeAndSize, @@ -43,9 +58,281 @@ public sealed class RegisterBanks return clone; } + // The parts a snapshot shares with these banks; each is copied before its next write here. + private bool _contextShared; + private bool _vertexShared; + private bool _pixelShared; + private bool _computeShared; + private bool _userConfigShared; + + // A read-only copy that shares every part with the live banks until they next write it: a + // deferred draw takes one per draw, and between draws only a few parts change. + public RegisterBanks Snapshot() + { + var snapshot = new RegisterBanks(Fatal, Context, Shader.ShareStages(), UserConfig) + { + CompositeDepthSizeXy = CompositeDepthSizeXy, + UserDataMarker = UserDataMarker, + IndexTypeAndSize = IndexTypeAndSize, + }; + _contextShared = _vertexShared = _pixelShared = _computeShared = _userConfigShared = true; + _pixelScalarsShared = _hullScalarsShared = _geometryScalarsShared = _legacyVertexScalarsShared = _exportScalarsShared = _computeScalarsShared = true; + return snapshot; + } + + // Called by the register writers before they store into a part. + public void PrepareContextWrite() + { + if (_contextShared) + { + Context = Context.ShallowCopy(); + _contextShared = false; + SetContextPartsShared(true); + } + } + + // The context is copied without its nested parts, and a part only when a writer asks for it + // through these accessors: a draw usually rewrites a few plain fields. + private bool _screenViewportShared; + private bool _viewportsShared; + private bool _sampleLocationsShared; + private bool _shaderInterfaceShared; + private bool _blendControlsShared; + private bool _colorTargetsShared; + private bool _colorClearWord1Shared; + private bool _unmodeledShared; + + private void SetContextPartsShared(bool shared) => + _screenViewportShared = _viewportsShared = _sampleLocationsShared = _shaderInterfaceShared = + _blendControlsShared = _colorTargetsShared = _colorClearWord1Shared = _unmodeledShared = shared; + + private void OwnContext(ContextRegisters context) + { + Context = context; + _contextShared = false; + SetContextPartsShared(false); + } + + public ScreenViewportRegisters ScreenViewportForWrite + { + get + { + PrepareContextWrite(); + if (_screenViewportShared) + { + Context.ScreenViewport = Context.ScreenViewport.CopySharingViewports(); + _screenViewportShared = false; + } + + return Context.ScreenViewport; + } + } + + public ViewportRegisters[] ViewportsForWrite + { + get + { + var screenViewport = ScreenViewportForWrite; + if (_viewportsShared) + { + screenViewport.Viewports = (ViewportRegisters[])screenViewport.Viewports.Clone(); + _viewportsShared = false; + } + + return screenViewport.Viewports; + } + } + + public SampleLocationRegisters SampleLocationsForWrite + { + get + { + PrepareContextWrite(); + if (_sampleLocationsShared) + { + Context.SampleLocations = Context.SampleLocations.Copy(); + _sampleLocationsShared = false; + } + + return Context.SampleLocations; + } + } + + public ShaderInterfaceRegisters ShaderInterfaceForWrite + { + get + { + PrepareContextWrite(); + if (_shaderInterfaceShared) + { + Context.ShaderInterface = Context.ShaderInterface.Copy(); + _shaderInterfaceShared = false; + } + + return Context.ShaderInterface; + } + } + + public BlendRegisters[] BlendControlsForWrite + { + get + { + PrepareContextWrite(); + if (_blendControlsShared) + { + Context.BlendControls = (BlendRegisters[])Context.BlendControls.Clone(); + _blendControlsShared = false; + } + + return Context.BlendControls; + } + } + + public ColorTargetWords[] ColorTargetsForWrite + { + get + { + PrepareContextWrite(); + if (_colorTargetsShared) + { + Context.ColorTargets = (ColorTargetWords[])Context.ColorTargets.Clone(); + _colorTargetsShared = false; + } + + return Context.ColorTargets; + } + } + + public uint[] ColorClearWord1ForWrite + { + get + { + PrepareContextWrite(); + if (_colorClearWord1Shared) + { + Context.ColorClearWord1 = (uint[])Context.ColorClearWord1.Clone(); + _colorClearWord1Shared = false; + } + + return Context.ColorClearWord1; + } + } + + public Dictionary UnmodeledTableRegistersForWrite + { + get + { + PrepareContextWrite(); + if (_unmodeledShared) + { + Context.UnmodeledTableRegisters = new Dictionary(Context.UnmodeledTableRegisters); + _unmodeledShared = false; + } + + return Context.UnmodeledTableRegisters; + } + } + + public void PrepareUserConfigWrite() + { + if (_userConfigShared) + { + UserConfig = UserConfig.Copy(); + _userConfigShared = false; + } + } + + // COMPUTE_DISPATCH_INITIATOR, the first register of the compute range. + private const uint ComputeRangeStart = 0x200; + + // A stage is copied without its user scalar sets, and a set only when its window is written: + // a draw usually rewrites one set of one or two stages. + private bool _pixelScalarsShared; + private bool _hullScalarsShared; + private bool _geometryScalarsShared; + private bool _legacyVertexScalarsShared; + private bool _exportScalarsShared; + private bool _computeScalarsShared; + private const uint UserScalarWindow = UserScalarRegisters.Capacity; + + private static bool Overlaps(uint first, uint last, uint windowStart) => + first < windowStart + UserScalarWindow && last >= windowStart; + + // The shader bank's offsets: pixel below SPI_SHADER_PGM_LO_VS, the geometry stages up to the + // compute range, compute from there. + public void PrepareShaderWrite(uint offset, uint count) + { + var last = offset + Math.Max(count, 1) - 1; + if (offset < ShaderRegisterOffset.SpiShaderPgmLoVs) + { + if (_pixelShared) + { + Shader.Pixel = Shader.Pixel.ShallowCopy(); + _pixelShared = false; + } + + if (_pixelScalarsShared && Overlaps(offset, last, ShaderRegisterOffset.SpiShaderUserDataPs0)) + { + Shader.Pixel.UserScalars = Shader.Pixel.UserScalars.Copy(); + _pixelScalarsShared = false; + } + } + + if (offset < ComputeRangeStart && last >= ShaderRegisterOffset.SpiShaderPgmLoVs) + { + if (_vertexShared) + { + Shader.Vertex = Shader.Vertex.ShallowCopy(); + _vertexShared = false; + } + + var vertex = Shader.Vertex; + if (_hullScalarsShared && Overlaps(offset, last, ShaderRegisterOffset.SpiShaderUserDataHs0)) + { + vertex.HullUserScalars = vertex.HullUserScalars.Copy(); + _hullScalarsShared = false; + } + + if (_geometryScalarsShared && Overlaps(offset, last, ShaderRegisterOffset.SpiShaderUserDataGs0)) + { + vertex.GeometryUserScalars = vertex.GeometryUserScalars.Copy(); + _geometryScalarsShared = false; + } + + if (_legacyVertexScalarsShared && Overlaps(offset, last, ShaderRegisterOffset.SpiShaderUserDataVs0)) + { + vertex.LegacyVertexUserScalars = vertex.LegacyVertexUserScalars.Copy(); + _legacyVertexScalarsShared = false; + } + + if (_exportScalarsShared && Overlaps(offset, last, ShaderRegisterOffset.SpiShaderUserDataEs0)) + { + vertex.ExportUserScalars = vertex.ExportUserScalars.Copy(); + _exportScalarsShared = false; + } + } + + if (last >= ComputeRangeStart) + { + if (_computeShared) + { + Shader.Compute = Shader.Compute.ShallowCopy(); + _computeShared = false; + } + + if (_computeScalarsShared && Overlaps(offset, last, ShaderRegisterOffset.ComputeUserData0)) + { + Shader.Compute.UserScalars = Shader.Compute.UserScalars.Copy(); + _computeScalarsShared = false; + } + } + } + public void Reset() { - Context = new ContextRegisters(); + _contextShared = _vertexShared = _pixelShared = _computeShared = _userConfigShared = false; + _pixelScalarsShared = _hullScalarsShared = _geometryScalarsShared = _legacyVertexScalarsShared = _exportScalarsShared = _computeScalarsShared = false; + OwnContext(new ContextRegisters()); Shader = new ShaderProgramRegisters(); UserConfig = new UserConfigRegisters(); _savedContext = null; @@ -61,7 +348,7 @@ public sealed class RegisterBanks switch (operation) { case ContextStateOperation.Clear: - Context = new ContextRegisters(); + OwnContext(new ContextRegisters()); CompositeDepthSizeXy = null; break; case ContextStateOperation.Push: @@ -73,14 +360,14 @@ public sealed class RegisterBanks throw Fatal("The context state is not pushed."); } - Context = _savedContext; + OwnContext(_savedContext); CompositeDepthSizeXy = _savedCompositeDepthSizeXy; _savedContext = null; _savedCompositeDepthSizeXy = null; break; case ContextStateOperation.PushClear: Push(); - Context = new ContextRegisters(); + OwnContext(new ContextRegisters()); CompositeDepthSizeXy = null; break; default: diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterWriteTable.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterWriteTable.cs index 3592016..a7258f8 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterWriteTable.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterWriteTable.cs @@ -28,6 +28,7 @@ public static class RegisterWriteTable public static uint WriteContextPacket(RegisterBanks banks, in PacketContext packet, uint offset, ReadOnlySpan values) { + banks.PrepareContextWrite(); var consumed = WritePacket(banks, in packet, offset, values, Context, ContextIndirect, "context", emptyIsHandled: true); if (Rendering.RenderTrace.Enabled) { @@ -43,6 +44,7 @@ public static class RegisterWriteTable // A shader packet without values has no writer; the other banks accept it. public static uint WriteShaderPacket(RegisterBanks banks, in PacketContext packet, uint offset, ReadOnlySpan values) { + banks.PrepareShaderWrite(offset, (uint)values.Length); var consumed = WritePacket(banks, in packet, offset, values, Shader, ShaderIndirect, "shader", emptyIsHandled: false); if (Rendering.RenderTrace.Enabled) { @@ -52,15 +54,19 @@ public static class RegisterWriteTable return consumed; } - public static uint WriteUserConfigPacket(RegisterBanks banks, in PacketContext packet, uint offset, ReadOnlySpan values) => - WritePacket(banks, in packet, offset, values, UserConfig, UserConfigIndirect, "user-config", emptyIsHandled: true); + public static uint WriteUserConfigPacket(RegisterBanks banks, in PacketContext packet, uint offset, ReadOnlySpan values) + { + banks.PrepareUserConfigWrite(); + return WritePacket(banks, in packet, offset, values, UserConfig, UserConfigIndirect, "user-config", emptyIsHandled: true); + } public static void WriteContextEntry(RegisterBanks banks, uint offset, uint value, ulong tableAddress) { + banks.PrepareContextWrite(); if (offset < ContextIndirect.Length && ContextIndirect[offset] is null) { // Keep unknown in-bank table values so a later renderer implementation can decode them. - banks.Context.UnmodeledTableRegisters[offset] = value; + banks.UnmodeledTableRegistersForWrite[offset] = value; } else { @@ -88,6 +94,7 @@ public static class RegisterWriteTable public static void WriteShaderEntry(RegisterBanks banks, uint offset, uint value, ulong tableAddress) { + banks.PrepareShaderWrite(offset, 1); WriteEntry(banks, offset, value, ShaderIndirect, "shader", tableAddress); if (Rendering.RenderTrace.Enabled) TracePixelRegisterWrite(banks, offset, value, tableAddress, "table"); @@ -104,8 +111,11 @@ public static class RegisterWriteTable $"value=0x{value:X8} shader=0x{banks.Shader.Pixel.Address:X16}"); } - public static void WriteUserConfigEntry(RegisterBanks banks, uint offset, uint value, ulong tableAddress) => + public static void WriteUserConfigEntry(RegisterBanks banks, uint offset, uint value, ulong tableAddress) + { + banks.PrepareUserConfigWrite(); WriteEntry(banks, offset, value, UserConfigIndirect, "user-config", tableAddress); + } private static uint WritePacket( RegisterBanks banks, diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterWriters.Context.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterWriters.Context.cs index ea46226..706e224 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterWriters.Context.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/RegisterWriters.Context.cs @@ -98,8 +98,8 @@ internal static partial class RegisterWriters { indirect[offset] = static (banks, register, value) => { - banks.Context.ShaderInterface.PixelInterpolatorSettings[register - SpiPsInputCntl0] = value; - banks.Context.ShaderInterface.PixelInterpolatorWritten |= 1u << (int)(register - SpiPsInputCntl0); + banks.ShaderInterfaceForWrite.PixelInterpolatorSettings[register - SpiPsInputCntl0] = value; + banks.ShaderInterfaceForWrite.PixelInterpolatorWritten |= 1u << (int)(register - SpiPsInputCntl0); }; } @@ -160,7 +160,7 @@ internal static partial class RegisterWriters for (var offset = PaScAaSampleLocations0; offset < PaScAaSampleLocations0 + SampleLocationRegisters.LocationCount; offset++) { - indirect[offset] = static (banks, register, value) => banks.Context.SampleLocations.Locations[register - PaScAaSampleLocations0] = value; + indirect[offset] = static (banks, register, value) => banks.SampleLocationsForWrite.Locations[register - PaScAaSampleLocations0] = value; } foreach (var offset in new[] @@ -175,36 +175,36 @@ internal static partial class RegisterWriters indirect[offset] = IgnoreEntry; } - indirect[SpiVsOutConfig] = static (banks, _, value) => banks.Context.ShaderInterface.VertexOutputConfiguration = value; + indirect[SpiVsOutConfig] = static (banks, _, value) => banks.ShaderInterfaceForWrite.VertexOutputConfiguration = value; indirect[DbRenderOverride] = static (banks, _, value) => banks.Context.DepthRenderOverride = DepthRenderOverrideRegisters.Decode(value); - indirect[SpiShaderPosFormat] = static (banks, _, value) => banks.Context.ShaderInterface.PositionExportFormat = value; - indirect[SpiShaderIdxFormat] = static (banks, _, value) => banks.Context.ShaderInterface.IndexExportFormat = value; - indirect[PaClVsOutCntl] = static (banks, _, value) => banks.Context.ShaderInterface.VertexOutputControl = value; - indirect[GeNggSubgroupCntl] = static (banks, _, value) => banks.Context.ShaderInterface.PrimitiveShaderSubgroupControl = value; - indirect[VgtGsInstanceCnt] = static (banks, _, value) => banks.Context.ShaderInterface.GeometryInstanceCount = value; - indirect[VgtGsOnchipCntl] = static (banks, _, value) => banks.Context.ShaderInterface.GeometryOnChipControl = value; - indirect[VgtHosMaxTessLevel] = static (banks, _, value) => banks.Context.ShaderInterface.MaxTessellationLevel = value; - indirect[VgtHosMinTessLevel] = static (banks, _, value) => banks.Context.ShaderInterface.MinTessellationLevel = value; - indirect[GeMaxOutputPerSubgroup] = static (banks, _, value) => banks.Context.ShaderInterface.MaxOutputPerSubgroup = value; - indirect[VgtEsgsRingItemSize] = static (banks, _, value) => banks.Context.ShaderInterface.ExportRingItemSize = value; - indirect[VgtGsMaxVertOut] = static (banks, _, value) => banks.Context.ShaderInterface.GeometryMaxVerticesOut = value; - indirect[VgtPrimitiveIdEn] = static (banks, _, value) => banks.Context.ShaderInterface.PrimitiveIdEnable = value; - indirect[VgtReuseOff] = static (banks, _, value) => banks.Context.ShaderInterface.VertexReuseOff = value; + indirect[SpiShaderPosFormat] = static (banks, _, value) => banks.ShaderInterfaceForWrite.PositionExportFormat = value; + indirect[SpiShaderIdxFormat] = static (banks, _, value) => banks.ShaderInterfaceForWrite.IndexExportFormat = value; + indirect[PaClVsOutCntl] = static (banks, _, value) => banks.ShaderInterfaceForWrite.VertexOutputControl = value; + indirect[GeNggSubgroupCntl] = static (banks, _, value) => banks.ShaderInterfaceForWrite.PrimitiveShaderSubgroupControl = value; + indirect[VgtGsInstanceCnt] = static (banks, _, value) => banks.ShaderInterfaceForWrite.GeometryInstanceCount = value; + indirect[VgtGsOnchipCntl] = static (banks, _, value) => banks.ShaderInterfaceForWrite.GeometryOnChipControl = value; + indirect[VgtHosMaxTessLevel] = static (banks, _, value) => banks.ShaderInterfaceForWrite.MaxTessellationLevel = value; + indirect[VgtHosMinTessLevel] = static (banks, _, value) => banks.ShaderInterfaceForWrite.MinTessellationLevel = value; + indirect[GeMaxOutputPerSubgroup] = static (banks, _, value) => banks.ShaderInterfaceForWrite.MaxOutputPerSubgroup = value; + indirect[VgtEsgsRingItemSize] = static (banks, _, value) => banks.ShaderInterfaceForWrite.ExportRingItemSize = value; + indirect[VgtGsMaxVertOut] = static (banks, _, value) => banks.ShaderInterfaceForWrite.GeometryMaxVerticesOut = value; + indirect[VgtPrimitiveIdEn] = static (banks, _, value) => banks.ShaderInterfaceForWrite.PrimitiveIdEnable = value; + indirect[VgtReuseOff] = static (banks, _, value) => banks.ShaderInterfaceForWrite.VertexReuseOff = value; indirect[VgtMultiPrimIbResetIndex] = static (banks, _, value) => banks.Context.PrimitiveResetIndex = value; - indirect[VgtTessDistribution] = static (banks, _, value) => banks.Context.ShaderInterface.TessellationDistribution = value; + indirect[VgtTessDistribution] = static (banks, _, value) => banks.ShaderInterfaceForWrite.TessellationDistribution = value; indirect[VgtShaderStagesEn] = static (banks, _, value) => banks.Context.ShaderStages = value; - indirect[VgtLsHsConfig] = static (banks, _, value) => banks.Context.ShaderInterface.LocalHullConfiguration = value; - indirect[VgtGsOutPrimType] = static (banks, _, value) => banks.Context.ShaderInterface.GeometryOutputPrimitiveType = value; - indirect[VgtTfParam] = static (banks, _, value) => banks.Context.ShaderInterface.TessellationFactorParameter = value; - indirect[SpiShaderZFormat] = static (banks, _, value) => banks.Context.ShaderInterface.DepthExportFormat = value; + indirect[VgtLsHsConfig] = static (banks, _, value) => banks.ShaderInterfaceForWrite.LocalHullConfiguration = value; + indirect[VgtGsOutPrimType] = static (banks, _, value) => banks.ShaderInterfaceForWrite.GeometryOutputPrimitiveType = value; + indirect[VgtTfParam] = static (banks, _, value) => banks.ShaderInterfaceForWrite.TessellationFactorParameter = value; + indirect[SpiShaderZFormat] = static (banks, _, value) => banks.ShaderInterfaceForWrite.DepthExportFormat = value; indirect[SpiShaderColFormat] = ShaderColorFormatEntry; - indirect[SpiPsInputEna] = static (banks, _, value) => banks.Context.ShaderInterface.PixelInputEnable = value; - indirect[SpiPsInputAddr] = static (banks, _, value) => banks.Context.ShaderInterface.PixelInputAddress = value; - indirect[SpiPsInControl] = static (banks, _, value) => banks.Context.ShaderInterface.PixelInputControl = value; - indirect[SpiBarycCntl] = static (banks, _, value) => banks.Context.ShaderInterface.BarycentricControl = value; - indirect[DbShaderControl] = static (banks, _, value) => banks.Context.ShaderInterface.DepthShaderControl = DepthShaderControlRegisters.Decode(value); - indirect[CbShaderMask] = static (banks, _, value) => banks.Context.ShaderInterface.ColorShaderMask = value; - indirect[PaScShaderControl] = static (banks, _, value) => banks.Context.ShaderInterface.ScanShaderControl = value; + indirect[SpiPsInputEna] = static (banks, _, value) => banks.ShaderInterfaceForWrite.PixelInputEnable = value; + indirect[SpiPsInputAddr] = static (banks, _, value) => banks.ShaderInterfaceForWrite.PixelInputAddress = value; + indirect[SpiPsInControl] = static (banks, _, value) => banks.ShaderInterfaceForWrite.PixelInputControl = value; + indirect[SpiBarycCntl] = static (banks, _, value) => banks.ShaderInterfaceForWrite.BarycentricControl = value; + indirect[DbShaderControl] = static (banks, _, value) => banks.ShaderInterfaceForWrite.DepthShaderControl = DepthShaderControlRegisters.Decode(value); + indirect[CbShaderMask] = static (banks, _, value) => banks.ShaderInterfaceForWrite.ColorShaderMask = value; + indirect[PaScShaderControl] = static (banks, _, value) => banks.ShaderInterfaceForWrite.ScanShaderControl = value; indirect[CbTargetMask] = static (banks, _, value) => banks.Context.RenderTargetMask = value; indirect[PaScScreenScissorTl] = ScreenScissorTopLeftEntry; indirect[PaScScreenScissorBr] = ScreenScissorBottomRightEntry; @@ -248,7 +248,7 @@ internal static partial class RegisterWriters indirect[DbStencilRefMaskBack] = StencilMaskBackEntry; indirect[PaClClipCntl] = static (banks, _, value) => banks.Context.Clip = ClipControlRegisters.Decode(value); indirect[PaSuLineCntl] = LineControlEntry; - indirect[PaClVteCntl] = static (banks, _, value) => banks.Context.ScreenViewport.TransformControl = value; + indirect[PaClVteCntl] = static (banks, _, value) => banks.ScreenViewportForWrite.TransformControl = value; indirect[PaScModeCntl0] = static (banks, _, value) => banks.Context.ScanMode = ScanModeRegisters.Decode(value); indirect[PaScAaConfig] = static (banks, _, value) => banks.Context.AntialiasingConfig = AntialiasingConfigRegisters.Decode(value); indirect[PaScCentroidPriority0] = CentroidPriorityEntry; @@ -369,8 +369,8 @@ internal static partial class RegisterWriters throw PacketError(banks, "The pixel interpolator packet has an invalid count", in packet, offset, values.Length); } - values.CopyTo(banks.Context.ShaderInterface.PixelInterpolatorSettings); - banks.Context.ShaderInterface.PixelInterpolatorWritten |= (uint)((1ul << values.Length) - 1); + values.CopyTo(banks.ShaderInterfaceForWrite.PixelInterpolatorSettings); + banks.ShaderInterfaceForWrite.PixelInterpolatorWritten |= (uint)((1ul << values.Length) - 1); return (uint)values.Length; } @@ -410,7 +410,7 @@ internal static partial class RegisterWriters // One location at any slot, or all sixteen from the first slot. private static uint SampleLocationsPacket(RegisterBanks banks, in PacketContext packet, uint offset, ReadOnlySpan values) { - var locations = banks.Context.SampleLocations.Locations; + var locations = banks.SampleLocationsForWrite.Locations; if (values.Length == 1) { locations[offset - PaScAaSampleLocations0] = values[0]; @@ -506,19 +506,19 @@ internal static partial class RegisterWriters private static void HardwareScreenOffsetEntry(RegisterBanks banks, uint offset, uint value) { - banks.Context.ScreenViewport.HardwareOffsetX = RegisterField.Get(value, 0, 0x1FF); - banks.Context.ScreenViewport.HardwareOffsetY = RegisterField.Get(value, 16, 0x1FF); + banks.ScreenViewportForWrite.HardwareOffsetX = RegisterField.Get(value, 0, 0x1FF); + banks.ScreenViewportForWrite.HardwareOffsetY = RegisterField.Get(value, 16, 0x1FF); } private static void WindowOffsetEntry(RegisterBanks banks, uint offset, uint value) { - banks.Context.ScreenViewport.WindowOffsetX = RegisterField.Coordinate16(value); - banks.Context.ScreenViewport.WindowOffsetY = RegisterField.Coordinate16(value >> 16); + banks.ScreenViewportForWrite.WindowOffsetX = RegisterField.Coordinate16(value); + banks.ScreenViewportForWrite.WindowOffsetY = RegisterField.Coordinate16(value >> 16); } private static void ClipRectangleEntry(RegisterBanks banks, uint offset, uint value) { - var viewport = banks.Context.ScreenViewport; + var viewport = banks.ScreenViewportForWrite; if (offset == PaScClipRectRule) { viewport.ClipRectangleRule = (ushort)(value & 0xFFFFu); @@ -598,7 +598,7 @@ internal static partial class RegisterWriters private static void CentroidPriorityEntry(RegisterBanks banks, uint offset, uint value) { - var locations = banks.Context.SampleLocations; + var locations = banks.SampleLocationsForWrite; switch (offset) { case PaScCentroidPriority0: @@ -612,7 +612,7 @@ internal static partial class RegisterWriters private static void ShaderColorFormatEntry(RegisterBanks banks, uint offset, uint value) { - var modes = banks.Context.ShaderInterface.TargetOutputModes; + var modes = banks.ShaderInterfaceForWrite.TargetOutputModes; for (var index = 0; index < modes.Length; index++) { modes[index] = (byte)((value >> (index * 4)) & 0xFu); @@ -621,19 +621,19 @@ internal static partial class RegisterWriters private static void ScreenScissorTopLeftEntry(RegisterBanks banks, uint offset, uint value) { - banks.Context.ScreenViewport.ScreenScissorLeft = RegisterField.Coordinate16(value); - banks.Context.ScreenViewport.ScreenScissorTop = RegisterField.Coordinate16(value >> 16); + banks.ScreenViewportForWrite.ScreenScissorLeft = RegisterField.Coordinate16(value); + banks.ScreenViewportForWrite.ScreenScissorTop = RegisterField.Coordinate16(value >> 16); } private static void ScreenScissorBottomRightEntry(RegisterBanks banks, uint offset, uint value) { - banks.Context.ScreenViewport.ScreenScissorRight = RegisterField.Coordinate16(value); - banks.Context.ScreenViewport.ScreenScissorBottom = RegisterField.Coordinate16(value >> 16); + banks.ScreenViewportForWrite.ScreenScissorRight = RegisterField.Coordinate16(value); + banks.ScreenViewportForWrite.ScreenScissorBottom = RegisterField.Coordinate16(value >> 16); } private static void GenericScissorTopLeftEntry(RegisterBanks banks, uint offset, uint value) { - var viewport = banks.Context.ScreenViewport; + var viewport = banks.ScreenViewportForWrite; viewport.GenericScissorLeft = RegisterField.Coordinate15(value); viewport.GenericScissorTop = RegisterField.Coordinate15(value >> 16); viewport.GenericScissorWindowOffsetEnable = !RegisterField.Bit(value, 31); @@ -641,13 +641,13 @@ internal static partial class RegisterWriters private static void GenericScissorBottomRightEntry(RegisterBanks banks, uint offset, uint value) { - banks.Context.ScreenViewport.GenericScissorRight = RegisterField.Coordinate15(value); - banks.Context.ScreenViewport.GenericScissorBottom = RegisterField.Coordinate15(value >> 16); + banks.ScreenViewportForWrite.GenericScissorRight = RegisterField.Coordinate15(value); + banks.ScreenViewportForWrite.GenericScissorBottom = RegisterField.Coordinate15(value >> 16); } private static void WindowScissorTopLeftEntry(RegisterBanks banks, uint offset, uint value) { - var viewport = banks.Context.ScreenViewport; + var viewport = banks.ScreenViewportForWrite; viewport.WindowScissorLeft = RegisterField.Coordinate15(value); viewport.WindowScissorTop = RegisterField.Coordinate15(value >> 16); viewport.WindowScissorWindowOffsetEnable = !RegisterField.Bit(value, 31); @@ -655,13 +655,13 @@ internal static partial class RegisterWriters private static void WindowScissorBottomRightEntry(RegisterBanks banks, uint offset, uint value) { - banks.Context.ScreenViewport.WindowScissorRight = RegisterField.Coordinate15(value); - banks.Context.ScreenViewport.WindowScissorBottom = RegisterField.Coordinate15(value >> 16); + banks.ScreenViewportForWrite.WindowScissorRight = RegisterField.Coordinate15(value); + banks.ScreenViewportForWrite.WindowScissorBottom = RegisterField.Coordinate15(value >> 16); } private static void ViewportScissorTopLeftEntry(RegisterBanks banks, uint offset, uint value) { - ref var viewport = ref banks.Context.ScreenViewport.Viewports[(offset - PaScViewportScissor0Tl) / 2]; + ref var viewport = ref banks.ViewportsForWrite[(offset - PaScViewportScissor0Tl) / 2]; viewport.ScissorLeft = RegisterField.Coordinate15(value); viewport.ScissorTop = RegisterField.Coordinate15(value >> 16); viewport.ScissorWindowOffsetEnable = !RegisterField.Bit(value, 31); @@ -669,21 +669,21 @@ internal static partial class RegisterWriters private static void ViewportScissorBottomRightEntry(RegisterBanks banks, uint offset, uint value) { - ref var viewport = ref banks.Context.ScreenViewport.Viewports[(offset - PaScViewportScissor0Br) / 2]; + ref var viewport = ref banks.ViewportsForWrite[(offset - PaScViewportScissor0Br) / 2]; viewport.ScissorRight = RegisterField.Coordinate15(value); viewport.ScissorBottom = RegisterField.Coordinate15(value >> 16); } private static void ViewportZMinEntry(RegisterBanks banks, uint offset, uint value) => - banks.Context.ScreenViewport.Viewports[(offset - PaScViewportZMin0) / 2].MinDepth = RegisterField.AsFloat(value); + banks.ViewportsForWrite[(offset - PaScViewportZMin0) / 2].MinDepth = RegisterField.AsFloat(value); private static void ViewportZMaxEntry(RegisterBanks banks, uint offset, uint value) => - banks.Context.ScreenViewport.Viewports[(offset - PaScViewportZMax0) / 2].MaxDepth = RegisterField.AsFloat(value); + banks.ViewportsForWrite[(offset - PaScViewportZMax0) / 2].MaxDepth = RegisterField.AsFloat(value); private static void ViewportScaleOffsetEntry(RegisterBanks banks, uint offset, uint value) { var relative = offset - PaClViewportXScale; - ref var viewport = ref banks.Context.ScreenViewport.Viewports[relative / ViewportFieldCount]; + ref var viewport = ref banks.ViewportsForWrite[relative / ViewportFieldCount]; var number = RegisterField.AsFloat(value); switch (relative % ViewportFieldCount) { @@ -718,7 +718,7 @@ internal static partial class RegisterWriters private static void GuardBandEntry(RegisterBanks banks, uint offset, uint value) { - var viewport = banks.Context.ScreenViewport; + var viewport = banks.ScreenViewportForWrite; var number = RegisterField.AsFloat(value); switch (offset) { @@ -760,7 +760,7 @@ internal static partial class RegisterWriters } private static void BlendControlEntry(RegisterBanks banks, uint offset, uint value) => - banks.Context.BlendControls[offset - CbBlend0Control] = BlendRegisters.Decode(value); + banks.BlendControlsForWrite[offset - CbBlend0Control] = BlendRegisters.Decode(value); private static void DepthZInfoEntry(RegisterBanks banks, uint offset, uint value) { @@ -804,7 +804,7 @@ internal static partial class RegisterWriters } private static ref ColorTargetWords ColorTarget(RegisterBanks banks, uint offset, uint first, uint stride) => - ref banks.Context.ColorTargets[(offset - first) / stride]; + ref banks.ColorTargetsForWrite[(offset - first) / stride]; private static void ColorBaseLowEntry(RegisterBanks banks, uint offset, uint value) { @@ -878,7 +878,7 @@ internal static partial class RegisterWriters } private static void ColorClearWord1Entry(RegisterBanks banks, uint offset, uint value) => - banks.Context.ColorClearWord1[(offset - CbColor0ClearWord1) / ColorSlotStride] = value; + banks.ColorClearWord1ForWrite[(offset - CbColor0ClearWord1) / ColorSlotStride] = value; private static void ColorDccBaseLowEntry(RegisterBanks banks, uint offset, uint value) { diff --git a/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/ShaderProgramRegisters.cs b/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/ShaderProgramRegisters.cs index e2d3e45..b29fe93 100644 --- a/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/ShaderProgramRegisters.cs +++ b/src/SharpEmu.Libs/Gpu/GpuCommands/Registers/ShaderProgramRegisters.cs @@ -199,6 +199,9 @@ public sealed class VertexStageRegisters public UserScalarRegisters LegacyVertexUserScalars = new(); public UserScalarRegisters ExportUserScalars = new(); + // The same registers sharing the user scalar sets, which a writer then copies one by one. + public VertexStageRegisters ShallowCopy() => (VertexStageRegisters)MemberwiseClone(); + public VertexStageRegisters Copy() { var copy = (VertexStageRegisters)MemberwiseClone(); @@ -217,6 +220,8 @@ public sealed class PixelStageRegisters public PixelResource2 Resource2; public UserScalarRegisters UserScalars = new(); + public PixelStageRegisters ShallowCopy() => (PixelStageRegisters)MemberwiseClone(); + public PixelStageRegisters Copy() { var copy = (PixelStageRegisters)MemberwiseClone(); @@ -253,6 +258,8 @@ public sealed class ComputeStageRegisters public byte SharedVectorRegisters; public UserScalarRegisters UserScalars = new(); + public ComputeStageRegisters ShallowCopy() => (ComputeStageRegisters)MemberwiseClone(); + public ComputeStageRegisters Copy() { var copy = (ComputeStageRegisters)MemberwiseClone(); @@ -264,6 +271,9 @@ public sealed class ComputeStageRegisters // The shader bank: the program registers of every stage and their user scalars. public sealed class ShaderProgramRegisters { + // A bank holding the same stage objects, for a snapshot that shares them. + public ShaderProgramRegisters ShareStages() => (ShaderProgramRegisters)MemberwiseClone(); + public VertexStageRegisters Vertex = new(); public PixelStageRegisters Pixel = new(); public ComputeStageRegisters Compute = new(); diff --git a/src/SharpEmu.Libs/Gpu/Images/CachedImage.Transfers.cs b/src/SharpEmu.Libs/Gpu/Images/CachedImage.Transfers.cs index f244c77..ae91f2d 100644 --- a/src/SharpEmu.Libs/Gpu/Images/CachedImage.Transfers.cs +++ b/src/SharpEmu.Libs/Gpu/Images/CachedImage.Transfers.cs @@ -31,10 +31,13 @@ public sealed unsafe partial class CachedImage SubresourceRange = new ImageSubresourceRange(ViewFormatRules.FullAspects(Backing.Format), baseLevel, levelCount, baseLayer, layerCount), }; + // What a state that needs no barrier returns, so the common case allocates nothing. Callers only read it. + private static readonly List NoBarriers = []; + // Barriers for the requested state; a repeated write always gets a barrier. public (List Barriers, PipelineStageFlags SourceStages) GetBarriers(ImageLayout layout, AccessFlags access, PipelineStageFlags stage, SubresourceRange? range) { - var barriers = new List(); + List? barriers = null; PipelineStageFlags sourceStages = 0; if ((access & (WriteAccess | AccessFlags.ColorAttachmentWriteBit | AccessFlags.DepthStencilAttachmentWriteBit)) != 0) { @@ -83,7 +86,7 @@ public sealed unsafe partial class CachedImage var repeatedWrite = (state.Access & WriteAccess) != 0; if (state.Layout != layout || state.Access != access || repeatedWrite) { - barriers.Add(MakeBarrier(state, layout, access, level, 1, layer, 1)); + (barriers ??= []).Add(MakeBarrier(state, layout, access, level, 1, layer, 1)); sourceStages |= state.Stage; states[index] = new ImageAccessState(stage, access, layout); } @@ -101,15 +104,15 @@ public sealed unsafe partial class CachedImage var repeatedWrite = (state.Access & WriteAccess) != 0; if (state.Layout == layout && state.Access == access && !repeatedWrite) { - return (barriers, sourceStages); + return (NoBarriers, sourceStages); } - barriers.Add(MakeBarrier(state, layout, access, 0, Vk.RemainingMipLevels, 0, Vk.RemainingArrayLayers)); + (barriers ??= []).Add(MakeBarrier(state, layout, access, 0, Vk.RemainingMipLevels, 0, Vk.RemainingArrayLayers)); sourceStages |= state.Stage; } Backing.State = new ImageAccessState(stage, access, layout); - return (barriers, sourceStages); + return (barriers ?? NoBarriers, sourceStages); } public void Transition(ImageLayout layout, AccessFlags access, SubresourceRange? range, CommandBuffer command) diff --git a/src/SharpEmu.Libs/Gpu/Images/ColorToMultisampleDepthBlit.cs b/src/SharpEmu.Libs/Gpu/Images/ColorToMultisampleDepthBlit.cs index 9648265..89c81e4 100644 --- a/src/SharpEmu.Libs/Gpu/Images/ColorToMultisampleDepthBlit.cs +++ b/src/SharpEmu.Libs/Gpu/Images/ColorToMultisampleDepthBlit.cs @@ -303,6 +303,7 @@ public sealed unsafe class ColorToMultisampleDepthBlit : IDisposable vk.CmdBeginRenderPass(command, &begin, SubpassContents.Inline); vk.CmdBindDescriptorSets(command, PipelineBindPoint.Graphics, _pipelineLayout, 0, 1, &set, 0, null); vk.CmdBindPipeline(command, PipelineBindPoint.Graphics, cached.Pipeline); + Vulkan.GraphicsDynamicStateEpoch.Advance(); var viewport = new Viewport(0.0f, 0.0f, extent.Width, extent.Height, 0.0f, 1.0f); var scissor = new Rect2D(new Offset2D(0, 0), extent); vk.CmdSetViewport(command, 0, 1, &viewport); diff --git a/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.Lookup.cs b/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.Lookup.cs index 807c88c..3707edb 100644 --- a/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.Lookup.cs +++ b/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.Lookup.cs @@ -35,6 +35,25 @@ public sealed partial class GuestImageCache private void InvalidateLookups() => _lookupGeneration++; + // Changes whenever an image a lookup could find is created, removed or replaced, so a caller + // that remembers a lookup's result can tell when it may have changed. + public ulong LookupGeneration => _lookupGeneration; + + // What a remembered lookup still does on reuse: the image counts as used now. False when the + // image is gone. + public bool TryTouchRemembered(ResourceSlotIdentifier imageIdentifier) + { + using var held = _lock.Hold(); + if (_slots.TryGet(imageIdentifier) is not { Registered: true } image) + { + return false; + } + + image.LastAccessTick = _scheduler.CurrentTick; + TouchImage(image); + return true; + } + private static int LookupSlot(in ImageRequest request, bool exactFormat) { var hash = HashCode.Combine(request.Description.Data.Address, request.Description.Data.Size, request.Description.PixelFormat, diff --git a/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.Metadata.cs b/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.Metadata.cs index f7aa599..ab099e0 100644 --- a/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.Metadata.cs +++ b/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.Metadata.cs @@ -287,13 +287,20 @@ public sealed partial class GuestImageCache public bool IsMetadataCleared(ulong address, uint slice) => IsMetadataCleared(address, slice, out _); + // DCC metadata of a bound target, or the range of a DCC fill seen before its target: both are + // filled on the host, which keeps them CPU-readable instead of GPU-written. public bool OverlapsDccMetadata(ulong address, ulong size) { using var held = _lock.Hold(); foreach (var (start, metadata) in _surfaceMetadata) { - if (metadata.Kind == SurfaceMetadataKind.Dcc && metadata.Size != 0 && - address < start + metadata.Size && start < address + size) + var extent = metadata.Kind switch + { + SurfaceMetadataKind.Dcc => metadata.Size, + SurfaceMetadataKind.PendingDcc => metadata.FillSize, + _ => 0UL, + }; + if (extent != 0 && address < start + extent && start < address + size) { return true; } diff --git a/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.Transfers.cs b/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.Transfers.cs index 000513d..d290fec 100644 --- a/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.Transfers.cs +++ b/src/SharpEmu.Libs/Gpu/Images/GuestImageCache.Transfers.cs @@ -761,6 +761,7 @@ public sealed unsafe partial class GuestImageCache _scheduler.QueuePriorityCompletionAction(() => { download.Invalidate(offset, range.Size); + GpuWritebackEpoch.Advance(); if (!backing.TryWriteBacking(range.Address, download.Mapped.Slice((int)offset, (int)range.Size))) { throw SubmissionScheduler.Fatal($"The image readback could not be written to guest memory: address=0x{range.Address:X16} size=0x{range.Size:X}."); diff --git a/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.ColorTarget.cs b/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.ColorTarget.cs index 12bac61..be088e2 100644 --- a/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.ColorTarget.cs +++ b/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.ColorTarget.cs @@ -76,8 +76,56 @@ public static partial class ImageRequestBuilders public static bool SupportsDccFixedClear(Format format) => PackedClearValue.SupportsDccFixedColor(format); + // Every draw rebuilds its targets' requests (a whole mip layout) from the same register words, + // twice: once for the attachments and once for the pixel outputs. The build reads nothing but + // its arguments, so a result is reused while they are equal. + private struct ColorTargetMemo + { + public bool Valid; + public ColorTargetWords Words; + public uint TargetMask; + public uint DrawLayerOffset; + public bool IgnoreTargetMask; + public ColorTargetResolution? Result; + } + + private const int TargetMemoCount = 8; + + [ThreadStatic] + private static ColorTargetMemo[]? _colorTargetMemos; + + private static int TargetMemoSlot(ulong address) => (int)((address >> 12) ^ (address >> 20)) & (TargetMemoCount - 1); + // Builds the request for a bound color target. Null when the slot carries no target. public static ColorTargetResolution? ColorTarget(in ColorTargetWords words, uint targetMask, uint drawLayerOffset, bool ignoreTargetMask) + { + if (Rendering.RenderTrace.Enabled) + { + return ColorTargetCore(in words, targetMask, drawLayerOffset, ignoreTargetMask); + } + + var memos = _colorTargetMemos ??= new ColorTargetMemo[TargetMemoCount]; + ref var memo = ref memos[TargetMemoSlot(words.BaseAddress)]; + if (memo.Valid && memo.TargetMask == targetMask && memo.DrawLayerOffset == drawLayerOffset && + memo.IgnoreTargetMask == ignoreTargetMask && memo.Words.Equals(words)) + { + return memo.Result; + } + + var result = ColorTargetCore(in words, targetMask, drawLayerOffset, ignoreTargetMask); + memo = new ColorTargetMemo + { + Valid = true, + Words = words, + TargetMask = targetMask, + DrawLayerOffset = drawLayerOffset, + IgnoreTargetMask = ignoreTargetMask, + Result = result, + }; + return result; + } + + private static ColorTargetResolution? ColorTargetCore(in ColorTargetWords words, uint targetMask, uint drawLayerOffset, bool ignoreTargetMask) { var mask = targetMask & 0xF; if (ignoreTargetMask && words.BaseAddress != 0 && mask == 0) @@ -235,8 +283,7 @@ public static partial class ImageRequestBuilders pitch = TileGeometry.TexturePitch(transferFormat, width, GuestTileMode.Linear); } - var mipSpans = new TileLevelSpan[TiledSurfaceLayout.MaxLevels]; - var mipPadded = new TilePaddedSize[TiledSurfaceLayout.MaxLevels]; + var (mipSpans, mipPadded) = MipLayoutScratch.Rent(); TiledSurfaceLayout? volumeLayout = null; ulong size; ulong backingSize = 0; diff --git a/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.DepthTarget.cs b/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.DepthTarget.cs index 8a0c357..3fed162 100644 --- a/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.DepthTarget.cs +++ b/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.DepthTarget.cs @@ -59,8 +59,28 @@ public static partial class ImageRequestBuilders } // Builds the request for the bound depth target. Null when no depth or stencil state is active. - public static DepthTargetResolution? DepthTarget(in DepthTargetWords depthWords, IImageFormatSupport device) => - DepthTargetCore(in depthWords, device, copyMode: false, writeBuffer: false); + // Reused while the words and the device are the same, as ColorTarget's requests are. + public static DepthTargetResolution? DepthTarget(in DepthTargetWords depthWords, IImageFormatSupport device) + { + if (Rendering.RenderTrace.Enabled) + { + return DepthTargetCore(in depthWords, device, copyMode: false, writeBuffer: false); + } + + if (_depthTargetMemo is { } memo && ReferenceEquals(memo.Device, device) && memo.Words.Equals(depthWords)) + { + return memo.Result; + } + + var result = DepthTargetCore(in depthWords, device, copyMode: false, writeBuffer: false); + _depthTargetMemo = new DepthTargetMemo(depthWords, device, result); + return result; + } + + private sealed record DepthTargetMemo(DepthTargetWords Words, IImageFormatSupport Device, DepthTargetResolution? Result); + + [ThreadStatic] + private static DepthTargetMemo? _depthTargetMemo; // Builds the source or destination image used by DB_RENDER_OVERRIDE depth/stencil copies. public static DepthTargetResolution? DepthTargetCopy(in DepthTargetWords depthWords, IImageFormatSupport device, bool writeBuffer) => diff --git a/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.Texture.cs b/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.Texture.cs index 1508dd0..f99fe0a 100644 --- a/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.Texture.cs +++ b/src/SharpEmu.Libs/Gpu/Images/ImageRequestBuilders.Texture.cs @@ -89,8 +89,7 @@ public static partial class ImageRequestBuilders return; } - var spans = new TileLevelSpan[TiledSurfaceLayout.MaxLevels]; - var padded = new TilePaddedSize[TiledSurfaceLayout.MaxLevels]; + var (spans, padded) = MipLayoutScratch.Rent(); TileGeometry.TryGetTextureSize( description.GuestFormat, description.Extent.Width, description.Extent.Height, description.Resources.Levels, description.TileMode, out _, spans, padded); diff --git a/src/SharpEmu.Libs/Gpu/Images/SamplerStore.cs b/src/SharpEmu.Libs/Gpu/Images/SamplerStore.cs index 466023e..1c23e50 100644 --- a/src/SharpEmu.Libs/Gpu/Images/SamplerStore.cs +++ b/src/SharpEmu.Libs/Gpu/Images/SamplerStore.cs @@ -48,7 +48,7 @@ public enum SamplerBorderColor : uint public sealed unsafe class SamplerStore : IDisposable { private readonly GpuDeviceInfo _device; - private readonly Dictionary<(uint, uint, uint, uint, bool), Sampler> _samplers = new(); + private readonly System.Collections.Concurrent.ConcurrentDictionary<(uint, uint, uint, uint, bool), Sampler> _samplers = new(); private readonly object _gate = new(); public SamplerStore(GpuDeviceInfo device) => _device = device; @@ -57,16 +57,22 @@ public sealed unsafe class SamplerStore : IDisposable public Sampler GetSampler(in SamplerDescriptorWords words, bool integerView) { + var key = (words[0], words[1], words[2], words[3], integerView); + // Every sampler binding of every draw asks; only the first of each kind creates. + if (_samplers.TryGetValue(key, out var found)) + { + return found; + } + lock (_gate) { - var key = (words[0], words[1], words[2], words[3], integerView); if (_samplers.TryGetValue(key, out var existing)) { return existing; } var sampler = CreateSampler(words, integerView); - _samplers.Add(key, sampler); + _samplers[key] = sampler; return sampler; } } diff --git a/src/SharpEmu.Libs/Gpu/Images/TextureTransferLayout.cs b/src/SharpEmu.Libs/Gpu/Images/TextureTransferLayout.cs index 989e5e3..fef468d 100644 --- a/src/SharpEmu.Libs/Gpu/Images/TextureTransferLayout.cs +++ b/src/SharpEmu.Libs/Gpu/Images/TextureTransferLayout.cs @@ -233,8 +233,7 @@ public sealed class TextureTransferLayout layout.Pitch = TileGeometry.TexturePitch(format, width, tile); if (tile == GuestTileMode.Linear) { - var levelSpans = new TileLevelSpan[TiledSurfaceLayout.MaxLevels]; - var paddedSizes = new TilePaddedSize[TiledSurfaceLayout.MaxLevels]; + var (levelSpans, paddedSizes) = MipLayoutScratch.Rent(); TileGeometry.TryGetTextureSize(format, width, height, levels, tile, out _, levelSpans, paddedSizes); for (uint level = 0; level < levels; level++) { diff --git a/src/SharpEmu.Libs/Gpu/Images/TileGeometry.cs b/src/SharpEmu.Libs/Gpu/Images/TileGeometry.cs index 2de8d20..837441e 100644 --- a/src/SharpEmu.Libs/Gpu/Images/TileGeometry.cs +++ b/src/SharpEmu.Libs/Gpu/Images/TileGeometry.cs @@ -21,6 +21,24 @@ public struct TileLevelSpan public readonly record struct TilePaddedSize(uint Width, uint Height); +// The per-level output arrays of the layout queries, kept per thread: every draw that resolves a +// target or a texture needs them for a moment and reads them before the next query. +internal static class MipLayoutScratch +{ + [ThreadStatic] private static TileLevelSpan[]? _spans; + [ThreadStatic] private static TilePaddedSize[]? _padded; + + // Zeroed, as a new array would be, so a level the query leaves out reads as empty. + public static (TileLevelSpan[] Spans, TilePaddedSize[] Padded) Rent() + { + var spans = _spans ??= new TileLevelSpan[TiledSurfaceLayout.MaxLevels]; + var padded = _padded ??= new TilePaddedSize[TiledSurfaceLayout.MaxLevels]; + Array.Clear(spans); + Array.Clear(padded); + return (spans, padded); + } +} + public enum TileBlockKind : uint { Standard256B, diff --git a/src/SharpEmu.Libs/Gpu/Pipelines/IShaderPipelineHost.cs b/src/SharpEmu.Libs/Gpu/Pipelines/IShaderPipelineHost.cs index 2eff4e4..0177ba2 100644 --- a/src/SharpEmu.Libs/Gpu/Pipelines/IShaderPipelineHost.cs +++ b/src/SharpEmu.Libs/Gpu/Pipelines/IShaderPipelineHost.cs @@ -36,6 +36,9 @@ internal interface IShaderPipelineHost bool ComputeWave64Supported { get; } + // True when a 64-invocation wave64 compute workgroup runs as one 64-lane host subgroup. + bool ComputeWave64SubgroupNative => false; + bool GraphicsSubgroupOperationsEnabled { get; } // The device supports shaderSharedInt64Atomics, so LDS 64-bit atomics can be diff --git a/src/SharpEmu.Libs/Gpu/Pipelines/PipelineKeys.cs b/src/SharpEmu.Libs/Gpu/Pipelines/PipelineKeys.cs index f4eba84..4394dcd 100644 --- a/src/SharpEmu.Libs/Gpu/Pipelines/PipelineKeys.cs +++ b/src/SharpEmu.Libs/Gpu/Pipelines/PipelineKeys.cs @@ -45,6 +45,13 @@ public sealed class PipelineStaticParameters : IEquatable public Format StencilFormat { get; set; } = Format.Undefined; public uint ColorCount { get; set; } + // The state of a new instance, for a lookup object reused across draws. + public void Reset() + { + Array.Clear(ColorFormats); + DepthFormat = Format.Undefined; + StencilFormat = Format.Undefined; + ColorCount = 0; + } + public bool Equals(PipelineRenderingState? other) => other is not null && ColorCount == other.ColorCount && DepthFormat == other.DepthFormat && StencilFormat == other.StencilFormat && ColorFormats.AsSpan().SequenceEqual(other.ColorFormats); @@ -179,6 +195,15 @@ public sealed class PipelineVertexInputState : IEquatable other is not null && BindingCount == other.BindingCount && AttributeCount == other.AttributeCount && Bindings.AsSpan().SequenceEqual(other.Bindings) && Attributes.AsSpan().SequenceEqual(other.Attributes); @@ -207,8 +232,9 @@ public sealed class PipelineVertexInputState : IEquatable { public required PipelineRenderingState Rendering { get; init; } - public ulong VertexProgramId { get; init; } - public ulong PixelProgramId { get; init; } + // Settable for the lookup key reused across draws; a key in the cache is never changed. + public ulong VertexProgramId { get; set; } + public ulong PixelProgramId { get; set; } public required PipelineVertexInputState VertexInput { get; init; } public required PipelineStaticParameters StaticParameters { get; init; } diff --git a/src/SharpEmu.Libs/Gpu/Pipelines/ShaderPipelineCache.cs b/src/SharpEmu.Libs/Gpu/Pipelines/ShaderPipelineCache.cs index 96a8f8d..0aafc52 100644 --- a/src/SharpEmu.Libs/Gpu/Pipelines/ShaderPipelineCache.cs +++ b/src/SharpEmu.Libs/Gpu/Pipelines/ShaderPipelineCache.cs @@ -27,6 +27,9 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider private readonly Dictionary _graphicsPipelines = new(); private readonly Dictionary _computePipelines = new(); private readonly object _gate = new(); + // The pipeline maps: a lookup takes this lock or _gate, an addition both; a cached draw so does + // not wait for the program preparation a resolving thread holds _gate for. + private readonly object _pipelineGate = new(); private readonly bool _strictShaders = Environment.GetEnvironmentVariable("SHARPEMU_STRICT_COMPUTE") != "0"; private readonly HashSet<(ShaderStage Stage, ulong Hash, uint CodeSize)> _reportedShaderSkips = []; @@ -70,6 +73,11 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider return new ShaderSource(registered, hash, userData, userDataBase, stage); } + // Program resolution runs on the render thread and on a thread resolving draws ahead of it; + // the program, decode and materialization caches it updates are used under this lock only. + // A pipeline lookup takes _pipelineGate instead, so it does not wait for a resolution. + private readonly object _resolveGate = new(); + public GraphicsPrograms GetGraphicsPrograms( VertexStageRegisters vertex, PixelStageRegisters pixel, @@ -78,6 +86,21 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider ReadOnlySpan targetExportMapping, bool pixelActive, bool depthBound) + { + lock (_resolveGate) + { + return GetGraphicsProgramsCore(vertex, pixel, shaderInterface, context, targetExportMapping, pixelActive, depthBound); + } + } + + private GraphicsPrograms GetGraphicsProgramsCore( + VertexStageRegisters vertex, + PixelStageRegisters pixel, + ShaderInterfaceRegisters shaderInterface, + ContextRegisters context, + ReadOnlySpan targetExportMapping, + bool pixelActive, + bool depthBound) { using var profileScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.ProgramPreparation); var vertexSource = PrepareSource( @@ -209,44 +232,53 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider shaderInterface.VertexOutputControl, clipSpace); } - // One past the highest parameter location the pixel program reads, resolved as its translator does. - private static uint ReadVertexOutputCount(Gen5ShaderProgram pixelProgram, PixelInputInfo info) - { - var attributes = pixelProgram.Instructions + // The interpolated attributes a pixel program reads, ascending. A program never changes, and this + // was rebuilt with LINQ on every draw. + private static readonly System.Runtime.CompilerServices.ConditionalWeakTable InterpolatedAttributeCache = new(); + + private static uint[] InterpolatedAttributesOf(Gen5ShaderProgram program) => + InterpolatedAttributeCache.GetValue(program, static program => program.Instructions .Select(static instruction => instruction.Control) .OfType() .Select(static control => control.Attribute) .Distinct() .Order() - .ToArray(); + .ToArray()); + + // One past the highest parameter location the pixel program reads, resolved as its translator does. + private static uint ReadVertexOutputCount(Gen5ShaderProgram pixelProgram, PixelInputInfo info) + { + var attributes = InterpolatedAttributesOf(pixelProgram); if (attributes.Length == 0) { return 0; } - var controls = new uint[32]; + Span controls = stackalloc uint[32]; for (var index = 0u; index < (uint)controls.Length; index++) { - controls[index] = index < info.InputCount && index < (uint)info.InterpolatorSettings.Length + controls[(int)index] = index < info.InputCount && index < (uint)info.InterpolatorSettings.Length ? info.InterpolatorSettings[index] : index; } - return Gen5PixelInputMapping.ResolveLocations(controls, attributes).Max() + 1; - } - - private static uint InterpolatedAttributeCount(Gen5ShaderProgram program) - { - var maxAttribute = -1; - foreach (var instruction in program.Instructions) + // Attributes are distinct and below 32, so a program never has more than 32 of them. + Span locations = attributes.Length <= 32 ? stackalloc uint[attributes.Length] : new uint[attributes.Length]; + Gen5PixelInputMapping.ResolveLocations(controls, attributes, locations); + var highest = 0u; + foreach (var location in locations) { - if (instruction.Control is Gen5InterpolationControl interpolation) - { - maxAttribute = Math.Max(maxAttribute, (int)interpolation.Attribute); - } + highest = Math.Max(highest, location); } - return (uint)(maxAttribute + 1); + return highest + 1; + } + + // The attribute list is ascending, so its last entry is the highest. + private static uint InterpolatedAttributeCount(Gen5ShaderProgram program) + { + var attributes = InterpolatedAttributesOf(program); + return attributes.Length == 0 ? 0 : attributes[^1] + 1; } // The bound colour slots in order; each output mode names the kind the pixel program exports. @@ -321,6 +353,20 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider uint dimensionX, uint dimensionY, uint dimensionZ) + { + lock (_resolveGate) + { + return GetComputeProgramCore(compute, shaderInterface, dispatchInitiator, dimensionX, dimensionY, dimensionZ); + } + } + + private ComputeProgram GetComputeProgramCore( + ComputeStageRegisters compute, + ShaderInterfaceRegisters shaderInterface, + uint dispatchInitiator, + uint dimensionX, + uint dimensionY, + uint dimensionZ) { using var profileScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.ProgramPreparation); var source = PrepareSource(compute.Address, ShaderStage.Compute, "compute", compute.UserScalars, compute.UserScalarCount, probeWrittenRegisters: false, userDataBase: 0); @@ -417,6 +463,28 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider ShaderProgram pixelProgram) { using var profileScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.PipelineCreation); + // The lookup fills objects this thread reuses for every draw; only a new pipeline gets its own. + var lookup = _graphicsLookup ??= new GraphicsPipelineKey + { + Rendering = new PipelineRenderingState(), + VertexInput = new PipelineVertexInputState(), + StaticParameters = new PipelineStaticParameters(), + }; + lookup.Rendering.Reset(); + lookup.VertexInput.Reset(); + lookup.StaticParameters.Reset(); + FillGraphicsState(colors, in depth, vertexInput, pixelInput, context, in rendering, topology, primitiveRestartEnabled, disableBlending, + vertexProgram, pixelProgram, _host.NoAttachmentSampleCounts, lookup.Rendering, lookup.StaticParameters, lookup.VertexInput); + lookup.VertexProgramId = vertexProgram.Id; + lookup.PixelProgramId = pixelInput is not null ? pixelProgram.Id : 0; + lock (_pipelineGate) + { + if (_graphicsPipelines.TryGetValue(lookup, out var cached)) + { + return cached; + } + } + var description = BuildGraphicsDescription( colors, in depth, vertexInput, pixelInput, context, in rendering, topology, primitiveRestartEnabled, disableBlending, vertexProgram, pixelProgram, _host.NoAttachmentSampleCounts); @@ -437,12 +505,19 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider } var created = _host.CreateGraphicsPipeline(description); - _graphicsPipelines.Add(key, created); + lock (_pipelineGate) + { + _graphicsPipelines.Add(key, created); + } + ShaderCacheCounters.CountGraphicsPipeline(); return created; } } + [ThreadStatic] + private static GraphicsPipelineKey? _graphicsLookup; + // The full static state of one graphics pipeline from the draw's targets, registers and programs. internal static GraphicsPipelineDescription BuildGraphicsDescription( ReadOnlySpan colors, @@ -457,6 +532,43 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider ShaderProgram vertexProgram, ShaderProgram pixelProgram, SampleCountFlags noAttachmentSampleCounts) + { + var renderingState = new PipelineRenderingState(); + var parameters = new PipelineStaticParameters(); + var vertexState = new PipelineVertexInputState(); + var (vertexStage, pixelStage) = FillGraphicsState(colors, in depth, vertexInput, pixelInput, context, in rendering, topology, + primitiveRestartEnabled, disableBlending, vertexProgram, pixelProgram, noAttachmentSampleCounts, renderingState, parameters, vertexState); + return new GraphicsPipelineDescription + { + Rendering = renderingState, + VertexInput = vertexState, + VertexInfo = vertexInput, + VertexProgram = vertexProgram, + VertexStage = vertexStage, + PixelInfo = pixelInput, + PixelProgram = pixelProgram, + PixelStage = pixelStage, + StaticParameters = parameters, + }; + } + + // Writes the pipeline state into objects in their initial state; returns the stage programs. + private static (ShaderProgramInfo VertexStage, ShaderProgramInfo? PixelStage) FillGraphicsState( + ReadOnlySpan colors, + in DepthAttachmentState depth, + VertexInputInfo vertexInput, + PixelInputInfo? pixelInput, + ContextRegisters context, + in RenderingState rendering, + PrimitiveTopology topology, + bool primitiveRestartEnabled, + bool disableBlending, + ShaderProgram vertexProgram, + ShaderProgram pixelProgram, + SampleCountFlags noAttachmentSampleCounts, + PipelineRenderingState renderingState, + PipelineStaticParameters parameters, + PipelineVertexInputState vertexState) { if (colors.Length > PipelineStaticParameters.ColorAttachmentCount) { @@ -477,8 +589,8 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider var vertexStage = vertexInput.Stage.Program ?? throw SubmissionScheduler.Fatal("The vertex stage has no program."); var pixelStage = pixelActive ? pixelInput!.Stage.Program ?? throw SubmissionScheduler.Fatal("The pixel stage has no program.") : null; var colorCount = (uint)colors.Length; - var parameters = new PipelineStaticParameters { ColorCount = colorCount }; - var renderingState = new PipelineRenderingState { ColorCount = colorCount }; + parameters.ColorCount = colorCount; + renderingState.ColorCount = colorCount; for (var index = 0; index < colors.Length; index++) { ref readonly var color = ref colors[index]; @@ -564,18 +676,8 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider parameters.SetBlendBypass(index, ((context.ColorTargets[slot].Info >> 16) & 0x1) != 0); } - return new GraphicsPipelineDescription - { - Rendering = renderingState, - VertexInput = BuildVertexInputState(vertexInput), - VertexInfo = vertexInput, - VertexProgram = vertexProgram, - VertexStage = vertexStage, - PixelInfo = pixelInput, - PixelProgram = pixelProgram, - PixelStage = pixelStage, - StaticParameters = parameters, - }; + FillVertexInputState(vertexInput, vertexState); + return (vertexStage, pixelStage); } internal static GraphicsPipelineKey KeyOf(GraphicsPipelineDescription description) => new() @@ -587,18 +689,15 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider StaticParameters = description.StaticParameters, }; - private static PipelineVertexInputState BuildVertexInputState(VertexInputInfo info) + private static void FillVertexInputState(VertexInputInfo info, PipelineVertexInputState state) { if (info.Buffers.Length > VertexInputInfo.MaxBuffers || info.Attributes.Length > VertexInputInfo.MaxBuffers) { throw SubmissionScheduler.Fatal($"The vertex input is too large: buffers={info.Buffers.Length} attributes={info.Attributes.Length}."); } - var state = new PipelineVertexInputState - { - BindingCount = (byte)info.Buffers.Length, - AttributeCount = (byte)info.Attributes.Length, - }; + state.BindingCount = (byte)info.Buffers.Length; + state.AttributeCount = (byte)info.Attributes.Length; for (var binding = 0; binding < info.Buffers.Length; binding++) { var buffer = info.Buffers[binding]; @@ -615,8 +714,6 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider state.Attributes[index] = new PipelineVertexAttribute(attribute.OffsetBytes, (byte)attribute.BufferIndex); } - - return state; } public PipelineHandle CreateComputePipeline(ComputeInputInfo input, ShaderProgram program) @@ -629,6 +726,14 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider var stage = input.Stage.Program ?? throw SubmissionScheduler.Fatal("The compute stage has no program."); var key = new ComputePipelineKey(program.Id); + lock (_pipelineGate) + { + if (_computePipelines.TryGetValue(key, out var cached)) + { + return cached; + } + } + lock (_gate) { if (_computePipelines.TryGetValue(key, out var cached)) @@ -642,7 +747,11 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider } var created = _host.CreateComputePipeline(new ComputePipelineDescription { Input = input, Program = program, Stage = stage }); - _computePipelines.Add(key, created); + lock (_pipelineGate) + { + _computePipelines.Add(key, created); + } + ShaderCacheCounters.CountComputePipeline(); return created; } @@ -659,6 +768,15 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider var stage = input.Stage.Program ?? throw SubmissionScheduler.Fatal("The compute stage has no program."); var key = new ComputePipelineKey(program.Id); + lock (_pipelineGate) + { + if (_computePipelines.TryGetValue(key, out var cached)) + { + handle = cached; + return true; + } + } + lock (_gate) { if (_computePipelines.TryGetValue(key, out var cached)) @@ -674,7 +792,11 @@ internal sealed partial class ShaderPipelineCache : IShaderPipelineProvider return false; } - _computePipelines.Add(key, created); + lock (_pipelineGate) + { + _computePipelines.Add(key, created); + } + ShaderCacheCounters.CountComputePipeline(); handle = created; return true; diff --git a/src/SharpEmu.Libs/Gpu/Pipelines/ShaderProgramCache.cs b/src/SharpEmu.Libs/Gpu/Pipelines/ShaderProgramCache.cs index 611b9c8..9e7943d 100644 --- a/src/SharpEmu.Libs/Gpu/Pipelines/ShaderProgramCache.cs +++ b/src/SharpEmu.Libs/Gpu/Pipelines/ShaderProgramCache.cs @@ -1,6 +1,7 @@ // Copyright (C) 2026 SharpEmu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +using System.Runtime.InteropServices; using SharpEmu.HLE; using SharpEmu.Libs.Gpu.Rendering; using SharpEmu.Libs.Gpu.Scheduling; @@ -68,6 +69,37 @@ public sealed class ProgramKey : IEquatable public override int GetHashCode() => HashCode.Combine(Stage, Hash, UserDataCount, CodeSize, StaticState.Length); } +// A program key as a lookup sees it: the static state words stay in the caller's list, so a draw +// that finds its program does not build a key object or copy the words. +internal readonly ref struct ProgramKeyProbe(ShaderStage stage, ulong hash, uint userDataCount, uint codeSize, ReadOnlySpan staticState) +{ + public readonly ShaderStage Stage = stage; + public readonly ulong Hash = hash; + public readonly uint UserDataCount = userDataCount; + public readonly uint CodeSize = codeSize; + public readonly ReadOnlySpan StaticState = staticState; +} + +internal sealed class ProgramKeyComparer : IEqualityComparer, IAlternateEqualityComparer +{ + public static readonly ProgramKeyComparer Instance = new(); + + public bool Equals(ProgramKey? x, ProgramKey? y) => x is null ? y is null : x.Equals(y); + + public int GetHashCode(ProgramKey key) => key.GetHashCode(); + + public bool Equals(ProgramKeyProbe alternate, ProgramKey key) => + alternate.Stage == key.Stage && alternate.Hash == key.Hash && alternate.UserDataCount == key.UserDataCount && + alternate.CodeSize == key.CodeSize && alternate.StaticState.SequenceEqual(key.StaticState); + + // The same value as ProgramKey.GetHashCode. + public int GetHashCode(ProgramKeyProbe alternate) => + HashCode.Combine(alternate.Stage, alternate.Hash, alternate.UserDataCount, alternate.CodeSize, alternate.StaticState.Length); + + public ProgramKey Create(ProgramKeyProbe alternate) => + new(alternate.Stage, alternate.Hash, alternate.UserDataCount, alternate.CodeSize, alternate.StaticState.ToArray()); +} + // One compiled module of a program entry for one specialization and push-data start. internal sealed class ProgramPermutation { @@ -102,7 +134,7 @@ internal sealed class ShaderProgramCache private readonly CpuContext _context; private readonly IGuestGpuBackend _compiler; private readonly IShaderPipelineHost _host; - private readonly Dictionary _programs = new(); + private readonly Dictionary _programs = new(ProgramKeyComparer.Instance); private readonly Dictionary<(ulong Hash, uint CodeSize), Gen5ShaderProgram> _decoded = new(); private readonly Dictionary<(ulong Hash, uint CodeSize), ShaderCodeCapture> _codeCaptures = new(); private readonly List _staticState = new(StageStaticKey.MaxWords); @@ -216,13 +248,14 @@ internal sealed class ShaderProgramCache private ShaderProgram GetOrCompileCore(ShaderSource source, StageCompileOptions options, ref uint pushDataCursor, out ShaderStageResources stage) { - ProgramKey key; + ProgramKey? key = null; ProgramSourceEntry? entry; using (RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.ProgramCacheLookup)) { BuildStaticState(source.Stage, options); - key = new ProgramKey(source.Stage, source.Hash, (uint)source.UserData.Length, source.CodeSize, _staticState.ToArray()); - _programs.TryGetValue(key, out entry); + var probe = new ProgramKeyProbe(source.Stage, source.Hash, (uint)source.UserData.Length, source.CodeSize, + CollectionsMarshal.AsSpan(_staticState)); + _programs.GetAlternateLookup().TryGetValue(probe, out entry); } var sourceWasCached = entry is not null; if (RenderTrace.Enabled && RenderTrace.Pipeline()) @@ -246,12 +279,14 @@ internal sealed class ShaderProgramCache if (entry is null) { entry = CreateEntry(source, options); + key = BuildProgramKey(source); _programs.Add(key, entry); ShaderCacheCounters.CountProgram(); } - var snapshot = new ResourceSnapshot(); - var specialization = new ResourceSpecialization(); + // Both are assigned by the materialization that follows; a failed one throws. + ResourceSnapshot snapshot = null!; + ResourceSpecialization specialization = null!; var captureIndirectImageFailure = _spirvDumpEnabled ? ShaderPermutationDump.CreateFailureCapture(source) : null; if (Diagnostics.GpuReadTrace.Enabled) { @@ -295,7 +330,7 @@ internal sealed class ShaderProgramCache } } - var permutation = CompilePermutation(source, options, entry, specialization, pushDataCursor, key, sourceWasCached); + var permutation = CompilePermutation(source, options, entry, specialization, pushDataCursor, key ?? BuildProgramKey(source), sourceWasCached); entry.Permutations.Add(permutation); ShaderCacheCounters.CountPermutation(); stage = CreateStageResources(permutation.Program, snapshot, source, options); @@ -308,6 +343,10 @@ internal sealed class ShaderProgramCache return permutation.Handle; } + // The key of the entry the static state was just built for. + private ProgramKey BuildProgramKey(ShaderSource source) => + new(source.Stage, source.Hash, (uint)source.UserData.Length, source.CodeSize, _staticState.ToArray()); + private static ShaderStageResources CreateStageResources( ShaderProgramInfo program, ResourceSnapshot snapshot, ShaderSource source, StageCompileOptions options) { @@ -646,7 +685,7 @@ internal sealed class ShaderProgramCache default: return BuildComputeRequest(entry.Plan, resources, layout, options.ComputeInfo!, options.ComputeSystemRegisters, - sharedInt64Atomics, _host.ExecGuardElisionEnabled, signedZeroInfNanPreserve); + sharedInt64Atomics, _host.ExecGuardElisionEnabled, signedZeroInfNanPreserve, _host.ComputeWave64SubgroupNative); } } @@ -663,10 +702,11 @@ internal sealed class ShaderProgramCache private static ShaderCompileRequest BuildComputeRequest(ShaderResourcePlan plan, SpecializedResourceInfo resources, BindingLayout layout, ComputeInputInfo info, Gen5ComputeSystemRegisters? systemRegisters, bool sharedInt64Atomics, bool execGuardElision, - bool signedZeroInfNanPreserve) => + bool signedZeroInfNanPreserve, bool hostWave64Supported = false) => new(plan, resources, layout) { WaveSize = info.WaveSize, + HostWave64Supported = info.WaveSize == 64 && hostWave64Supported, EnableExecGuardElision = info.WaveSize != 64 || execGuardElision, TraceDeviceAddressFaults = SharpEmu.HLE.GpuMemory.GuestGpuMemoryHook.TraceEnabled, ScratchDwords = info.ScratchDwords, @@ -681,7 +721,7 @@ internal sealed class ShaderProgramCache internal static bool TryCompilePrewarm(ComputePrewarmRecord record, ShaderCodeCapture code, IGuestGpuBackend compiler, bool sharedInt64Atomics, bool execGuardElision, bool signedZeroInfNanPreserve, out IGuestCompiledShader? compiled, - out BindingLayout? layout, out string error) + out BindingLayout? layout, out string error, bool hostWave64Supported = false) { compiled = null; layout = null; @@ -698,7 +738,7 @@ internal sealed class ShaderProgramCache layout = AllocateLayout(program, plan, resources, record.UserDataBase, record.UserDataCount, record.PushDataCursor, record.Info.DispatchThreadDimensions); var request = BuildComputeRequest(plan, resources, layout, record.Info, record.SystemRegisters, - sharedInt64Atomics, execGuardElision, signedZeroInfNanPreserve); + sharedInt64Atomics, execGuardElision, signedZeroInfNanPreserve, hostWave64Supported); return compiler.TryCompileProgram(request, out compiled, out error) && compiled is not null; } catch (Exception exception) diff --git a/src/SharpEmu.Libs/Gpu/Rendering/ParallelProgramPrefetch.cs b/src/SharpEmu.Libs/Gpu/Rendering/ParallelProgramPrefetch.cs new file mode 100644 index 0000000..0ea527f --- /dev/null +++ b/src/SharpEmu.Libs/Gpu/Rendering/ParallelProgramPrefetch.cs @@ -0,0 +1,282 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Libs.Gpu.GpuCommands.Registers; +using SharpEmu.Libs.Gpu.Scheduling; + +namespace SharpEmu.Libs.Gpu.Rendering; + +// Resolves the programs of deferred draws on a worker thread, in issue order, while the render +// thread interprets and records. The render thread takes a result for the draw it runs only when +// the resolution completed, used the inputs the draw derives and still matches the host's +// GPU-ownership state; otherwise it resolves the draw itself. +public sealed class ParallelProgramPrefetch : IProgramPrefetch, IDisposable +{ + public sealed class Job(RegisterBanks banks, long gate) + { + internal readonly RegisterBanks Banks = banks; + // The count of stream items other than draws that must have run before the job resolves. + internal readonly long Gate = gate; + internal GraphicsProgramInputs Inputs; + internal GraphicsPrograms? Programs; + internal object? Assumptions; + // 0 queued, 1 resolved, 2 abandoned. + internal volatile int State; + } + + private readonly RenderExecutor _executor; + private readonly IRenderHost _host; + private readonly IConcurrentResolveHost _concurrent; + private readonly object _lock = new(); + private readonly Queue _queue = new(); + private readonly Thread _thread; + private int _outstanding; + private long _finished; + private long _executedGate; + // The gate the worker sleeps on; long.MaxValue while it does not. + private long _sleepingOnGate = long.MaxValue; + private bool _paused; + private bool _running; + private bool _stopping; + + public ParallelProgramPrefetch(RenderExecutor executor, IRenderHost host, IConcurrentResolveHost concurrent) + { + _executor = executor; + _host = host; + _concurrent = concurrent; + _thread = new Thread(Run) { IsBackground = true, Name = "SharpEmu program resolve" }; + _thread.Start(); + } + + // The job of the draw the render thread runs now; set around each deferred draw. + public Job? Current { get; set; } + + public long Submitted { get; private set; } + + // A gate above zero holds the job until AdvanceGate reaches it: with the command stream on its own + // thread, draws are submitted ahead of the work before them, which may change what they read. + public Job Submit(RegisterBanks banks, long gate = 0) + { + var job = new Job(banks, gate); + lock (_lock) + { + _queue.Enqueue(job); + Submitted++; + _outstanding++; + Monitor.PulseAll(_lock); + } + + return job; + } + + // Returns once the worker has finished every submitted job: it reads guest memory and cache + // state, which the render thread changes only with the worker idle. + public void WaitIdle() + { + if (Volatile.Read(ref _outstanding) == 0) + { + return; + } + + lock (_lock) + { + while (_outstanding != 0) + { + Monitor.Wait(_lock); + } + } + } + + // The render thread ran this many non-draw items: jobs gated on them may resolve. + public void AdvanceGate(long executed) + { + Interlocked.Exchange(ref _executedGate, executed); + // Only a worker asleep on a gate this reaches needs a wake: one per item costs the render + // thread a kernel call each. + if (Volatile.Read(ref _sleepingOnGate) <= executed) + { + lock (_lock) + { + Monitor.PulseAll(_lock); + } + } + } + + // Returns once the worker finished the first count jobs it was given, so a non-draw item runs + // with no resolution of the draws before it still reading. + public void WaitFinished(long count) + { + if (Volatile.Read(ref _finished) >= count) + { + return; + } + + lock (_lock) + { + while (_finished < count) + { + Monitor.Wait(_lock); + } + } + } + + // Holds the worker between jobs, for host work that changes what a resolution reads. + public void Pause() + { + lock (_lock) + { + _paused = true; + while (_running) + { + Monitor.Wait(_lock); + } + } + } + + public void Resume() + { + lock (_lock) + { + _paused = false; + Monitor.PulseAll(_lock); + } + } + + public bool TryTakeGraphics(RegisterBanks banks, in GraphicsProgramInputs inputs, out GraphicsPrograms programs) + { + programs = null!; + var job = Current; + if (job is null || !ReferenceEquals(job.Banks, banks)) + { + return false; + } + + Current = null; + WaitFor(job); + if (job.State != 1 || !job.Inputs.Equals(inputs) || !_concurrent.IsStillCurrent(job.Assumptions!)) + { + return false; + } + + programs = job.Programs!; + return true; + } + + private void WaitFor(Job job) + { + var spinner = new SpinWait(); + for (var spin = 0; spin < 64 && job.State == 0; spin++) + { + spinner.SpinOnce(sleep1Threshold: -1); + } + + if (job.State != 0) + { + return; + } + + lock (_lock) + { + while (job.State == 0) + { + Monitor.Wait(_lock); + } + } + } + + private void Run() + { + SubmissionScheduler.ThrowOnFatal = true; + while (true) + { + Job job; + lock (_lock) + { + while (_queue.Count == 0 && !_stopping) + { + Monitor.Wait(_lock); + } + + if (_stopping) + { + return; + } + + job = _queue.Peek(); + if (job.Gate > Volatile.Read(ref _executedGate)) + { + // The gate usually opens within a few items: spin before sleeping. + Monitor.Exit(_lock); + try + { + var spinner = new SpinWait(); + for (var spin = 0; spin < 50 && job.Gate > Volatile.Read(ref _executedGate); spin++) + { + spinner.SpinOnce(sleep1Threshold: -1); + } + } + finally + { + Monitor.Enter(_lock); + } + } + + while ((_paused || job.Gate > Volatile.Read(ref _executedGate)) && !_stopping) + { + Interlocked.Exchange(ref _sleepingOnGate, _paused ? long.MaxValue : job.Gate); + if (!_paused && job.Gate <= Volatile.Read(ref _executedGate)) + { + break; + } + + Monitor.Wait(_lock, 50); + } + + Volatile.Write(ref _sleepingOnGate, long.MaxValue); + if (_stopping) + { + return; + } + + _queue.Dequeue(); + _running = true; + } + + Resolve(job); + lock (_lock) + { + _outstanding--; + _finished++; + _running = false; + Monitor.PulseAll(_lock); + } + } + } + + private void Resolve(Job job) + { + _concurrent.BeginConcurrentResolve(); + try + { + job.Inputs = RenderExecutor.ProgramInputsOf(job.Banks, _host.FormatSupport, _host.Fatal); + job.Programs = _executor.ResolveGraphicsPrograms(job.Banks, in job.Inputs); + job.Assumptions = _concurrent.EndConcurrentResolve(); + job.State = 1; + } + catch (Exception) + { + // The render thread resolves the draw itself, and fails there if the error is real. + job.Assumptions = _concurrent.EndConcurrentResolve(); + job.State = 2; + } + } + + public void Dispose() + { + lock (_lock) + { + _stopping = true; + Monitor.PulseAll(_lock); + } + } +} diff --git a/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Compute.cs b/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Compute.cs index bf29ca8..ac3a7fd 100644 --- a/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Compute.cs +++ b/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Compute.cs @@ -714,15 +714,19 @@ public sealed partial class RenderExecutor var hash = input.Stage.Program!.Hash; if (!_host.TryClearImageFromBuffer(address, clear.Size, clear.PackedClear)) { - // A metadata fill may run before its target is bound; the store keeps it pending and the dispatch runs. + // A metadata fill may run before its target is bound; the store keeps it pending. The fill + // itself then runs on the host when the range is known DCC metadata: run as a dispatch it + // leaves the metadata GPU-written, and every later bind of the surface waits for the GPU to + // read it back (Astro Bot fills several surfaces this way each frame). var registered = _host.TryAbsorbDccFill(address, clear.Size, clear.PackedClear); + var filledOnHost = !registered && _host.TryFillDccMetadata(address, clear.Size, clear.PackedClear); if (RenderTrace.Enabled && RenderTrace.MetadataClear()) { RenderTrace.Write( - $"{(registered ? "Tracked" : "Deferred")} a metadata clear: shader=0x{hash:X16} address=0x{address:X16} size=0x{clear.Size:X16} value=0x{clear.PackedClear:X8}"); + $"{(registered ? "Tracked" : filledOnHost ? "Filled on the host" : "Deferred")} a metadata clear: shader=0x{hash:X16} address=0x{address:X16} size=0x{clear.Size:X16} value=0x{clear.PackedClear:X8}"); } - return registered; + return registered || filledOnHost; } if (RenderTrace.Enabled && RenderTrace.ImageClear()) diff --git a/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Prefetch.cs b/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Prefetch.cs new file mode 100644 index 0000000..31c96ac --- /dev/null +++ b/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Prefetch.cs @@ -0,0 +1,115 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Libs.Gpu.Buffers; +using SharpEmu.Libs.Gpu.GpuCommands.Registers; +using SharpEmu.Libs.Gpu.Images; + +namespace SharpEmu.Libs.Gpu.Rendering; + +// What a draw's program resolution reads besides the stage and context registers: the export +// mapping of every bound color slot, whether the pixel stage runs and whether a depth target is +// bound. All three follow from the register banks alone. +public struct GraphicsProgramInputs : IEquatable +{ + [System.Runtime.CompilerServices.InlineArray(RenderingState.ColorAttachmentCapacity)] + public struct ExportMappings + { + private ColorComponentMap _element0; + } + + public ExportMappings Mapping; + public bool PixelActive; + public bool DepthBound; + + public static GraphicsProgramInputs Create() + { + var inputs = new GraphicsProgramInputs(); + ((Span)inputs.Mapping).Fill(ColorComponentMap.Identity); + return inputs; + } + + public readonly bool Equals(GraphicsProgramInputs other) => + PixelActive == other.PixelActive && DepthBound == other.DepthBound && + ((ReadOnlySpan)Mapping).SequenceEqual(other.Mapping); + + public override readonly bool Equals(object? obj) => obj is GraphicsProgramInputs other && Equals(other); + + public override readonly int GetHashCode() + { + var hash = new HashCode(); + foreach (var map in (ReadOnlySpan)Mapping) + { + hash.Add(map.Packed); + } + + hash.Add(PixelActive); + hash.Add(DepthBound); + return hash.ToHashCode(); + } +} + +// The host side of resolving programs on a thread other than the render thread. The resolving +// thread records what it assumed of the host's GPU-ownership state; the render thread checks, +// when it takes the result, that the assumptions still hold. +public interface IConcurrentResolveHost +{ + // On the resolving thread, around one resolution. + void BeginConcurrentResolve(); + + object EndConcurrentResolve(); + + // On the render thread. + bool IsStillCurrent(object assumptions); +} + +// Program resolutions computed ahead of their draws. A draw takes one only when it was resolved +// from the same register banks with the same inputs; otherwise it resolves its programs itself. +public interface IProgramPrefetch +{ + bool TryTakeGraphics(RegisterBanks banks, in GraphicsProgramInputs inputs, out GraphicsPrograms programs); +} + +public sealed partial class RenderExecutor +{ + public IProgramPrefetch? Prefetch { get; set; } + + // The inputs ResolveShaderPrograms derives from the targets TryResolveDrawTargets binds, from the + // registers alone: the target requests and the pixel and depth activity, without the images. + public static GraphicsProgramInputs ProgramInputsOf(RegisterBanks banks, IImageFormatSupport formats, Func fatal) + { + var inputs = GraphicsProgramInputs.Create(); + var context = banks.Context; + Span mapping = inputs.Mapping; + for (var slot = 0u; slot < ContextRegisters.ColorTargetCount; slot++) + { + if (slot != 0 && (context.RenderTargetMaskForSlot(slot) == 0 || context.ColorTargets[slot].BaseAddress == 0)) + { + continue; + } + + if (ColorTargetResolver.Resolve(context, slot, DrawLayerOffset, ignoreTargetMask: false, out var resolvedSlot) is { } resolution) + { + mapping[(int)resolvedSlot] = resolution.ExportMapping; + } + } + + inputs.DepthBound = DepthTargetResolver.Resolve(context, formats, fatal) is not null; + inputs.PixelActive = HasActivePixelShader(banks); + return inputs; + } + + // The programs of a draw with these banks and inputs, as ResolveShaderPrograms resolves them. + public GraphicsPrograms ResolveGraphicsPrograms(RegisterBanks banks, in GraphicsProgramInputs inputs) + { + var context = banks.Context; + return _pipelines.GetGraphicsPrograms( + banks.Shader.Vertex, + banks.Shader.Pixel, + context.ShaderInterface, + context, + inputs.Mapping, + inputs.PixelActive, + inputs.DepthBound); + } +} diff --git a/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Resources.cs b/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Resources.cs index 5831fae..e1569cc 100644 --- a/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Resources.cs +++ b/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.Resources.cs @@ -35,7 +35,8 @@ public sealed partial class RenderExecutor throw _host.Fatal($"The vertex input has too many buffers: count={buffers.Length} max={VertexInputInfo.MaxBuffers}."); } - Span ranges = stackalloc VertexBufferRange[VertexInputInfo.MaxBuffers]; + // Sized to this draw's buffers: a MaxBuffers span is over a kilobyte, zeroed on every draw. + Span ranges = stackalloc VertexBufferRange[buffers.Length]; var rangeCount = 0; foreach (ref readonly var vertex in buffers.AsSpan()) { @@ -54,7 +55,7 @@ public sealed partial class RenderExecutor } ranges[..rangeCount].Sort(static (left, right) => left.BaseAddress.CompareTo(right.BaseAddress)); - Span merged = stackalloc VertexBufferRange[VertexInputInfo.MaxBuffers]; + Span merged = stackalloc VertexBufferRange[buffers.Length]; var mergedCount = 0; for (var i = 0; i < rangeCount; i++) { diff --git a/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.cs b/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.cs index 8558cf4..cc6dd14 100644 --- a/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.cs +++ b/src/SharpEmu.Libs/Gpu/Rendering/RenderExecutor.cs @@ -667,29 +667,22 @@ public sealed partial class RenderExecutor private void ResolveShaderPrograms(RegisterBanks banks, ref DrawState state) { using var profileScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.DrawProgramResolution); - var context = banks.Context; - Span mappingStorage = stackalloc ColorComponentMapArray[1]; - Span targetExportMapping = mappingStorage[0]; - targetExportMapping.Fill(ColorComponentMap.Identity); + var inputs = GraphicsProgramInputs.Create(); + Span targetExportMapping = inputs.Mapping; foreach (ref readonly var color in BoundColors(ref state)) { targetExportMapping[(int)color.Slot] = color.Resolution.ExportMapping; } - state.Programs = _pipelines.GetGraphicsPrograms( - banks.Shader.Vertex, - banks.Shader.Pixel, - context.ShaderInterface, - context, - targetExportMapping, - state.PixelActive, - state.Depth.HasTarget); - } + inputs.PixelActive = state.PixelActive; + inputs.DepthBound = state.Depth.HasTarget; + if (Prefetch is { } prefetch && prefetch.TryTakeGraphics(banks, in inputs, out var prefetched)) + { + state.Programs = prefetched; + return; + } - [System.Runtime.CompilerServices.InlineArray(RenderingState.ColorAttachmentCapacity)] - private struct ColorComponentMapArray - { - private ColorComponentMap _element0; + state.Programs = ResolveGraphicsPrograms(banks, in inputs); } private static void TraceDrawState(ulong submitId, RegisterBanks banks, in DrawCall draw, in DrawState state) diff --git a/src/SharpEmu.Libs/Gpu/Rendering/RenderingState.cs b/src/SharpEmu.Libs/Gpu/Rendering/RenderingState.cs index 30b36d8..0f1bd06 100644 --- a/src/SharpEmu.Libs/Gpu/Rendering/RenderingState.cs +++ b/src/SharpEmu.Libs/Gpu/Rendering/RenderingState.cs @@ -19,7 +19,29 @@ public readonly record struct RenderingAttachment( bool HasDepth, bool DepthClear, bool HasStencil, - bool StencilClear); + bool StencilClear) +{ + // Compared by handle value: the generated comparison goes through the object comparer of ImageView + // and boxes both views, which a draw pays for every attachment it re-checks. + public bool Equals(RenderingAttachment other) => + View.Handle == other.View.Handle && + Layout == other.Layout && + Format == other.Format && + ClearWord0 == other.ClearWord0 && + ClearWord1 == other.ClearWord1 && + ClearWord2 == other.ClearWord2 && + ClearWord3 == other.ClearWord3 && + IsClear == other.IsClear && + HasDepth == other.HasDepth && + DepthClear == other.DepthClear && + HasStencil == other.HasStencil && + StencilClear == other.StencilClear; + + public override int GetHashCode() => HashCode.Combine( + View.Handle, + HashCode.Combine(Layout, Format, ClearWord0, ClearWord1, ClearWord2, ClearWord3), + HashCode.Combine(IsClear, HasDepth, DepthClear, HasStencil, StencilClear)); +} [InlineArray(RenderingState.ColorAttachmentCapacity)] public struct RenderingColorAttachments diff --git a/src/SharpEmu.Libs/Gpu/Scheduling/GpuWritebackEpoch.cs b/src/SharpEmu.Libs/Gpu/Scheduling/GpuWritebackEpoch.cs new file mode 100644 index 0000000..5d329cd --- /dev/null +++ b/src/SharpEmu.Libs/Gpu/Scheduling/GpuWritebackEpoch.cs @@ -0,0 +1,16 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.Libs.Gpu.Scheduling; + +// Counts the stores of GPU data into guest memory: buffer and image downloads, and buffer transfers +// the caches complete there. Work done ahead of the render thread from guest memory is still +// current while the count has not moved. +internal static class GpuWritebackEpoch +{ + private static long _value; + + public static long Value => Volatile.Read(ref _value); + + public static void Advance() => Interlocked.Increment(ref _value); +} diff --git a/src/SharpEmu.Libs/Gpu/Scheduling/SubmissionScheduler.cs b/src/SharpEmu.Libs/Gpu/Scheduling/SubmissionScheduler.cs index 7999a4d..5535658 100644 --- a/src/SharpEmu.Libs/Gpu/Scheduling/SubmissionScheduler.cs +++ b/src/SharpEmu.Libs/Gpu/Scheduling/SubmissionScheduler.cs @@ -88,9 +88,21 @@ public sealed class SubmissionScheduler : IGpuTickScheduler, IDisposable internal bool PriorityWorkerAlive => _priorityThread.IsAlive; + // Set on a thread that does work ahead of the render thread: there a fatal condition only + // abandons that work, which the render thread then does itself and fails on if it is real. + [ThreadStatic] + internal static bool ThrowOnFatal; + + public sealed class AbandonedWorkException(string message) : Exception(message); + // Report the fatal error, then return an exception to stop the caller. internal static Exception Fatal(string message) { + if (ThrowOnFatal) + { + throw new AbandonedWorkException(message); + } + Console.Error.WriteLine($"[GPU][FATAL] {message}"); OnFatal(message); return new InvalidOperationException(message); diff --git a/src/SharpEmu.Libs/Gpu/Vulkan/GraphicsDynamicStateEpoch.cs b/src/SharpEmu.Libs/Gpu/Vulkan/GraphicsDynamicStateEpoch.cs new file mode 100644 index 0000000..9df63b5 --- /dev/null +++ b/src/SharpEmu.Libs/Gpu/Vulkan/GraphicsDynamicStateEpoch.cs @@ -0,0 +1,16 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.Libs.Gpu.Vulkan; + +// Advanced by every graphics pipeline bind or dynamic state change outside the guest draw path: +// such a pipeline may fix the state the guest pipelines leave dynamic, so the dynamic state a +// guest draw last set no longer holds in the command buffer. +internal static class GraphicsDynamicStateEpoch +{ + private static long _value; + + public static long Value => Volatile.Read(ref _value); + + public static void Advance() => Interlocked.Increment(ref _value); +} diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CachedImages.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CachedImages.cs index 8d57211..4d1a7ad 100644 --- a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CachedImages.cs +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CachedImages.cs @@ -130,6 +130,37 @@ internal static unsafe partial class VulkanVideoPresenter // A draw-local copy used when the guest samples a depth aspect while the // same image is attached for writes. It retires with the submission. public bool IsFeedbackSnapshot; + // Taken from the presenter's pool; goes back to it when its submission retires. + public bool Pooled; + public bool InPool; + + // Back to the state of a new object, so a reused one never carries a previous binding. + public void Reset() + { + Address = 0; + ImageIdentifier = default; + Request = default; + CachedImage = null; + MipLevel = 0; + Image = default; + View = default; + Layout = ImageLayout.ShaderReadOnlyOptimal; + Sampler = default; + SamplerState = default; + IsStorage = false; + IsHostMovie = false; + HostMoviePlane = -1; + HostMovieFrameSerial = 0; + NeedsUpload = false; + StagingBuffer = default; + StagingMemory = default; + Width = 0; + Height = 0; + RowLength = 0; + DestinationSelect = 0; + MipViews = []; + IsFeedbackSnapshot = false; + } } private sealed partial class Presenter diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CommandStream.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CommandStream.cs index 36e741d..874a27d 100644 --- a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CommandStream.cs +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CommandStream.cs @@ -176,9 +176,6 @@ internal static unsafe partial class VulkanVideoPresenter private EndOfPipe _endOfPipe = null!; private AgcExports.CommandStreamTranslation _translation = null!; private ICpuMemory _guestMemory = null!; - private long _lastCommandStreamProgressTicks; - private static readonly long _blockedRetryTicks = - CommandStreamQueue.AllBlockedRetryMilliseconds * System.Diagnostics.Stopwatch.Frequency / 1000L; public CommandStreamQueue CommandStream => _commandStream; @@ -188,7 +185,6 @@ internal static unsafe partial class VulkanVideoPresenter _guestMemory = guest; _endOfPipe = new EndOfPipe(_endOfPipeEvents, this); _translation = AgcExports.CreateCommandStreamTranslation(guest, this); - _lastCommandStreamProgressTicks = System.Diagnostics.Stopwatch.GetTimestamp(); } public void EnqueueCommandStream(uint queue, ulong address, uint dwordCount, ulong submissionId, object? geometrySnapshots) @@ -204,25 +200,42 @@ internal static unsafe partial class VulkanVideoPresenter } } - // Runs slices until the budget ends or nothing is runnable; blocked heads retry every 100 ms. + // The command stream thread interprets the submissions; this thread runs what it hands over + // until the budget ends or nothing is queued. Null until the first render tick. + private CommandThread? _commandThread; + private Action? _betweenCommandItems; + + private bool OnCommandThread => _commandThread is { IsProducerThread: true }; + + // Items run in batches; completions are collected between them so blocked waits retry. + private const int CommandItemBatch = 256; + private void RunCommandStreamSlices(long renderWorkDeadline) { - var slices = 0; - while (slices < _maxGuestWorkPerRender) + if (_commandThread is null) + { + _commandThread = new CommandThread(_commandStream, _guestMemory, RunDeferredWork, _translation.Prefetch); + _betweenCommandItems = RunRelayBetweenCommandItems; + _translation.Sink = _commandThread; + _commandThread.Start(); + } + + var thread = _commandThread; + for (;;) { using (RenderPhaseProfile.Measure(RenderPhaseProfile.Phase.Collect)) { CollectCompletedGuestSubmissions(waitForOldest: false); } - SliceResult result; + int ran; using (RenderPhaseProfile.Measure(RenderPhaseProfile.Phase.CommandStream)) { var aliasAccess = _guestBacking?.TryEnterBackingAliasAccess() == true; _backingAliasAccess = aliasAccess; try { - result = _commandStream.ProcessOne(); + ran = thread.RunBatch(renderWorkDeadline, CommandItemBatch, _betweenCommandItems!); } finally { @@ -234,64 +247,162 @@ internal static unsafe partial class VulkanVideoPresenter } } - switch (result) - { - case SliceResult.NoWork: - return; - case SliceResult.AllBlocked: - RetryBlockedCommandStreamIfDue(); - return; - case SliceResult.BlockedWithoutProgress: - if (!_commandStream.HasUnblockedPending) - { - RetryBlockedCommandStreamIfDue(); - return; - } - - break; - default: - _lastCommandStreamProgressTicks = System.Diagnostics.Stopwatch.GetTimestamp(); - break; - } - - slices++; - if (System.Diagnostics.Stopwatch.GetTimestamp() >= renderWorkDeadline) + if (ran == 0 || System.Diagnostics.Stopwatch.GetTimestamp() >= renderWorkDeadline) { return; } } } - private void RetryBlockedCommandStreamIfDue() + private void RunDeferredWork(AgcExports.DeferredWork work) { - var now = System.Diagnostics.Stopwatch.GetTimestamp(); - if (now - _lastCommandStreamProgressTicks < _blockedRetryTicks) + using var translationScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.CommandDrawTranslation); + var started = System.Diagnostics.Stopwatch.GetTimestamp(); + try { - return; + _translation.RunDeferred(work); + } + finally + { + Interlocked.Add(ref _perfDrawTicks, System.Diagnostics.Stopwatch.GetTimestamp() - started); } - - _commandStream.RetryBlocked(); - _lastCommandStreamProgressTicks = now; } - // Milliseconds until the blocked heads are due for a retry; null when none is blocked. - private int? BlockedRetryWaitMilliseconds() + // Commands other threads post to the GPU worker run between items, with the resolution + // worker held: they change the caches it reads. + private void RunRelayBetweenCommandItems() { - if (!_vulkanReady || !_commandStream.HasPending || _commandStream.HasUnblockedPending) + if (_relay.HasPendingCommands) { - return null; + RunRelayCommandsWithPrefetchHeld(); } - - var remaining = _blockedRetryTicks - (System.Diagnostics.Stopwatch.GetTimestamp() - _lastCommandStreamProgressTicks); - return remaining <= 0 ? 0 : (int)Math.Max(1, remaining * 1000 / System.Diagnostics.Stopwatch.Frequency); } - ICpuMemory ICommandStreamHost.Memory => _guestMemory; + private void RunRelayCommandsWithPrefetchHeld() + { + var prefetch = _commandThread is not null ? _translation.Prefetch : null; + prefetch?.Pause(); + try + { + using var relayScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.QueueRelay); + _relay.RunPendingCommands(); + } + finally + { + prefetch?.Resume(); + } + } + + // The interpreter's own stores (labels, atomics, semaphores) follow every deferred draw. + private DrainingWriteMemory? _commandMemory; + + ICpuMemory ICommandStreamHost.Memory => _commandMemory ??= new DrainingWriteMemory(_guestMemory, DrainDraws, this); + + // On the command stream thread a store is queued behind the work before it and reads see + // the queued stores; elsewhere the deferred draws run first. + private sealed class DrainingWriteMemory(ICpuMemory inner, Action drain, Presenter presenter) : ICpuMemory, ICpuMemoryWrapper + { + public ICpuMemory Inner => inner; + + public bool TryRead(ulong virtualAddress, Span destination) => + presenter.OnCommandThread + ? presenter._commandThread!.ReadOverlay(virtualAddress, destination) + : inner.TryRead(virtualAddress, destination); + + public bool TryWrite(ulong virtualAddress, ReadOnlySpan source) + { + if (presenter.OnCommandThread) + { + presenter._commandThread!.PostWrite(virtualAddress, source); + return true; + } + + drain(); + return inner.TryWrite(virtualAddress, source); + } + + public bool TryCompare(ulong virtualAddress, ReadOnlySpan expected, out bool equal) + { + if (presenter.OnCommandThread) + { + var bytes = expected.ToArray(); + var compared = false; + var result = presenter._commandThread!.Run(() => TryCompare(virtualAddress, bytes, out compared)); + equal = compared; + return result; + } + + drain(); + return inner.TryCompare(virtualAddress, expected, out equal); + } + + public bool TryCopy(ulong destinationAddress, ulong sourceAddress, ulong length) + { + if (presenter.OnCommandThread) + { + return presenter._commandThread!.Run(() => TryCopy(destinationAddress, sourceAddress, length)); + } + + drain(); + return inner.TryCopy(destinationAddress, sourceAddress, length); + } + + public bool TryScanCString(ulong address, byte needle, bool findLast, ulong maxLength, out ulong match) => + inner.TryScanCString(address, needle, findLast, maxLength, out match); + + public bool CanRead(ulong address, ulong size) => inner.CanRead(address, size); + + public string DescribeReadRange(ulong address, ulong size) => inner.DescribeReadRange(address, size); + } + + // A read the render thread runs after everything queued before it, for the command thread. + private bool ReadOnRenderThread(ulong address, Span destination, bool operand) + { + var buffer = new byte[destination.Length]; + var read = _commandThread!.Run(() => operand ? TryReadGuestOperand(address, buffer) : TryReadGuest(address, buffer)); + buffer.CopyTo(destination); + return read; + } + + // Command memory and polled labels: GPU-written pages go through the render thread, the + // rest is read here with the stream's queued stores applied. + private bool ReadOnCommandThread(ulong address, Span destination) + { + _commandThread!.WaitForPendingRange(address, (ulong)destination.Length); + return _bufferCache.MayHaveGpuDirtyPages(address, (ulong)destination.Length) + ? ReadOnRenderThread(address, destination, operand: false) + : _commandThread!.ReadOverlay(address, destination); + } + + public bool TryReadGuestWaitOperand(ulong address, Span destination) => + OnCommandThread ? ReadOnCommandThread(address, destination) : TryReadGuestOperand(address, destination); + + public bool TryReadGuestOperand(ulong address, Span destination) + { + if (OnCommandThread) + { + return ReadOnRenderThread(address, destination, operand: true); + } + + DrainDraws(); + return TryReadGuest(address, destination); + } public bool TryReadGuest(ulong address, Span destination) { + if (OnCommandThread) + { + return ReadOnCommandThread(address, destination); + } + using (RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.CommandMemorySync)) { + // GPU-written command memory is downloaded first: deferred draws are recorded before it. + if (_translation is { HasPendingDraws: true } && _bufferCache.MayHaveGpuDirtyPages(address, (ulong)destination.Length)) + { + DrainDraws(); + } + if (!_bufferCache.TrySynchronizeCpuRead(address, (ulong)destination.Length, SharpEmu.HLE.GuestMemory.GuestMemoryProfile.ReadbackSource.CommandMemoryRead)) { @@ -305,27 +416,66 @@ internal static unsafe partial class VulkanVideoPresenter public void RunPendingCommands() { + // With the stream on its own thread the render thread runs them between its items. + if (OnCommandThread) + { + return; + } + + if (_relay.HasPendingCommands) + { + DrainDraws(); + } + using var relayScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.QueueRelay); _relay.RunPendingCommands(); } public void BeginSubmission(int queueId, ulong submissionId, object? geometrySnapshots) + { + if (OnCommandThread) + { + // The translation state belongs to this thread; the queue context to the render thread. + _translation.BeginSubmission(queueId, submissionId, geometrySnapshots, _commandStream.GetInterpreter(queueId)); + _commandThread!.Post(() => BeginSubmissionContext(queueId, submissionId)); + return; + } + + DrainDraws(); + BeginSubmissionContext(queueId, submissionId); + _translation.BeginSubmission(queueId, submissionId, geometrySnapshots, _commandStream.GetInterpreter(queueId)); + } + + private void BeginSubmissionContext(int queueId, ulong submissionId) { using var contextScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.QueueContext); _activeGuestQueue = new VulkanGuestQueueIdentity(_commandQueueNames[queueId], submissionId); BindSubmissionContext(_activeGuestQueue); _ = CurrentRecordingBuffer(); - _translation.BeginSubmission(queueId, submissionId, geometrySnapshots, _commandStream.GetInterpreter(queueId)); } public void Flush() { + if (OnCommandThread) + { + _commandThread!.Post(Flush); + return; + } + + DrainDraws(); using var flushScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.Flush); FlushBatchedGuestCommands(); } public void FlushAndWait() { + if (OnCommandThread) + { + _commandThread!.Run(FlushAndWait); + return; + } + + DrainDraws(); using var waitScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.CommandGpuWait); _ = CurrentRecordingBuffer(); _scheduler.FlushAndWait(); @@ -334,16 +484,40 @@ internal static unsafe partial class VulkanVideoPresenter public void SynchronizeGpu() { + if (OnCommandThread) + { + _commandThread!.Run(SynchronizeGpu); + return; + } + + DrainDraws(); using var waitScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.CommandGpuWait); _ = CurrentRecordingBuffer(); _scheduler.Finish(); CollectCompletedGuestSubmissions(waitForOldest: false); } - public void RunGarbageCollector() => RunGuestCacheCollection(endsFrame: false); + public void RunGarbageCollector() + { + if (OnCommandThread) + { + _commandThread!.Post(RunGarbageCollector); + return; + } + + DrainDraws(); + RunGuestCacheCollection(endsFrame: false); + } public void EmitGlobalBarrier() { + if (OnCommandThread) + { + _commandThread!.Post(EmitGlobalBarrier); + return; + } + + DrainDraws(); // Inside a rendering scope whose draws only wrote attachments, the barrier matters only // to what runs after the scope: everything before the scope is ordered by the barrier // BeginRendering records, and a draw that stores to memory ends a deferred scope first. @@ -380,6 +554,13 @@ internal static unsafe partial class VulkanVideoPresenter public void FillBuffer(ulong address, ulong size, uint value, bool isGds) { + if (OnCommandThread) + { + _commandThread!.PostRange(() => FillBuffer(address, size, value, isGds), address, isGds ? 0 : size); + return; + } + + DrainDraws(); using var transferScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.CommandMemoryTransfer); // Transfers the buffer cache completes in guest memory leave the rendering scope open; // its GPU paths end the scope before they record. @@ -389,6 +570,13 @@ internal static unsafe partial class VulkanVideoPresenter public void CopyBuffer(ulong destination, ulong source, ulong size, bool destinationIsGds, bool sourceIsGds) { + if (OnCommandThread) + { + _commandThread!.PostRange(() => CopyBuffer(destination, source, size, destinationIsGds, sourceIsGds), destination, destinationIsGds ? 0 : size); + return; + } + + DrainDraws(); using var transferScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.CommandMemoryTransfer); // Transfers the buffer cache completes in guest memory leave the rendering scope open; // its GPU paths end the scope before they record. @@ -396,11 +584,30 @@ internal static unsafe partial class VulkanVideoPresenter _bufferCache.CopyBuffer(destination, source, size, destinationIsGds, sourceIsGds); } - public void ReadGds(Span destination, uint wordOffset, uint wordCount) => + public void ReadGds(Span destination, uint wordOffset, uint wordCount) + { + if (OnCommandThread) + { + var words = new uint[destination.Length]; + _commandThread!.Run(() => ReadGds(words, wordOffset, wordCount)); + words.CopyTo(destination); + return; + } + + DrainDraws(); EndOfPipe.ReadGdsWords(_bufferCache.GdsBuffer.Mapped, destination, wordOffset, wordCount); + } public void RecordEndOfPipe(in EndOfPipeWrite write) { + if (OnCommandThread) + { + var queued = write; + _commandThread!.Post(() => RecordEndOfPipe(in queued)); + return; + } + + DrainDraws(); using var completionScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.CommandEndOfPipe); _ = BeginBatchedGuestCommands(); var buffer = _scheduler.Current; @@ -456,10 +663,26 @@ internal static unsafe partial class VulkanVideoPresenter } } - public void TriggerInterrupt(int eventId, uint contextId) => _endOfPipeEvents.TriggerInterrupt(eventId, contextId); + public void TriggerInterrupt(int eventId, uint contextId) + { + if (OnCommandThread) + { + _commandThread!.Post(() => TriggerInterrupt(eventId, contextId)); + return; + } + + DrainDraws(); + _endOfPipeEvents.TriggerInterrupt(eventId, contextId); + } public bool HasFlipSlot() { + if (OnCommandThread) + { + return _commandThread!.Run(HasFlipSlot); + } + + DrainDraws(); lock (_gate) { return _pendingGuestImagePresentations.Count < MaxPendingGuestFlipVersions; @@ -468,17 +691,47 @@ internal static unsafe partial class VulkanVideoPresenter public ulong PrepareFlip(int handle, int index, int flipMode, long flipArgument) { + if (OnCommandThread) + { + return _commandThread!.Run(() => PrepareFlip(handle, index, flipMode, flipArgument)); + } + + DrainDraws(); _ = BeginBatchedGuestCommands(); return _endOfPipe.PrepareVideoOutFlip(_scheduler.Current, handle, index, flipMode, flipArgument); } - public bool IsFlipDone(int handle, int index) => VideoOutExports.IsFlipDone(handle, index); + public bool IsFlipDone(int handle, int index) + { + if (OnCommandThread) + { + return _commandThread!.Run(() => IsFlipDone(handle, index)); + } - public void PrepareCpuFlip(int handle, int index, ulong requestId) => + DrainDraws(); + return VideoOutExports.IsFlipDone(handle, index); + } + + public void PrepareCpuFlip(int handle, int index, ulong requestId) + { + if (OnCommandThread) + { + _commandThread!.Run(() => PrepareCpuFlip(handle, index, requestId)); + return; + } + + DrainDraws(); CaptureFlip(handle, index, requestId, flipMode: 0, flipArg: 0); + } public void DrawIndexed(ulong submitId, in DrawIndexedArguments arguments) { + if (OnCommandThread) + { + _translation.DrawIndexed(submitId, in arguments); + return; + } + using var translationScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.CommandDrawTranslation); var started = System.Diagnostics.Stopwatch.GetTimestamp(); try @@ -493,6 +746,12 @@ internal static unsafe partial class VulkanVideoPresenter public void DrawAuto(ulong submitId, in DrawAutoArguments arguments) { + if (OnCommandThread) + { + _translation.DrawAuto(submitId, in arguments); + return; + } + using var translationScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.CommandDrawTranslation); var started = System.Diagnostics.Stopwatch.GetTimestamp(); try @@ -521,6 +780,13 @@ internal static unsafe partial class VulkanVideoPresenter public void DispatchDirect(ulong submitId, uint groupsX, uint groupsY, uint groupsZ, uint dispatchInitiator, ulong indirectArgumentsAddress = 0) { + if (OnCommandThread) + { + _translation.Dispatch(submitId, groupsX, groupsY, groupsZ, dispatchInitiator, indirectArgumentsAddress); + return; + } + + DrainDraws(); using var translationScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.CommandDispatchTranslation); var started = System.Diagnostics.Stopwatch.GetTimestamp(); try @@ -533,13 +799,48 @@ internal static unsafe partial class VulkanVideoPresenter } } - public void OnQueueReset(int queueId) => _translation.QueueReset(queueId); + public void OnQueueReset(int queueId) + { + if (OnCommandThread) + { + _translation.QueueReset(queueId); + return; + } + + DrainDraws(); + _translation.QueueReset(queueId); + } + + // Deferred draws run before anything else the stream asks of the host: everything but a + // draw may record, read or write what they use. Their time counts as draw time. Completion + // callbacks reach some of the same entry points from the GPU priority worker; they are not + // ordered with the stream and leave the draws to the render thread. + private void DrainDraws() + { + // Hosts built without a command stream have no translation. + if (_translation is not { HasPendingDraws: true } || !_relay.IsGpuQueueThread) + { + return; + } + + using var translationScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.CommandDrawTranslation); + var started = System.Diagnostics.Stopwatch.GetTimestamp(); + try + { + _translation.DrainPendingDraws(); + } + finally + { + Interlocked.Add(ref _perfDrawTicks, System.Diagnostics.Stopwatch.GetTimestamp() - started); + } + } public Exception Fatal(string message) => SubmissionScheduler.Fatal(message); // The flip packet: translation first, then the request, then the capture in stream order. public int SubmitFlipFromGpu(RecordingBuffer buffer, int handle, int index, int flipMode, long flipArg, out ulong requestId) { + DrainDraws(); _ = buffer; _translation.PrepareFlip(handle, index); var result = VideoOutExports.TryReserveFlipRequest(handle, index, flipMode, flipArg, gpuQueued: true, out requestId); @@ -563,11 +864,16 @@ internal static unsafe partial class VulkanVideoPresenter // The interpreter must suspend before capture so this worker can present queued frames. public void WaitForSubmitSlot() => throw SubmissionScheduler.Fatal("The GPU worker cannot wait for a flip slot. Suspend the command stream before capture."); - public void CompleteFlip(ulong requestId) => VideoOutExports.CompleteFlip(requestId); + public void CompleteFlip(ulong requestId) + { + DrainDraws(); + VideoOutExports.CompleteFlip(requestId); + } // Copies the display surface through the store; presentation shows the copy once its tick retires. internal void CaptureFlip(int handle, int index, ulong requestId, int flipMode, long flipArg) { + DrainDraws(); // A no-buffer flip has no image to capture; its completion still retires in order. if (VideoOutExports.ReleaseNoBufferFlip(index, requestId)) { @@ -586,7 +892,7 @@ internal static unsafe partial class VulkanVideoPresenter FlushBatchedGuestCommands(); RunGuestCacheCollection(endsFrame: true); - EnsureGuestSubmissionCapacity(); + EnsureGuestSubmissionCapacity(collectNow: true); long version; lock (_gate) { diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CommandThread.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CommandThread.cs new file mode 100644 index 0000000..0db0cd4 --- /dev/null +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.CommandThread.cs @@ -0,0 +1,511 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.Libs.VideoOut; + +using System.Diagnostics; +using SharpEmu.HLE; +using SharpEmu.Libs.Agc; +using SharpEmu.Libs.Gpu.GpuCommands; +using SharpEmu.Libs.Gpu.Rendering; +using SharpEmu.Libs.Gpu.Scheduling; + +internal static partial class VulkanVideoPresenter +{ + // The command stream is interpreted on its own thread, which hands its work to the render thread + // in stream order. Draws, dispatches, barriers, completions and the interpreter's own stores are + // queued; a callback that returns a result waits for the render thread to run everything before it. + private sealed class CommandThread : AgcExports.IDeferredWorkSink + { + // The thread runs this far ahead of the render thread at most: enough to keep it busy, and + // no more, since a GPU round trip (a readback, a label wait) waits for everything queued. + // Measured in Astro Bot: 4096 items halved the frame rate of a crowd scene, 16 starved the + // render thread, 32 to 64 kept it busy. + private const int MaxOutstanding = 64; + + private enum Kind : byte + { + Deferred, + Call, + Write, + } + + private sealed class Item + { + public Kind Kind; + public bool IsDraw; + public AgcExports.DeferredWork? Work; + public Action? Action; + public ulong Address; + public byte[]? Data; + // The guest range a queued fill or copy writes; zero for other items. + public ulong RangeSize; + // Resolution jobs submitted before this non-draw item; they finish before it runs. + public long JobsBefore; + public bool Sync; + public volatile bool Done; + public Exception? Failure; + } + + private readonly CommandStreamQueue _queue; + private readonly ICpuMemory _memory; + private readonly Action _runDeferred; + private readonly ParallelProgramPrefetch? _prefetch; + private readonly Thread _thread; + private readonly object _lock = new(); + private List _incoming = new(); + private List _draining = new(); + private int _drainIndex; + private int _outstanding; + // What the producer sleeps on: room in the queue, or its synchronous item. + private volatile bool _waitingForSpace; + private volatile bool _waitingForSync; + private volatile bool _consumerIdle; + private volatile bool _stopping; + private volatile Exception? _failure; + private long _enqueuedNonDraw; + private long _executedNonDraw; + + // The interpreter's stores the render thread has not applied yet, oldest first; reads on + // this thread see them over guest memory. + private readonly object _writeLock = new(); + private readonly Queue _pendingWrites = new(); + private volatile int _pendingWriteCount; + + // The destinations of queued fills and copies; a read on this thread that overlaps one waits + // for the render thread to run it. + private readonly List _pendingRanges = new(); + private volatile int _pendingRangeCount; + + private static readonly long RetryTicks = CommandStreamQueue.AllBlockedRetryMilliseconds * Stopwatch.Frequency / 1000L; + + public CommandThread(CommandStreamQueue queue, ICpuMemory memory, Action runDeferred, ParallelProgramPrefetch? prefetch) + { + _queue = queue; + _memory = memory; + _runDeferred = runDeferred; + _prefetch = prefetch; + _thread = new Thread(Run) { IsBackground = true, Name = "SharpEmu command stream" }; + } + + public void Start() => _thread.Start(); + + public bool IsProducerThread => Environment.CurrentManagedThreadId == _thread.ManagedThreadId; + + public long Gate => _enqueuedNonDraw; + + public bool HasWork => Volatile.Read(ref _outstanding) != 0 || _failure is not null; + + // Read by the render loop before it waits, so a queued item wakes it. + public bool ConsumerIdle + { + set => _consumerIdle = value; + } + + public bool IsAlive => _thread.IsAlive; + + // The producer side, on the command stream thread. + + public void EnqueueDeferred(AgcExports.DeferredWork work, bool isDraw) => + Enqueue(new Item { Kind = Kind.Deferred, Work = work, IsDraw = isDraw }); + + public void Post(Action action) => Enqueue(new Item { Kind = Kind.Call, Action = action }); + + public void Run(Action action) + { + var item = new Item { Kind = Kind.Call, Action = action, Sync = true }; + Enqueue(item); + WaitFor(item); + } + + public T Run(Func function) + { + T result = default!; + Run(() => { result = function(); }); + return result; + } + + public void PostWrite(ulong address, ReadOnlySpan data) + { + var item = new Item { Kind = Kind.Write, Address = address, Data = data.ToArray() }; + lock (_writeLock) + { + _pendingWrites.Enqueue(item); + _pendingWriteCount = _pendingWrites.Count; + } + + Enqueue(item); + } + + // A fill or copy the render thread runs in order; reads of its destination wait for it. + public void PostRange(Action action, ulong address, ulong size) + { + var item = new Item { Kind = Kind.Call, Action = action, Address = address, RangeSize = size }; + if (size != 0) + { + lock (_writeLock) + { + _pendingRanges.Add(item); + _pendingRangeCount = _pendingRanges.Count; + } + } + + Enqueue(item); + } + + // Runs every queued item when one of them is a fill or copy into this range. + public void WaitForPendingRange(ulong address, ulong size) + { + if (_pendingRangeCount != 0 && OverlapsPendingRange(address, size)) + { + Run(static () => { }); + } + } + + private bool OverlapsPendingRange(ulong address, ulong size) + { + lock (_writeLock) + { + foreach (var pending in _pendingRanges) + { + if (address < pending.Address + pending.RangeSize && pending.Address < address + size) + { + return true; + } + } + + return false; + } + } + + // Guest memory as the stream sees it: with its own stores applied, queued or not. + public bool ReadOverlay(ulong address, Span destination) + { + WaitForPendingRange(address, (ulong)destination.Length); + if (_pendingWriteCount == 0) + { + return _memory.TryRead(address, destination); + } + + lock (_writeLock) + { + if (!_memory.TryRead(address, destination)) + { + return false; + } + + var end = address + (ulong)destination.Length; + foreach (var write in _pendingWrites) + { + var writeEnd = write.Address + (ulong)write.Data!.Length; + if (write.Address >= end || writeEnd <= address) + { + continue; + } + + var start = Math.Max(address, write.Address); + var stop = Math.Min(end, writeEnd); + write.Data.AsSpan((int)(start - write.Address), (int)(stop - start)) + .CopyTo(destination[(int)(start - address)..]); + } + + return true; + } + } + + private void Enqueue(Item item) + { + ThrowIfFailed(); + if (!item.IsDraw) + { + item.JobsBefore = _prefetch?.Submitted ?? 0; + _enqueuedNonDraw++; + } + + if (Volatile.Read(ref _outstanding) >= MaxOutstanding) + { + WaitForSpace(); + } + + lock (_lock) + { + _incoming.Add(item); + } + + Interlocked.Increment(ref _outstanding); + if (_consumerIdle || item.Sync) + { + Presenter.WakeRenderThread(); + } + } + + private void WaitForSpace() + { + lock (_lock) + { + // The flag is set before the check: the render thread reads it after it changes what is checked. + for (;;) + { + _waitingForSpace = true; + Interlocked.MemoryBarrier(); + if (Volatile.Read(ref _outstanding) < MaxOutstanding / 2 || _failure is not null) + { + break; + } + + Monitor.Wait(_lock, 10); + } + + _waitingForSpace = false; + } + + ThrowIfFailed(); + } + + private void WaitFor(Item item) + { + var spinner = new SpinWait(); + for (var spin = 0; spin < 100 && !item.Done; spin++) + { + spinner.SpinOnce(sleep1Threshold: -1); + } + + if (!item.Done) + { + lock (_lock) + { + for (;;) + { + _waitingForSync = true; + Interlocked.MemoryBarrier(); + if (item.Done || _failure is not null) + { + break; + } + + Monitor.Wait(_lock, 10); + } + + _waitingForSync = false; + } + } + + if (item.Failure is { } failure) + { + throw new InvalidOperationException("A command stream callback failed on the render thread.", failure); + } + + if (!item.Done) + { + ThrowIfFailed(); + } + } + + // The consumer side, on the render thread. Runs queued items until the deadline or until + // none is left; returns how many ran. + public int RunBatch(long deadline, int maxItems, Action betweenItems) + { + if (_drainIndex == _draining.Count) + { + _draining.Clear(); + _drainIndex = 0; + lock (_lock) + { + (_incoming, _draining) = (_draining, _incoming); + } + + if (_draining.Count == 0) + { + ThrowIfFailed(); + return 0; + } + } + + var ran = 0; + while (_drainIndex < _draining.Count) + { + var item = _draining[_drainIndex]; + _draining[_drainIndex++] = null!; + Execute(item); + ran++; + var outstanding = Interlocked.Decrement(ref _outstanding); + // One pulse per wait: the producer sets its flag again if it keeps waiting. + var wakeForSync = item.Sync && _waitingForSync; + var wakeForSpace = _waitingForSpace && outstanding < MaxOutstanding / 2; + if (wakeForSync || wakeForSpace) + { + if (wakeForSync) + { + _waitingForSync = false; + } + + if (wakeForSpace) + { + _waitingForSpace = false; + } + + lock (_lock) + { + Monitor.PulseAll(_lock); + } + } + + betweenItems(); + if (ran >= maxItems || ((ran & 31) == 0 && Stopwatch.GetTimestamp() >= deadline)) + { + break; + } + } + + return ran; + } + + private void Execute(Item item) + { + if (item.IsDraw) + { + _runDeferred(item.Work!); + return; + } + + _prefetch?.WaitFinished(item.JobsBefore); + try + { + switch (item.Kind) + { + case Kind.Deferred: + _runDeferred(item.Work!); + break; + case Kind.Call: + item.Action!(); + break; + case Kind.Write: + ApplyWrite(item); + break; + } + } + catch (Exception exception) when (item.Sync) + { + item.Failure = exception; + } + catch (Exception exception) + { + _failure ??= exception; + throw; + } + finally + { + _executedNonDraw++; + if (item.RangeSize != 0) + { + lock (_writeLock) + { + _pendingRanges.Remove(item); + _pendingRangeCount = _pendingRanges.Count; + } + } + + _prefetch?.AdvanceGate(_executedNonDraw); + if (item.Sync) + { + item.Done = true; + } + } + } + + private void ApplyWrite(Item item) + { + lock (_writeLock) + { + var written = _memory.TryWrite(item.Address, item.Data); + if (!ReferenceEquals(_pendingWrites.Peek(), item)) + { + throw SubmissionScheduler.Fatal("The command stream stores ran out of order."); + } + + _pendingWrites.Dequeue(); + _pendingWriteCount = _pendingWrites.Count; + if (!written) + { + throw SubmissionScheduler.Fatal( + $"The command stream cannot write guest memory: address=0x{item.Address:X16} size={item.Data!.Length}."); + } + } + } + + public void ThrowIfFailed() + { + if (_failure is { } failure) + { + throw new InvalidOperationException("The command stream thread failed.", failure); + } + } + + // Ends the thread after its current slice; the render thread keeps running its items + // meanwhile, since the slice may wait for one. + public void Stop(Action runItems) + { + _stopping = true; + _queue.Wake(); + var deadline = Stopwatch.GetTimestamp() + 5 * Stopwatch.Frequency; + while (_thread.IsAlive && Stopwatch.GetTimestamp() < deadline) + { + runItems(); + _thread.Join(1); + } + + runItems(); + } + + private void Run() + { + try + { + var lastProgress = Stopwatch.GetTimestamp(); + while (!_stopping) + { + var result = _queue.ProcessOne(); + switch (result) + { + case SliceResult.NoWork: + _ = _queue.WaitForWork(8); + break; + case SliceResult.AllBlocked: + WaitBlocked(ref lastProgress); + break; + case SliceResult.BlockedWithoutProgress: + if (!_queue.HasUnblockedPending) + { + WaitBlocked(ref lastProgress); + } + + break; + default: + lastProgress = Stopwatch.GetTimestamp(); + break; + } + } + } + catch (Exception exception) + { + _failure ??= exception; + Presenter.WakeRenderThread(); + } + } + + // Blocked heads retry when a GPU completion or a presentation unblocks them, or every + // retry interval without progress. + private void WaitBlocked(ref long lastProgress) + { + var remaining = RetryTicks - (Stopwatch.GetTimestamp() - lastProgress); + if (remaining <= 0) + { + _queue.RetryBlocked(); + lastProgress = Stopwatch.GetTimestamp(); + return; + } + + _ = _queue.WaitForRetryInterval((int)Math.Max(1, remaining * 1000 / Stopwatch.Frequency)); + } + } +} diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.ConcurrentResolve.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.ConcurrentResolve.cs new file mode 100644 index 0000000..41a404d --- /dev/null +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.ConcurrentResolve.cs @@ -0,0 +1,134 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.Libs.VideoOut; + +using SharpEmu.Libs.Gpu.Rendering; +using SharpEmu.Libs.Gpu.Scheduling; + +internal static unsafe partial class VulkanVideoPresenter +{ + private sealed partial class Presenter : IConcurrentResolveHost + { + // This partial serves the guest readers of program resolution on a thread resolving draws + // ahead of the render thread. That thread never synchronizes GPU-owned memory: a word the + // GPU owns abandons the resolution, which the render thread then does itself. Every + // ownership answer it relied on is recorded and checked again when the result is taken. + + private enum ConcurrentQuery : byte + { + DirtyBytes, + CleanWord, + Resident, + ResidentClean, + } + + private sealed class ConcurrentAssumptions(long bufferVersion, long writebackEpoch) + { + public readonly long BufferVersion = bufferVersion; + public readonly long WritebackEpoch = writebackEpoch; + public readonly List<(ConcurrentQuery Query, ulong Address, ulong Size, bool Answer)> Answers = new(); + public bool ReadsImageOwnership; + + public void Record(ConcurrentQuery query, ulong address, ulong size, bool answer) + { + Answers.Add((query, address, size, answer)); + ReadsImageOwnership |= query is ConcurrentQuery.CleanWord or ConcurrentQuery.ResidentClean; + } + } + + // Set only on the resolving thread, around one resolution. + [ThreadStatic] + private static ConcurrentAssumptions? _concurrentAssumptions; + + void IConcurrentResolveHost.BeginConcurrentResolve() => + _concurrentAssumptions = new ConcurrentAssumptions(_bufferCache.GpuModifiedVersion, GpuWritebackEpoch.Value); + + object IConcurrentResolveHost.EndConcurrentResolve() + { + var assumptions = _concurrentAssumptions ?? throw new InvalidOperationException("No concurrent resolution is open."); + _concurrentAssumptions = null; + return assumptions; + } + + // GPU data stored into guest memory since the resolution may have changed what it read; + // otherwise the answers stand while the ownership they came from has not changed. + bool IConcurrentResolveHost.IsStillCurrent(object assumptionsObject) + { + var assumptions = (ConcurrentAssumptions)assumptionsObject; + if (assumptions.WritebackEpoch != GpuWritebackEpoch.Value) + { + return false; + } + + if (assumptions.BufferVersion == _bufferCache.GpuModifiedVersion && !assumptions.ReadsImageOwnership) + { + return true; + } + + foreach (var (query, address, size, answer) in assumptions.Answers) + { + var now = query switch + { + ConcurrentQuery.DirtyBytes => _bufferCache.HasGpuDirtyBytes(address, size), + ConcurrentQuery.CleanWord => IsGpuOwnedForCleanRead(address, size), + ConcurrentQuery.Resident => IsGpuOwnedForResidentRead(address, size, clean: false), + _ => IsGpuOwnedForResidentRead(address, size, clean: true), + }; + if (now != answer) + { + return false; + } + } + + return true; + } + + private bool IsGpuOwnedForCleanRead(ulong address, ulong size) => + (_bufferCache.MayHaveGpuDirtyPages(address, size) && _bufferCache.HasGpuDirtyPages(address, size)) || + _bufferCache.HasGpuDirtyBytes(address, size) || + _imageCache.HasGpuModifiedImageBytes(address, size); + + private bool IsGpuOwnedForResidentRead(ulong address, ulong size, bool clean) => + _bufferCache.HasGpuDirtyBytes(address, size) || + (clean && (_bufferCache.HasGpuDirtyPages(address, size) || _imageCache.HasGpuModifiedImageBytes(address, size))); + + private bool TryReadGuestWordConcurrent(ulong address, out uint word, ConcurrentAssumptions assumptions) + { + word = 0; + if (!_guestMemory.CanRead(address, sizeof(uint))) + { + return false; + } + + var dirty = _bufferCache.HasGpuDirtyBytes(address, sizeof(uint)); + assumptions.Record(ConcurrentQuery.DirtyBytes, address, sizeof(uint), dirty); + if (dirty) + { + throw new SubmissionScheduler.AbandonedWorkException($"The resolution reads GPU-owned memory: address=0x{address:X16}."); + } + + Span bytes = stackalloc byte[sizeof(uint)]; + if (!_guestMemory.TryRead(address, bytes)) + { + return false; + } + + word = System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(bytes); + return true; + } + + private bool TryReadResidentGuestBytesConcurrent(ulong address, Span destination, bool clean, ConcurrentAssumptions assumptions) + { + var size = (ulong)destination.Length; + if (!_guestMemory.CanRead(address, size)) + { + return false; + } + + var owned = IsGpuOwnedForResidentRead(address, size, clean); + assumptions.Record(clean ? ConcurrentQuery.ResidentClean : ConcurrentQuery.Resident, address, size, owned); + return !owned && _guestMemory.TryRead(address, destination); + } + } +} diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Descriptors.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Descriptors.cs index cdc3b6d..0473fb0 100644 --- a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Descriptors.cs +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Descriptors.cs @@ -111,21 +111,81 @@ internal static unsafe partial class VulkanVideoPresenter Atomic: image.Atomic); private static readonly System.Collections.Concurrent.ConcurrentDictionary _tracedTextureBindings = new(); + // The last lookup of one image of one program. A draw binds the same descriptor as the + // previous draw of its program most of the time; while no image the lookup could find has + // changed (the cache's lookup generation), the request and the image found are the same. + private struct ImageBindingMemo + { + public uint[]? Words; + public ulong Generation; + public TextureRequestResolution Resolution; + public ResourceSlotIdentifier Found; + + public readonly bool Matches(uint[] words, ulong generation) => + Words is not null && Generation == generation && words.AsSpan().SequenceEqual(Words); + } + + private Dictionary? _imageBindingMemos; + private ShaderProgramInfo? _lastMemoProgram; + private ImageBindingMemo[] _lastMemos = []; + + // A draw's images resolve one after another for one program, so the last program's memos + // are kept at hand. + private ref ImageBindingMemo ImageBindingMemoOf(ShaderProgramInfo program, int index) + { + if (!ReferenceEquals(program, _lastMemoProgram) || index >= _lastMemos.Length) + { + var all = _imageBindingMemos ??= new Dictionary(ReferenceEqualityComparer.Instance); + if (!all.TryGetValue(program, out var memos) || index >= memos.Length) + { + Array.Resize(ref memos, Math.Max(index + 1, program.Resources?.Info.Images.Count ?? 0)); + all[program] = memos; + } + + _lastMemoProgram = program; + _lastMemos = memos; + } + + return ref _lastMemos[index]; + } // Render-state discovery for one shader image; the view is acquired later with the draw. private TextureResource ResolveImageBinding(ImageResource image, uint[] words, ShaderProgramInfo program, int index) { + ref var memo = ref ImageBindingMemoOf(program, index); if (words.Length < 4) { throw SubmissionScheduler.Fatal($"An image descriptor is too short: image={index} words={words.Length} hash=0x{program.Hash:X16}."); } var storage = image.ResourceClass == ShaderCompiler.Resources.ImageResourceClass.Storage; - var resolution = ImageRequestBuilders.Texture(words, ShapeOf(image)); _ = BeginBatchedGuestCommands(); - var request = resolution.Request; - var imageIdentifier = _imageCache.FindImage(ref request, resolution.ExactFormat); - resolution = resolution with { Request = request }; + TextureRequestResolution resolution; + ResourceSlotIdentifier imageIdentifier; + if (memo.Matches(words, _imageCache.LookupGeneration) && _imageCache.TryTouchRemembered(memo.Found)) + { + resolution = memo.Resolution; + imageIdentifier = memo.Found; + } + else + { + resolution = ImageRequestBuilders.Texture(words, ShapeOf(image)); + var request = resolution.Request; + imageIdentifier = _imageCache.FindImage(ref request, resolution.ExactFormat); + resolution = resolution with { Request = request }; + // An empty range or DCC metadata needs the full lookup on every draw. + memo = !ImageDescription.IsEmptyRange(request.Description.Data) && + _imageCache.GetImage(imageIdentifier).Description.DccSliceSize == 0 + ? new ImageBindingMemo + { + Words = (uint[])words.Clone(), + Generation = _imageCache.LookupGeneration, + Resolution = resolution, + Found = imageIdentifier, + } + : default; + } + imageIdentifier = ImageRequestBuilders.ValidateTextureOwner(_imageCache, imageIdentifier, resolution); BindImage(imageIdentifier, storage); var descriptor = new TextureDescriptorWords(words); @@ -151,16 +211,47 @@ internal static unsafe partial class VulkanVideoPresenter Console.Error.WriteLine(line); } } - return new TextureResource + var texture = RentTextureResource(); + texture.Address = descriptor.BaseAddress; + texture.ImageIdentifier = imageIdentifier; + texture.Request = resolution.Request; + texture.IsStorage = storage; + texture.DestinationSelect = words[3] & 0xFFFu; + texture.Width = descriptor.Width; + texture.Height = descriptor.Height; + return texture; + } + + // Binding descriptions are large (an image request carries its whole mip layout) and one is made + // per shader image per draw, so they are reused once the submission that read them has retired. + // Only the render thread rents and returns them. + // Sized for the bindings of every draw still in flight (a few frames of a thousand draws each); + // a smaller cap drops returns during busy scenes and the pool allocates again. + private const int MaxPooledTextureResources = 32768; + private Stack? _texturePool; + + private TextureResource RentTextureResource() + { + if (_texturePool is not null && _texturePool.TryPop(out var texture)) { - Address = descriptor.BaseAddress, - ImageIdentifier = imageIdentifier, - Request = request, - IsStorage = storage, - DestinationSelect = words[3] & 0xFFFu, - Width = descriptor.Width, - Height = descriptor.Height, - }; + texture.InPool = false; + return texture; + } + + return new TextureResource { Pooled = true }; + } + + private void ReturnTextureResource(TextureResource texture) + { + _texturePool ??= new Stack(); + if (!texture.Pooled || texture.InPool || _texturePool.Count >= MaxPooledTextureResources) + { + return; + } + + texture.Reset(); + texture.InPool = true; + _texturePool.Push(texture); } // Compare bits stay only on depth-compare samplers; a forced point sampler drops its filters. @@ -768,8 +859,11 @@ internal static unsafe partial class VulkanVideoPresenter throw SubmissionScheduler.Fatal($"A stage does not belong to the bind point: stage={stage.Program.Stage} bindPoint={bindPoint}."); } - foreach (var binding in stage.Layout.Descriptors) + // Indexed loops over the layout's lists: a foreach on IReadOnlyList allocates an enumerator per draw. + var countedBindings = stage.Layout.Descriptors; + for (var bindingIndex = 0; bindingIndex < countedBindings.Count; bindingIndex++) { + var binding = countedBindings[bindingIndex]; writeCount++; var count = (int)DescriptorWriter.DescriptorCount(binding); if (ImageDescriptorBinding.ResourceClass(binding.Kind) != ShaderCompiler.Resources.ImageResourceClass.None || binding.Kind == DescriptorBindingKind.Samplers) @@ -862,8 +956,11 @@ internal static unsafe partial class VulkanVideoPresenter var occurrences = occurrenceScratch[..descriptors.Images.Length]; occurrences.Clear(); - foreach (var binding in stage.Layout.Descriptors) + var bindings = stage.Layout.Descriptors; + for (var bindingIndex = 0; bindingIndex < bindings.Count; bindingIndex++) { + var binding = bindings[bindingIndex]; + var resources = binding.Resources; var write = new WriteDescriptorSet { SType = StructureType.WriteDescriptorSet, @@ -875,8 +972,9 @@ internal static unsafe partial class VulkanVideoPresenter var imageStart = imageIndex; if (ImageDescriptorBinding.ResourceClass(binding.Kind) != ShaderCompiler.Resources.ImageResourceClass.None) { - foreach (var resource in binding.Resources) + for (var resourceIndex = 0; resourceIndex < resources.Count; resourceIndex++) { + var resource = resources[resourceIndex]; imageInfos[imageIndex++] = ImageInfo(descriptors.Images[(int)resource], occurrences[(int)resource]++, program, (int)resource); } } @@ -885,8 +983,9 @@ internal static unsafe partial class VulkanVideoPresenter switch (binding.Kind) { case DescriptorBindingKind.Buffers: - foreach (var resource in binding.Resources) + for (var resourceIndex = 0; resourceIndex < resources.Count; resourceIndex++) { + var resource = resources[resourceIndex]; var view = descriptors.Buffers[(int)resource]; if (view.Buffer.Handle == 0) { @@ -925,8 +1024,9 @@ internal static unsafe partial class VulkanVideoPresenter } case DescriptorBindingKind.Samplers: - foreach (var resource in binding.Resources) + for (var resourceIndex = 0; resourceIndex < resources.Count; resourceIndex++) { + var resource = resources[resourceIndex]; var sampler = descriptors.Samplers[(int)resource]; if (sampler.Handle == 0) { diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Device.Setup.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Device.Setup.cs index 9d0d1d0..057605d 100644 --- a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Device.Setup.cs +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Device.Setup.cs @@ -613,6 +613,10 @@ internal static unsafe partial class VulkanVideoPresenter subgroupSizeControl.MinSubgroupSize <= RdnaSubgroupSize && subgroupSizeControl.MaxSubgroupSize >= RdnaSubgroupSize && (subgroupSizeControl.RequiredSubgroupSizeStages & ShaderStageFlags.ComputeBit) != 0; + _canRequireComputeSubgroup64 = + subgroupSizeControl.MinSubgroupSize <= Wave64SubgroupSize && + subgroupSizeControl.MaxSubgroupSize >= Wave64SubgroupSize && + (subgroupSizeControl.RequiredSubgroupSizeStages & ShaderStageFlags.ComputeBit) != 0; _maxComputeWorkgroupSubgroups = subgroupSizeControl.MaxComputeWorkgroupSubgroups; // MoltenVK reports a zero OpArrayLength for storage buffers bound through push // descriptors, which turns every bounds-checked load into zero and drops every store. @@ -945,6 +949,7 @@ internal static unsafe partial class VulkanVideoPresenter var supportsNullDescriptor = robustness2Features.NullDescriptor; var supportsRobustness2 = supportsRobustImageAccess2 || supportsNullDescriptor; _canRequireComputeSubgroup32 &= vulkan13Features.SubgroupSizeControl; + _canRequireComputeSubgroup64 &= vulkan13Features.SubgroupSizeControl; SetSharedInt64AtomicsCapability(supportsSharedInt64Atomics); if (!supportsSharedInt64Atomics) { @@ -1157,7 +1162,7 @@ internal static unsafe partial class VulkanVideoPresenter SType = StructureType.PhysicalDeviceVulkan13Features, DynamicRendering = true, Synchronization2 = true, - SubgroupSizeControl = _canRequireComputeSubgroup32, + SubgroupSizeControl = _canRequireComputeSubgroup32 || _canRequireComputeSubgroup64, PNext = renderingChain, }; var features2 = new PhysicalDeviceFeatures2 diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Draws.Recording.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Draws.Recording.cs index 08a48bb..552baeb 100644 --- a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Draws.Recording.cs +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Draws.Recording.cs @@ -275,8 +275,12 @@ internal static unsafe partial class VulkanVideoPresenter { RecycleHostBuffer(texture.StagingBuffer, texture.StagingMemory); } + + ReturnTextureResource(texture); } + resources.Textures = []; + foreach (var snapshot in resources.FeedbackSnapshots) { RetireFeedbackSnapshot(snapshot); diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.GuestSubmissions.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.GuestSubmissions.cs index 4c56738..43d2010 100644 --- a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.GuestSubmissions.cs +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.GuestSubmissions.cs @@ -108,9 +108,23 @@ internal static unsafe partial class VulkanVideoPresenter retireBuffers)); } - private void EnsureGuestSubmissionCapacity() + // Collecting completed submissions polls the GPU timeline twice and may wait for the priority + // worker. Every draw used to pay that; completed work only frees resources and runs deferred + // callbacks, so a collection a few hundred microseconds later changes nothing but how long + // those resources stay reserved. A full in-flight queue still collects, and waits, at once. + private static readonly long CollectIntervalTicks = System.Diagnostics.Stopwatch.Frequency / 4000; + private long _lastCollectTicks; + + private void EnsureGuestSubmissionCapacity(bool collectNow = false) { using var profileScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.SubmissionCapacity); + var now = System.Diagnostics.Stopwatch.GetTimestamp(); + if (!collectNow && _pendingGuestSubmissions.Count < MaxInFlightGuestSubmissions && now - _lastCollectTicks < CollectIntervalTicks) + { + return; + } + + _lastCollectTicks = now; CollectCompletedGuestSubmissions(waitForOldest: false); if (_pendingGuestSubmissions.Count >= MaxInFlightGuestSubmissions) { diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Pipelines.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Pipelines.cs index 3b0e354..caa9137 100644 --- a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Pipelines.cs +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Pipelines.cs @@ -67,23 +67,43 @@ internal static unsafe partial class VulkanVideoPresenter uint IShaderPipelineHost.MaxPushDescriptors => _maxPushDescriptors; // Wave64 compute runs natively when the device's subgroup is that wide; smaller devices emulate it. + bool IShaderPipelineHost.ComputeWave64SubgroupNative => + Volatile.Read(ref _nativeSubgroupSize) >= 64 && (_canRequireComputeSubgroup64 || Volatile.Read(ref _nativeSubgroupSize) == 64); + bool IShaderPipelineHost.ComputeWave64Supported => Volatile.Read(ref _nativeSubgroupSize) >= 64; - // Only a 64-invocation wave64 workgroup is translated for either host subgroup width. Every - // other compute translation maps a guest wave to 32-lane host subgroups, and a 64-lane host - // subgroup (AMD's default) left lanes 32..63 inactive: a wave64 8x8x8 group lost rows 4..7. + // Wave32 compute maps a guest wave to 32-lane host subgroups, and a 64-lane host subgroup + // (AMD's default) left lanes 32..63 inactive, so those pipelines are pinned to 32. Wave64 + // compute is pinned to 64 instead when the host runs it natively. private const uint RdnaSubgroupSize = 32; private bool _canRequireComputeSubgroup32; private uint _maxComputeWorkgroupSubgroups; + private const uint Wave64SubgroupSize = 64; + private bool _canRequireComputeSubgroup64; + + // A native wave64 translation reads one guest wave as one host subgroup, so the driver must + // not pick 32-lane subgroups for it (RDNA drivers may do so for compute). + private bool RequiresComputeSubgroup64(ComputeInputInfo input) + { + return _canRequireComputeSubgroup64 && input.WaveSize == 64 && + ((IShaderPipelineHost)this).ComputeWave64SubgroupNative; + } + private bool RequiresComputeSubgroup32(ComputeInputInfo input) { var invocations = (ulong)Math.Max(input.ThreadsX, 1) * Math.Max(input.ThreadsY, 1) * Math.Max(input.ThreadsZ, 1); return _canRequireComputeSubgroup32 && - !(input.WaveSize == 64 && invocations == 64) && + !(input.WaveSize == 64 && ((IShaderPipelineHost)this).ComputeWave64SubgroupNative) && invocations <= (ulong)_maxComputeWorkgroupSubgroups * RdnaSubgroupSize; } + // The subgroup width a compute pipeline must be pinned to, or 0 to leave it to the driver. + private uint ComputeRequiredSubgroupSize(ComputeInputInfo input) => + RequiresComputeSubgroup64(input) ? Wave64SubgroupSize + : RequiresComputeSubgroup32(input) ? RdnaSubgroupSize + : 0u; + bool IShaderPipelineHost.GraphicsSubgroupOperationsEnabled => GraphicsSubgroupOperationsEnabled; bool IShaderPipelineHost.SharedInt64AtomicsEnabled => SharedInt64AtomicsEnabled; @@ -101,6 +121,11 @@ internal static unsafe partial class VulkanVideoPresenter // A range the GPU wrote is downloaded first, so the word is what the guest CPU would read. public bool TryReadGuestWord(ulong address, out uint word) { + if (_concurrentAssumptions is { } assumptions) + { + return TryReadGuestWordConcurrent(address, out word, assumptions); + } + using var profile = ResourceMaterializationProfile.Measure(ResourceMaterializationProfile.Phase.GuestRead); word = 0; var synchronized = false; @@ -170,9 +195,9 @@ internal static unsafe partial class VulkanVideoPresenter return false; } - if ((_bufferCache.MayHaveGpuDirtyPages(address, sizeof(uint)) && _bufferCache.HasGpuDirtyPages(address, sizeof(uint))) || - _bufferCache.HasGpuDirtyBytes(address, sizeof(uint)) || - _imageCache.HasGpuModifiedImageBytes(address, sizeof(uint))) + var owned = IsGpuOwnedForCleanRead(address, sizeof(uint)); + _concurrentAssumptions?.Record(ConcurrentQuery.CleanWord, address, sizeof(uint), owned); + if (owned) { return false; } @@ -190,6 +215,11 @@ internal static unsafe partial class VulkanVideoPresenter // The same ownership rules as the two word readers, checked once for a whole range. public bool TryReadResidentGuestBytes(ulong address, Span destination, bool clean) { + if (_concurrentAssumptions is { } assumptions) + { + return TryReadResidentGuestBytesConcurrent(address, destination, clean, assumptions); + } + var size = (ulong)destination.Length; if (clean || !IsCleanReadPage(address, size)) { @@ -967,15 +997,16 @@ internal static unsafe partial class VulkanVideoPresenter Pipeline pipeline; try { + var pin = ComputeRequiredSubgroupSize(description.Input); var requiredSubgroupSize = new PipelineShaderStageRequiredSubgroupSizeCreateInfo { SType = StructureType.PipelineShaderStageRequiredSubgroupSizeCreateInfo, - RequiredSubgroupSize = RdnaSubgroupSize, + RequiredSubgroupSize = pin, }; var stageInfo = new PipelineShaderStageCreateInfo { SType = StructureType.PipelineShaderStageCreateInfo, - PNext = RequiresComputeSubgroup32(description.Input) ? &requiredSubgroupSize : null, + PNext = pin != 0 ? &requiredSubgroupSize : null, Stage = ShaderStageFlags.ComputeBit, Module = computeModule, PName = entryPoint, @@ -1099,6 +1130,7 @@ internal static unsafe partial class VulkanVideoPresenter var device = _device; var cacheSource = GetGuestPipelineCacheSource(ComputeCacheKey(description.Stage.Hash, computeModule.Handle)); var vk = _vk; + var requiredSubgroupSize = ComputeRequiredSubgroupSize(description.Input); var started = new PendingComputePipeline { SetLayout = setLayout, @@ -1122,7 +1154,7 @@ internal static unsafe partial class VulkanVideoPresenter // Importing a MoltenVK cache compiles its MSL libraries. // Keep that work inside the same bounded compiler slot. var cache = ResolveGuestPipelineCache(cacheSource); - return CompileComputePipeline(vk, device, cache, computeModule, layout); + return CompileComputePipeline(vk, device, cache, computeModule, layout, requiredSubgroupSize); } finally { @@ -1146,14 +1178,21 @@ internal static unsafe partial class VulkanVideoPresenter Device device, PipelineCache cache, ShaderModule module, - PipelineLayout layout) + PipelineLayout layout, + uint requiredSubgroupSize) { var entryPoint = (byte*)SilkMarshal.StringToPtr("main"); try { + var subgroupInfo = new PipelineShaderStageRequiredSubgroupSizeCreateInfo + { + SType = StructureType.PipelineShaderStageRequiredSubgroupSizeCreateInfo, + RequiredSubgroupSize = requiredSubgroupSize, + }; var stageInfo = new PipelineShaderStageCreateInfo { SType = StructureType.PipelineShaderStageCreateInfo, + PNext = requiredSubgroupSize != 0 ? &subgroupInfo : null, Stage = ShaderStageFlags.ComputeBit, Module = module, PName = entryPoint, diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Present.Hdr.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Present.Hdr.cs index 025312b..ce28660 100644 --- a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Present.Hdr.cs +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Present.Hdr.cs @@ -357,6 +357,7 @@ internal static unsafe partial class VulkanVideoPresenter _commandBuffer, PipelineBindPoint.Graphics, isHdr ? _hdrPqPipeline : _hdrPipeline); + SharpEmu.Libs.Gpu.Vulkan.GraphicsDynamicStateEpoch.Advance(); var descriptorSet = isHdr ? _hdrPqDescriptorSets[imageIndex] : _hdrDescriptorSets[imageIndex]; diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.RenderLoop.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.RenderLoop.cs index 307d6f9..a238d2b 100644 --- a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.RenderLoop.cs +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.RenderLoop.cs @@ -46,38 +46,50 @@ internal static unsafe partial class VulkanVideoPresenter using var profileScope = RenderPhaseProfile.Measure(RenderPhaseProfile.Phase.Idle); var gpuWorkInFlight = _pendingGuestSubmissions.Count > 0 || Array.Exists(_frameInFlight, static pending => pending); - var blockedRetryWait = BlockedRetryWaitMilliseconds(); - lock (_gate) + // The command stream thread retries its blocked heads itself and wakes this thread for its + // items; before it starts, queued submissions start it on the next tick. + var commandThread = _commandThread; + if (commandThread is not null) { - if (_closed || - Volatile.Read(ref _presenterCloseRequested) || - _relay.HasPendingCommands || - (_vulkanReady && _commandStream.HasUnblockedPending) || - blockedRetryWait == 0 || - HasReadyPresentationLocked()) - { - return; - } + commandThread.ConsumerIdle = true; + } - var waitMilliseconds = gpuWorkInFlight ? 1 : 8; - if (blockedRetryWait is { } retryWait) + try + { + lock (_gate) { - waitMilliseconds = Math.Min(waitMilliseconds, retryWait); - } + if (_closed || + Volatile.Read(ref _presenterCloseRequested) || + _relay.HasPendingCommands || + (commandThread is not null ? commandThread.HasWork : _vulkanReady && _commandStream.HasPending) || + HasReadyPresentationLocked()) + { + return; + } - var waitPhase = _commandStream.HasPending - ? RenderPhaseProfile.Phase.IdleBlockedCommands - : _pendingGuestImagePresentations.Count > 0 || _pendingVideoPresentations.Count > 0 || - (_latestPresentation is { } latest && latest.Sequence != _presentedSequence) - ? RenderPhaseProfile.Phase.IdlePendingPresentation - : RenderPhaseProfile.Phase.IdleNoQueuedWork; - using var waitProfile = RenderPhaseProfile.MeasureDetail(waitPhase); - SubmissionFlowProfile.Record(SubmissionFlowProfile.EventKind.WaitStarted, - detail: waitMilliseconds); - var signaled = System.Threading.Monitor.Wait(_gate, waitMilliseconds); - // The result describes the monitor wait, not the arrival of runnable guest work. - SubmissionFlowProfile.Record(signaled ? SubmissionFlowProfile.EventKind.WaitSignaled - : SubmissionFlowProfile.EventKind.WaitTimedOut, detail: (int)waitPhase); + var waitMilliseconds = gpuWorkInFlight ? 1 : 8; + + var waitPhase = _commandStream.HasPending + ? RenderPhaseProfile.Phase.IdleBlockedCommands + : _pendingGuestImagePresentations.Count > 0 || _pendingVideoPresentations.Count > 0 || + (_latestPresentation is { } latest && latest.Sequence != _presentedSequence) + ? RenderPhaseProfile.Phase.IdlePendingPresentation + : RenderPhaseProfile.Phase.IdleNoQueuedWork; + using var waitProfile = RenderPhaseProfile.MeasureDetail(waitPhase); + SubmissionFlowProfile.Record(SubmissionFlowProfile.EventKind.WaitStarted, + detail: waitMilliseconds); + var signaled = System.Threading.Monitor.Wait(_gate, waitMilliseconds); + // The result describes the monitor wait, not the arrival of runnable guest work. + SubmissionFlowProfile.Record(signaled ? SubmissionFlowProfile.EventKind.WaitSignaled + : SubmissionFlowProfile.EventKind.WaitTimedOut, detail: (int)waitPhase); + } + } + finally + { + if (commandThread is not null) + { + commandThread.ConsumerIdle = false; + } } } @@ -117,9 +129,16 @@ internal static unsafe partial class VulkanVideoPresenter return; } - using (RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.QueueRelay)) + if (_commandThread is not null) { - _relay.RunPendingCommands(); + RunRelayCommandsWithPrefetchHeld(); + } + else + { + using (RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.QueueRelay)) + { + _relay.RunPendingCommands(); + } } if (_deviceLost) { @@ -388,6 +407,7 @@ internal static unsafe partial class VulkanVideoPresenter _commandBuffer, PipelineBindPoint.Graphics, _barycentricPipeline); + SharpEmu.Libs.Gpu.Vulkan.GraphicsDynamicStateEpoch.Advance(); _vk.CmdDraw(_commandBuffer, 3, 1, 0, 0); _vk.CmdEndRenderPass(_commandBuffer); waitStage = PipelineStageFlags.ColorAttachmentOutputBit; diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Rendering.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Rendering.cs index 799601f..3c0fe2c 100644 --- a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Rendering.cs +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Rendering.cs @@ -12,6 +12,7 @@ using SharpEmu.Libs.Gpu.GpuCommands.Registers; using SharpEmu.Libs.Gpu.Images; using SharpEmu.Libs.Gpu.Pipelines; using SharpEmu.Libs.Gpu.Rendering; +using SharpEmu.Libs.Gpu.Vulkan; using SharpEmu.Libs.Gpu.Scheduling; using SharpEmu.Libs.Gpu.Vulkan; using Silk.NET.Core; @@ -280,14 +281,21 @@ internal static unsafe partial class VulkanVideoPresenter _boundGraphicsPipeline = null; } + // The acquisition below reads the attachment back into its result and keeps no reference to it, + // so one render-thread instance serves every draw. + private ColorAttachment? _colorAttachmentScratch; + ColorAttachmentAcquisition IRenderHost.AcquireColorAttachment(in ColorTargetState target) { - var attachment = new ColorAttachment - { - ImageIdentifier = target.Image, - Request = target.Resolution.Request, - Resolution = target.Resolution, - }; + var attachment = _colorAttachmentScratch ??= new ColorAttachment(); + attachment.ImageIdentifier = target.Image; + attachment.Request = target.Resolution.Request; + attachment.Resolution = target.Resolution; + attachment.Image = null!; + attachment.View = default; + attachment.Layout = default; + attachment.Clear = false; + attachment.ClearValue = default; AcquireColorAttachment(attachment); return new ColorAttachmentAcquisition( attachment.ImageIdentifier, @@ -644,29 +652,80 @@ internal static unsafe partial class VulkanVideoPresenter _vk.CmdClearDepthStencilImage(command, image.Backing.Handle, ImageLayout.TransferDstOptimal, &value, 1, &vkRange); } + // The dynamic state the last guest draw set, and where it holds: the command buffer and its + // tick (a new command buffer starts without state), and no other graphics pipeline since. + private DynamicDrawState _lastDynamicState; + private CommandBuffer _lastDynamicStateBuffer; + private ulong _lastDynamicStateTick; + private long _lastDynamicStateEpoch = -1; + public void SetDynamicState(in DynamicDrawState state) { using var profileScope = RenderPhaseProfile.MeasureDetail(RenderPhaseProfile.Phase.DrawDynamicStateRecording); var command = BeginBatchedGuestCommands(); - var viewport = new Viewport(state.ViewportX, state.ViewportY, state.ViewportWidth, state.ViewportHeight, state.ViewportMinDepth, state.ViewportMaxDepth); - _vk.CmdSetViewport(command, 0, 1, &viewport); - var scissor = new Rect2D( - new Offset2D(state.Scissor.Left, state.Scissor.Top), - new Extent2D((uint)(state.Scissor.Right - state.Scissor.Left), (uint)(state.Scissor.Bottom - state.Scissor.Top))); - _vk.CmdSetScissor(command, 0, 1, &scissor); - _vk.CmdSetLineWidth(command, state.LineWidth); - var blendConstants = stackalloc float[4] { state.BlendRed, state.BlendGreen, state.BlendBlue, state.BlendAlpha }; - _vk.CmdSetBlendConstants(command, blendConstants); - _vk.CmdSetDepthTestEnable(command, state.DepthTestEnabled); - _vk.CmdSetDepthWriteEnable(command, state.DepthWriteEnabled); - _vk.CmdSetDepthCompareOp(command, state.DepthCompare); - _vk.CmdSetDepthBiasEnable(command, state.DepthBiasEnabled); - if (state.DepthBiasEnabled) + // Every guest pipeline keeps these states dynamic, so they persist across its binds; + // only what changed since the last draw in this command buffer is set again. + var known = command.Handle == _lastDynamicStateBuffer.Handle && _scheduler.CurrentTick == _lastDynamicStateTick && + GraphicsDynamicStateEpoch.Value == _lastDynamicStateEpoch; + ref readonly var last = ref _lastDynamicState; + if (!known || state.ViewportX != last.ViewportX || state.ViewportY != last.ViewportY || + state.ViewportWidth != last.ViewportWidth || state.ViewportHeight != last.ViewportHeight || + state.ViewportMinDepth != last.ViewportMinDepth || state.ViewportMaxDepth != last.ViewportMaxDepth) + { + var viewport = new Viewport(state.ViewportX, state.ViewportY, state.ViewportWidth, state.ViewportHeight, state.ViewportMinDepth, state.ViewportMaxDepth); + _vk.CmdSetViewport(command, 0, 1, &viewport); + } + + if (!known || state.Scissor != last.Scissor) + { + var scissor = new Rect2D( + new Offset2D(state.Scissor.Left, state.Scissor.Top), + new Extent2D((uint)(state.Scissor.Right - state.Scissor.Left), (uint)(state.Scissor.Bottom - state.Scissor.Top))); + _vk.CmdSetScissor(command, 0, 1, &scissor); + } + + if (!known || state.LineWidth != last.LineWidth) + { + _vk.CmdSetLineWidth(command, state.LineWidth); + } + + if (!known || state.BlendRed != last.BlendRed || state.BlendGreen != last.BlendGreen || + state.BlendBlue != last.BlendBlue || state.BlendAlpha != last.BlendAlpha) + { + var blendConstants = stackalloc float[4] { state.BlendRed, state.BlendGreen, state.BlendBlue, state.BlendAlpha }; + _vk.CmdSetBlendConstants(command, blendConstants); + } + + if (!known || state.DepthTestEnabled != last.DepthTestEnabled) + { + _vk.CmdSetDepthTestEnable(command, state.DepthTestEnabled); + } + + if (!known || state.DepthWriteEnabled != last.DepthWriteEnabled) + { + _vk.CmdSetDepthWriteEnable(command, state.DepthWriteEnabled); + } + + if (!known || state.DepthCompare != last.DepthCompare) + { + _vk.CmdSetDepthCompareOp(command, state.DepthCompare); + } + + if (!known || state.DepthBiasEnabled != last.DepthBiasEnabled) + { + _vk.CmdSetDepthBiasEnable(command, state.DepthBiasEnabled); + } + + // The factors only matter while the bias is on; they are set when it turns on or changes. + if (state.DepthBiasEnabled && (!known || !last.DepthBiasEnabled || + state.DepthBiasConstantFactor != last.DepthBiasConstantFactor || state.DepthBiasClamp != last.DepthBiasClamp || + state.DepthBiasSlopeFactor != last.DepthBiasSlopeFactor)) { _vk.CmdSetDepthBias(command, state.DepthBiasConstantFactor, _supportsDepthBiasClamp ? state.DepthBiasClamp : 0f, state.DepthBiasSlopeFactor); } - if (state.StencilTestEnabled) + if (state.StencilTestEnabled && (!known || !last.StencilTestEnabled || + state.FrontStencil != last.FrontStencil || state.BackStencil != last.BackStencil)) { _vk.CmdSetStencilCompareMask(command, StencilFaceFlags.FaceFrontBit, state.FrontStencil.CompareMask); _vk.CmdSetStencilCompareMask(command, StencilFaceFlags.FaceBackBit, state.BackStencil.CompareMask); @@ -686,6 +745,11 @@ internal static unsafe partial class VulkanVideoPresenter colorWriteEnable.CmdSetColorWriteEnable(command, state.ColorWriteCount, enables); } + + _lastDynamicState = state; + _lastDynamicStateBuffer = command; + _lastDynamicStateTick = _scheduler.CurrentTick; + _lastDynamicStateEpoch = GraphicsDynamicStateEpoch.Value; } diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Scheduler.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Scheduler.cs index dbdff71..1a7e29f 100644 --- a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Scheduler.cs +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.Scheduler.cs @@ -120,6 +120,8 @@ internal static unsafe partial class VulkanVideoPresenter // Drain accepted submissions before closing the relay. // Cancel blocked submissions if a full retry cycle makes no progress. _commandStream.StopAccepting(); + // The command stream thread ends after its slice; the rest drains here. + _commandThread?.Stop(() => RunCommandStreamSlices(long.MaxValue)); var outcome = _commandStream.DrainForShutdown(cancelBlockedOnNoProgress: true); FlushBatchedGuestCommands(); Console.Error.WriteLine( diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.ShaderPrewarm.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.ShaderPrewarm.cs index d063eae..7b0cba7 100644 --- a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.ShaderPrewarm.cs +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.ShaderPrewarm.cs @@ -163,7 +163,8 @@ internal static unsafe partial class VulkanVideoPresenter { if (!ShaderProgramCache.TryCompilePrewarm( record, code, compiler, sharedInt64Atomics, execGuardElision, - _supportsShaderSignedZeroInfNanPreserveFloat32, out var compiled, out var layout, out var error)) + _supportsShaderSignedZeroInfNanPreserveFloat32, out var compiled, out var layout, out var error, + ((IShaderPipelineHost)this).ComputeWave64SubgroupNative)) { NoteShaderPrewarmFailure(record, error); return; @@ -181,7 +182,7 @@ internal static unsafe partial class VulkanVideoPresenter setLayout = CreateDescriptorSetLayout(bindings, out _, out _); pipelineLayout = CreatePipelineLayout(setLayout, ShaderStageFlags.ComputeBit); module = CreateShaderModule(payload); - var pipeline = CompileComputePipeline(_vk, _device, _pipelineCache, module, pipelineLayout); + var pipeline = CompileComputePipeline(_vk, _device, _pipelineCache, module, pipelineLayout, ComputeRequiredSubgroupSize(record.Info)); _vk.DestroyPipeline(_device, pipeline, null); Interlocked.Increment(ref _shaderPrewarmCompiled); } diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Alu.cs b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Alu.cs index bce01f0..a12642c 100644 --- a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Alu.cs +++ b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Alu.cs @@ -52,39 +52,7 @@ public static partial class Gen5SpirvTranslator } var value = GetRawSource(instruction, 0); - if (_subgroupInvocationIdInput != 0) - { - if (_emulateWave64) - { - value = BroadcastFirstWave64Active(value); - } - else - { - // SPIR-V's BroadcastFirst uses the first host-active - // invocation. Guest EXEC is modeled as data, so obtain the - // guest-active mask explicitly and broadcast from its first - // set lane instead. This also updates the private SGPR copy - // for lanes that are currently disabled and may be restored - // by a later saveexec sequence. - var activeLanes = _module.AddInstruction( - SpirvOp.GroupNonUniformBallot, - _uvec4Type, - UInt(3), - Load(_boolType, _exec)); - var activeLow = _module.AddInstruction( - SpirvOp.CompositeExtract, - _uintType, - activeLanes, - 0); - var firstActiveLane = Ext(73, _uintType, activeLow); - value = _module.AddInstruction( - SpirvOp.GroupNonUniformBroadcast, - _uintType, - UInt(3), - value, - firstActiveLane); - } - } + value = BroadcastFirstWaveActive(value); StoreS(instruction.Destinations[0].Value, value); return true; @@ -3422,9 +3390,7 @@ public static partial class Gen5SpirvTranslator var left = GetRawSource64(instruction, 0); if (instruction.Opcode.EndsWith("SaveexecB64", StringComparison.Ordinal)) { - var oldExec = _emulateWave64 && _subgroupInvocationIdInput != 0 - ? LoadS64(126) - : BooleanToWaveMask(Load(_boolType, _exec)); + var oldExec = CurrentExecMask(); var notLeft = _module.AddInstruction(SpirvOp.Not, _ulongType, left); var newExec = instruction.Opcode switch { @@ -3493,12 +3459,7 @@ public static partial class Gen5SpirvTranslator return false; } - if (!_emulateWave64 && _waveLaneCount != 64) - { - newExec = BitwiseAnd64( - newExec, - _module.Constant64(_ulongType, 0xFFFF_FFFFUL)); - } + newExec = NormalizeExecMask(newExec); StoreS64(destination, oldExec); StoreS64(126, newExec); @@ -4958,52 +4919,6 @@ public static partial class Gen5SpirvTranslator private uint GuestLaneSelect(uint selector) => BitwiseAnd(selector, UInt(_waveLaneCount == 64 ? 63u : RdnaWaveLaneCount - 1)); - // An emulated wave64 spans two host subgroups, so a subgroup broadcast cannot reach - // the other half. The selected guest lane publishes through workgroup scratch, like - // BroadcastFirstWave64Active. - private uint BroadcastWave64Lane(uint value, uint lane) => - ExchangeWave64Value(_module.AddInstruction(SpirvOp.IEqual, _boolType, GuestWaveLane(), lane), value); - - private uint BroadcastFirstWave64Active(uint value) - { - var lane = GuestWaveLane(); - var upperHalf = ShiftRightLogical(lane, UInt(5)); - var activeInHalf = OwnHalfBallot(Load(_boolType, _exec)); - var halfHasActive = IsNotZero(activeInHalf); - var firstInHalf = _module.AddInstruction( - SpirvOp.Select, - _uintType, - halfHasActive, - Ext(73, _uintType, activeInHalf), - UInt(0)); - var halfBase = BitwiseAnd(Load(_uintType, _subgroupInvocationIdInput), UInt(~31u)); - var halfValue = _module.AddInstruction( - SpirvOp.GroupNonUniformBroadcast, - _uintType, - UInt(3), - value, - IAdd(halfBase, firstInHalf)); - var exchange = BeginWave64Exchange(); - EmitConditional(IsHalfWaveLeader(lane), () => - { - Store(Wave64ExchangePointer(exchange, upperHalf), halfValue); - Store( - Wave64ExchangePointer(exchange, IAdd(UInt(2), upperHalf)), - _module.AddInstruction(SpirvOp.Select, _uintType, halfHasActive, UInt(1), UInt(0))); - }); - EmitWave64Barrier(); - var lowerValue = Load(_uintType, Wave64ExchangePointer(exchange, UInt(0))); - var upperValue = Load(_uintType, Wave64ExchangePointer(exchange, UInt(1))); - var lowerActive = IsNotZero(Load(_uintType, Wave64ExchangePointer(exchange, UInt(2)))); - var upperActive = IsNotZero(Load(_uintType, Wave64ExchangePointer(exchange, UInt(3)))); - return _module.AddInstruction( - SpirvOp.Select, - _uintType, - _module.AddInstruction(SpirvOp.LogicalAnd, _boolType, LogicalNot(lowerActive), upperActive), - upperValue, - lowerValue); - } - private void StoreCarryOut( Gen5ShaderInstruction instruction, uint carry) @@ -5038,20 +4953,15 @@ public static partial class Gen5SpirvTranslator var sourceValue = GetRawSource(instruction, 0); var selectedLane = BitwiseAnd(GetRawSource(instruction, 1), UInt(LaneSelectMask)); - if (_emulateWave64) + if (_stage == Gen5SpirvStage.Compute && _waveLowering is EmulatedWave64Lowering) { // The selected guest lane can belong to another host subgroup. // Read it even when the guest execution mask disables that lane. - StoreS(destination, BroadcastWave64Lane(sourceValue, selectedLane)); + StoreS(destination, BroadcastWaveLane(sourceValue, selectedLane)); } else if (_subgroupInvocationIdInput != 0 && _stage == Gen5SpirvStage.Compute) { - var broadcast = _module.AddInstruction( - SpirvOp.GroupNonUniformBroadcast, - _uintType, - UInt(3), - sourceValue, - selectedLane); + var broadcast = BroadcastWaveLane(sourceValue, selectedLane); StoreS(destination, ReadLaneSpillSlot(instruction, selectedLane, broadcast)); } else diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.CommonWave.cs b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.CommonWave.cs new file mode 100644 index 0000000..b843e5c --- /dev/null +++ b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.CommonWave.cs @@ -0,0 +1,27 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.ShaderCompiler.Vulkan; + +public static partial class Gen5SpirvTranslator +{ + private sealed partial class CompilationContext + { + private uint BroadcastFirstWaveActive(uint value) => + _waveLowering.BroadcastFirstActive(value); + + private uint BroadcastWaveLane(uint value, uint lane) => + _waveLowering.BroadcastLane(value, lane); + + private uint BroadcastWaveValue(uint writer, uint value, uint lane) => + _waveLowering.BroadcastValue(writer, value, lane); + + private uint CurrentExecMask() => + _waveLowering.CurrentExecMask(); + + private uint NormalizeExecMask(uint mask) => + _waveLowering.NormalizeExecMask(mask); + } +} + + diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.EmulatedWave64Lowering.cs b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.EmulatedWave64Lowering.cs new file mode 100644 index 0000000..7361761 --- /dev/null +++ b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.EmulatedWave64Lowering.cs @@ -0,0 +1,578 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.ShaderCompiler.Resources; + +namespace SharpEmu.ShaderCompiler.Vulkan; + +public static partial class Gen5SpirvTranslator +{ + private sealed partial class CompilationContext + { + private sealed class EmulatedWave64Lowering : IWaveLowering + { + private const uint ExchangeSlotCount = 4; + private const uint ExchangeDwordCount = ExchangeSlotCount * 2; + private const uint GuestWaveLaneCount = 64; + private static readonly bool HalfWaveMasks = !string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_WAVE64_HALF_MASKS"), + "0", + StringComparison.Ordinal); + + private readonly CompilationContext _context; + private uint _exchange; + private uint _exchangeElementPointer; + private uint _exchangeOffset; + private uint _exchangeParity; + // Several guest waves share one workgroup. Each gets its own exchange area plus an + // arrival counter, and its two halves meet at that counter instead of at a workgroup + // barrier, which other waves (possibly in other blocks) would never reach. + private readonly uint _waveCount; + private readonly bool _multiWave; + private readonly uint _waveStride; + private uint _rendezvousEpoch; + private Ir.Gen5Wave64HalfMaskPlan? _halfMaskPlan; + private bool _halfMaskPlanBuilt; + private bool _synchronizeSharedMemory; + private SharedMemoryPhase _sharedMemoryPhase; + private uint? _emittingPc; + + private enum SharedMemoryPhase { None, Read, Write } + + public EmulatedWave64Lowering(CompilationContext context) + { + _context = context; + var threads = (ulong)context._localSizeX * context._localSizeY * context._localSizeZ; + _waveCount = (uint)Math.Max((threads + GuestWaveLaneCount - 1) / GuestWaveLaneCount, 1); + _multiWave = _waveCount > 1; + _waveStride = _multiWave ? ExchangeDwordCount + 1 : ExchangeDwordCount; + } + + private uint ExchangeTotalDwords => _waveCount * _waveStride; + + public bool RequiresCrossSubgroupSynchronization => true; + + private bool UsesExchange => _context.UsesSubgroupOperations(); + + public uint ReserveLdsDwords(uint guestDwordCount) => + UsesExchange && guestDwordCount < LdsDwordCount + ? guestDwordCount + ExchangeTotalDwords + : guestDwordCount; + + public void DeclareResources() + { + var context = _context; + if (!UsesExchange) + { + return; + } + + if (context._lds != 0) + { + // Metal exposes 32 KiB of threadgroup memory on the Apple + // GPUs we target. Reuse the tail of guest LDS so the + // cross-subgroup bridge stays within that limit. + _exchange = context._lds; + _exchangeElementPointer = context._ldsElementPointer; + _exchangeOffset = context._ldsDwordMask + 1 < LdsDwordCount + ? context._ldsDwordMask + 1 + : LdsDwordCount - ExchangeTotalDwords; + return; + } + + var exchangeArrayType = context._module.TypeArray(context._uintType, ExchangeTotalDwords); + _exchangeElementPointer = context._module.TypePointer(SpirvStorageClass.Workgroup, context._uintType); + _exchange = context._module.AddGlobalVariable( + context._module.TypePointer(SpirvStorageClass.Workgroup, exchangeArrayType), + SpirvStorageClass.Workgroup); + _exchangeOffset = 0; + context._module.AddName(_exchange, "wave64Exchange"); + context._interfaces.Add(_exchange); + } + + public void DeclareFunctionResources() + { + if (!UsesExchange) + { + return; + } + + _exchangeParity = _context.DeclareFunctionUint( + _context._module.Constant(_context._uintType, 0)); + _context._module.AddName(_exchangeParity, "wave64ExchangeParity"); + if (_multiWave) + { + _rendezvousEpoch = _context.DeclareFunctionUint( + _context._module.Constant(_context._uintType, 0)); + _context._module.AddName(_rendezvousEpoch, "wave64RendezvousEpoch"); + } + } + + public void InitializeMasks() + { + var context = _context; + if (context._subgroupInvocationIdInput != 0) + { + context.StoreS64(106, BothHalvesOfOwnBallot(context._module.ConstantBool(false))); + context.StoreS64(126, BothHalvesOfOwnBallot(context._module.ConstantBool(true))); + InitializeRendezvous(); + return; + } + + context.StoreS64(106, context._module.Constant64(context._ulongType, 0)); + context.StoreS64(126, context._module.Constant64(context._ulongType, 1)); + } + + public void BeginBlock() + { + _synchronizeSharedMemory = HalfMaskPlan() is null; + _sharedMemoryPhase = SharedMemoryPhase.None; + } + + public void BeforeInstruction(Gen5ShaderInstruction instruction) + { + var context = _context; + _emittingPc = instruction.Pc; + if (HalfMaskPlan() is { } plan) + { + if (plan.ExactPairsBefore.TryGetValue(instruction.Pc, out var exactPairs)) + { + EmitExactWaveMasks(exactPairs); + } + + if (plan.SharedMemoryBarriersBefore.Contains(instruction.Pc)) + { + Barrier(); + } + } + + if (!_synchronizeSharedMemory) + { + return; + } + + var nextPhase = instruction.Control is Gen5DataShareControl { Gds: false } + ? instruction.Opcode.StartsWith("DsRead", StringComparison.Ordinal) ? SharedMemoryPhase.Read + : instruction.Opcode.StartsWith("DsWrite", StringComparison.Ordinal) ? SharedMemoryPhase.Write + : SharedMemoryPhase.None + : SharedMemoryPhase.None; + if (instruction.Opcode == "SBarrier") + { + _sharedMemoryPhase = SharedMemoryPhase.None; + } + if (nextPhase != SharedMemoryPhase.None) + { + if (_sharedMemoryPhase != SharedMemoryPhase.None && _sharedMemoryPhase != nextPhase) + { + Barrier(); + } + _sharedMemoryPhase = nextPhase; + } + } + + public void EndBlock() + { + if (_synchronizeSharedMemory && _sharedMemoryPhase != SharedMemoryPhase.None) + { + Barrier(); + } + } + + public uint SubgroupAny(uint condition) => + _context._subgroupInvocationIdInput == 0 + ? condition + : _context.IsNotZero64(BooleanToWaveMask(condition)); + + public uint CurrentLaneBit() + { + var context = _context; + if (context._subgroupInvocationIdInput == 0) + { + return context._module.Constant64(context._ulongType, 1); + } + + return context.ShiftLeftLogical64( + context._module.Constant64(context._ulongType, 1), + context._module.AddInstruction( + SpirvOp.UConvert, + context._ulongType, + context.GuestWaveLane())); + } + + public uint BooleanToWaveMask(uint condition) + { + var context = _context; + if (context._subgroupInvocationIdInput == 0) + { + return context.BooleanToLaneMask(condition); + } + + var lane = context.GuestWaveLane(); + var owned = OwnHalfBallot(condition); + var exchange = BeginExchange(); + context.EmitConditional(IsHalfWaveLeader(lane), () => + context.Store(ExchangePointer(exchange, context.ShiftRightLogical(lane, context.UInt(5))), owned)); + Barrier(); + return context.Pair64( + context.Load(context._uintType, ExchangePointer(exchange, context.UInt(0))), + context.Load(context._uintType, ExchangePointer(exchange, context.UInt(1)))); + } + + public uint BroadcastFirstActive(uint value) + { + var context = _context; + if (context._subgroupInvocationIdInput == 0) + { + return value; + } + + var lane = context.GuestWaveLane(); + var upperHalf = context.ShiftRightLogical(lane, context.UInt(5)); + var activeInHalf = OwnHalfBallot(context.Load(context._boolType, context._exec)); + var halfHasActive = context.IsNotZero(activeInHalf); + var firstInHalf = context._module.AddInstruction( + SpirvOp.Select, + context._uintType, + halfHasActive, + context.Ext(73, context._uintType, activeInHalf), + context.UInt(0)); + var halfBase = context.BitwiseAnd( + context.Load(context._uintType, context._subgroupInvocationIdInput), + context.UInt(~31u)); + var halfValue = context._module.AddInstruction( + SpirvOp.GroupNonUniformBroadcast, + context._uintType, + context.UInt(3), + value, + context.IAdd(halfBase, firstInHalf)); + var exchange = BeginExchange(); + context.EmitConditional(IsHalfWaveLeader(lane), () => + { + context.Store(ExchangePointer(exchange, upperHalf), halfValue); + context.Store( + ExchangePointer(exchange, context.IAdd(context.UInt(2), upperHalf)), + context._module.AddInstruction( + SpirvOp.Select, + context._uintType, + halfHasActive, + context.UInt(1), + context.UInt(0))); + }); + Barrier(); + var lowerValue = context.Load(context._uintType, ExchangePointer(exchange, context.UInt(0))); + var upperValue = context.Load(context._uintType, ExchangePointer(exchange, context.UInt(1))); + var lowerActive = context.IsNotZero(context.Load(context._uintType, ExchangePointer(exchange, context.UInt(2)))); + var upperActive = context.IsNotZero(context.Load(context._uintType, ExchangePointer(exchange, context.UInt(3)))); + return context._module.AddInstruction( + SpirvOp.Select, + context._uintType, + context._module.AddInstruction( + SpirvOp.LogicalAnd, + context._boolType, + context.LogicalNot(lowerActive), + upperActive), + upperValue, + lowerValue); + } + + public uint BroadcastLane(uint value, uint lane) => + ExchangeValue( + _context._module.AddInstruction( + SpirvOp.IEqual, + _context._boolType, + _context.GuestWaveLane(), + lane), + value); + + public uint BroadcastValue(uint writer, uint value, uint lane) => ExchangeValue(writer, value); + + public uint WaveMaskAny(uint register, uint laneActive) => + _context._subgroupInvocationIdInput != 0 + ? _context.IsNotZero64(_context.LoadS64(register)) + : SubgroupAny(_context.Load(_context._boolType, laneActive)); + + public uint CurrentExecMask() => + _context._subgroupInvocationIdInput != 0 + ? _context.LoadS64(126) + : BooleanToWaveMask(_context.Load(_context._boolType, _context._exec)); + + public uint NormalizeExecMask(uint mask) => mask; + + public void StoreWaveMask(uint register, uint condition) + { + var context = _context; + if (context._waveLaneCount == 32) + { + context.StoreS(register, context.Narrow(BooleanToWaveMask(condition))); + return; + } + + if (context._subgroupInvocationIdInput != 0 && + _emittingPc is { } pc && + HalfMaskPlan() is { } plan && + plan.HalfMaskWrites.TryGetValue(pc, out var halfRegister) && + halfRegister == register) + { + context.StoreS64(register, BooleanToHalfWaveMask(condition)); + return; + } + + context.StoreS64(register, BooleanToWaveMask(condition)); + } + + private uint OwnHalfBallot(uint condition) + { + var context = _context; + var ballot = context._module.AddInstruction( + SpirvOp.GroupNonUniformBallot, + context._uvec4Type, + context.UInt(3), + condition); + return context._module.AddInstruction( + SpirvOp.VectorExtractDynamic, + context._uintType, + ballot, + context.ShiftRightLogical( + context.Load(context._uintType, context._subgroupInvocationIdInput), + context.UInt(5))); + } + + private uint BooleanToHalfWaveMask(uint condition) + { + var context = _context; + var owned = context._module.AddInstruction(SpirvOp.UConvert, context._ulongType, OwnHalfBallot(condition)); + return context._module.AddInstruction( + SpirvOp.Select, + context._ulongType, + context._module.AddInstruction( + SpirvOp.UGreaterThanEqual, + context._boolType, + context.GuestWaveLane(), + context.UInt(32)), + context.ShiftLeftLogical64(owned, context._module.Constant64(context._ulongType, 32)), + owned); + } + + private uint BothHalvesOfOwnBallot(uint condition) + { + var context = _context; + var owned = context._module.AddInstruction(SpirvOp.UConvert, context._ulongType, OwnHalfBallot(condition)); + return context.BitwiseOr64( + owned, + context.ShiftLeftLogical64(owned, context._module.Constant64(context._ulongType, 32))); + } + + private uint IsHalfWaveLeader(uint lane) => + _context._module.AddInstruction( + SpirvOp.IEqual, + _context._boolType, + _context.BitwiseAnd(lane, _context.UInt(31)), + _context.UInt(0)); + + private uint BeginExchange() + { + var context = _context; + var parity = context.Load(context._uintType, _exchangeParity); + context.Store(_exchangeParity, context.BitwiseXor(parity, context.UInt(1))); + return context.IAdd( + WaveExchangeBase(), + context._module.AddInstruction( + SpirvOp.IMul, + context._uintType, + parity, + context.UInt(ExchangeSlotCount))); + } + + private uint WaveExchangeBase() + { + var context = _context; + if (!_multiWave) + { + return context.UInt(_exchangeOffset); + } + + var wave = context.ShiftRightLogical( + context.Load(context._uintType, context._localInvocationIndexInput), + context.UInt(6)); + return context.IAdd( + context.UInt(_exchangeOffset), + context._module.AddInstruction(SpirvOp.IMul, context._uintType, wave, context.UInt(_waveStride))); + } + + // Zeroes each wave's arrival counter. Runs once at entry, where the whole workgroup is + // converged, so the workgroup barrier is safe here. + private void InitializeRendezvous() + { + if (!_multiWave || !UsesExchange) + { + return; + } + + var context = _context; + context.EmitConditional( + context._module.AddInstruction( + SpirvOp.IEqual, + context._boolType, + context.GuestWaveLane(), + context.UInt(0)), + () => context.Store( + ExchangePointer(WaveExchangeBase(), context.UInt(ExchangeDwordCount)), + context.UInt(0))); + WorkgroupBarrier(); + } + + // Both halves of the wave arrive, then wait until the counter shows two arrivals per + // rendezvous so far. The halves of one wave are co-resident in the workgroup, so the + // wait always ends. + private void WaveRendezvous() + { + var context = _context; + var counter = ExchangePointer(WaveExchangeBase(), context.UInt(ExchangeDwordCount)); + context.EmitConditional( + IsHalfWaveLeader(context.GuestWaveLane()), + () => context._module.AddInstruction( + SpirvOp.AtomicIAdd, + context._uintType, + counter, + context.UInt(2), + context.UInt(0x108), + context.UInt(1))); + var epoch = context.IAdd(context.Load(context._uintType, _rendezvousEpoch), context.UInt(1)); + context.Store(_rendezvousEpoch, epoch); + var target = context._module.AddInstruction(SpirvOp.IMul, context._uintType, epoch, context.UInt(2)); + + var header = context._module.AllocateId(); + var continueLabel = context._module.AllocateId(); + var mergeLabel = context._module.AllocateId(); + context._module.AddStatement(SpirvOp.Branch, header); + context._module.AddLabel(header); + var arrived = context._module.AddInstruction( + SpirvOp.AtomicLoad, + context._uintType, + counter, + context.UInt(2), + context.UInt(0x102)); + var done = context._module.AddInstruction( + SpirvOp.UGreaterThanEqual, + context._boolType, + arrived, + target); + context._module.AddStatement(SpirvOp.LoopMerge, mergeLabel, continueLabel, 0); + context._module.AddStatement(SpirvOp.BranchConditional, done, mergeLabel, continueLabel); + context._module.AddLabel(continueLabel); + context._module.AddStatement(SpirvOp.Branch, header); + context._module.AddLabel(mergeLabel); + } + + private uint ExchangePointer(uint exchange, uint slot) => + _context._module.AddInstruction( + SpirvOp.AccessChain, + _exchangeElementPointer, + _exchange, + _context.IAdd(exchange, slot)); + + private uint ExchangeValue(uint writer, uint value) + { + var context = _context; + var exchange = BeginExchange(); + context.EmitConditional(writer, () => context.Store(ExchangePointer(exchange, context.UInt(0)), value)); + Barrier(); + return context.Load(context._uintType, ExchangePointer(exchange, context.UInt(0))); + } + + private void EmitExactWaveMasks(IReadOnlyList pairs) + { + var context = _context; + var lane = context.GuestWaveLane(); + var upperHalf = context.ShiftRightLogical(lane, context.UInt(5)); + var lowerHalf = context._module.AddInstruction(SpirvOp.IEqual, context._boolType, upperHalf, context.UInt(0)); + const int pairsPerExchange = (int)ExchangeSlotCount / 2; + for (var start = 0; start < pairs.Count; start += pairsPerExchange) + { + var count = Math.Min(pairsPerExchange, pairs.Count - start); + var owned = new uint[count]; + for (var index = 0; index < count; index++) + { + var pair = pairs[start + index]; + owned[index] = context._module.AddInstruction( + SpirvOp.Select, + context._uintType, + lowerHalf, + context.LoadS(pair), + context.LoadS(pair + 1)); + } + + var exchange = BeginExchange(); + context.EmitConditional(IsHalfWaveLeader(lane), () => + { + for (var index = 0; index < count; index++) + { + context.Store( + ExchangePointer(exchange, context.IAdd(context.UInt((uint)index * 2), upperHalf)), + owned[index]); + } + }); + Barrier(); + for (var index = 0; index < count; index++) + { + var pair = pairs[start + index]; + context.StoreS(pair, context.Load(context._uintType, ExchangePointer(exchange, context.UInt((uint)index * 2)))); + context.StoreS(pair + 1, context.Load(context._uintType, ExchangePointer(exchange, context.UInt((uint)index * 2 + 1)))); + } + } + } + + private Ir.Gen5Wave64HalfMaskPlan? HalfMaskPlan() + { + if (_halfMaskPlanBuilt) + { + return _halfMaskPlan; + } + + _halfMaskPlanBuilt = true; + var context = _context; + if (HalfWaveMasks && UsesExchange && context._subgroupInvocationIdInput != 0) + { + var sharedFlatPcs = context._request.Memory.Entries + .Where(static memory => memory.AddressSpace is FlatAddressSpace.Shared or FlatAddressSpace.SharedOrPrivate) + .Select(static memory => memory.Pc) + .ToHashSet(); + _halfMaskPlan = Ir.Gen5Wave64HalfMaskAnalysis.Analyze(context._request.Program, sharedFlatPcs); + } + + return _halfMaskPlan; + } + + private void Barrier() + { + if (_multiWave) + { + // The rendezvous needs the exchange area, declared only with subgroup operations. + if (UsesExchange) + { + WaveRendezvous(); + } + + return; + } + + WorkgroupBarrier(); + } + + private void WorkgroupBarrier() + { + var workgroup = _context.UInt(2); + _context._module.AddStatement( + SpirvOp.ControlBarrier, + workgroup, + workgroup, + _context.UInt(0x108)); + } + } + } +} + + + + + diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.NativeWave32Lowering.cs b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.NativeWave32Lowering.cs new file mode 100644 index 0000000..3491e4f --- /dev/null +++ b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.NativeWave32Lowering.cs @@ -0,0 +1,156 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.ShaderCompiler.Vulkan; + +public static partial class Gen5SpirvTranslator +{ + private sealed partial class CompilationContext + { + // A guest Wave32 maps directly to one 32-lane host subgroup. This is the + // former implicit/default lowering, made explicit so every guest wave size + // has a concrete lowering strategy. + private sealed class NativeWave32Lowering : IWaveLowering + { + private readonly CompilationContext _context; + + public NativeWave32Lowering(CompilationContext context) + { + _context = context; + } + + public bool RequiresCrossSubgroupSynchronization => false; + + public uint ReserveLdsDwords(uint guestDwordCount) => guestDwordCount; + + public void DeclareResources() { } + public void DeclareFunctionResources() { } + public void BeginBlock() { } + public void BeforeInstruction(Gen5ShaderInstruction instruction) { } + public void EndBlock() { } + + public void InitializeMasks() + { + var context = _context; + if (context._subgroupInvocationIdInput != 0) + { + StoreWaveMask(106, context._module.ConstantBool(false)); + StoreWaveMask(126, context._module.ConstantBool(true)); + return; + } + + context.StoreS64(106, context._module.Constant64(context._ulongType, 0)); + context.StoreS64(126, context._module.Constant64(context._ulongType, 1)); + } + + public uint BooleanToWaveMask(uint condition) + { + var context = _context; + if (context._subgroupInvocationIdInput == 0) + { + return context._module.AddInstruction( + SpirvOp.Select, + context._ulongType, + condition, + CurrentLaneBit(), + context._module.Constant64(context._ulongType, 0)); + } + + var ballot = context._module.AddInstruction( + SpirvOp.GroupNonUniformBallot, + context._uvec4Type, + context.UInt(3), + condition); + var low = context._module.AddInstruction( + SpirvOp.CompositeExtract, + context._uintType, + ballot, + 0); + return context._module.AddInstruction(SpirvOp.UConvert, context._ulongType, low); + } + + public uint SubgroupAny(uint condition) => + _context._subgroupInvocationIdInput == 0 + ? condition + : _context._module.AddInstruction( + SpirvOp.GroupNonUniformAny, + _context._boolType, + _context.UInt(3), + condition); + + public uint CurrentLaneBit() + { + var context = _context; + if (context._subgroupInvocationIdInput == 0) + { + return context._module.Constant64(context._ulongType, 1); + } + + var shifted = context.ShiftLeftLogical64( + context._module.Constant64(context._ulongType, 1), + context._module.AddInstruction( + SpirvOp.UConvert, + context._ulongType, + context.GuestWaveLane())); + return context._module.AddInstruction( + SpirvOp.Select, + context._ulongType, + context.IsCurrentLaneInRdnaWave(), + shifted, + context._module.Constant64(context._ulongType, 0)); + } + + public uint BroadcastFirstActive(uint value) + { + var context = _context; + if (context._subgroupInvocationIdInput == 0) + { + return value; + } + + var activeLanes = context._module.AddInstruction( + SpirvOp.GroupNonUniformBallot, + context._uvec4Type, + context.UInt(3), + context.Load(context._boolType, context._exec)); + var activeLow = context._module.AddInstruction( + SpirvOp.CompositeExtract, + context._uintType, + activeLanes, + 0); + var firstActiveLane = context.Ext(73, context._uintType, activeLow); + return context._module.AddInstruction( + SpirvOp.GroupNonUniformBroadcast, + context._uintType, + context.UInt(3), + value, + firstActiveLane); + } + + public uint BroadcastLane(uint value, uint lane) => + _context._subgroupInvocationIdInput == 0 + ? value + : _context._module.AddInstruction( + SpirvOp.GroupNonUniformBroadcast, + _context._uintType, + _context.UInt(3), + value, + lane); + + public uint BroadcastValue(uint writer, uint value, uint lane) => + _context.ShuffleLane(value, lane); + + public uint WaveMaskAny(uint register, uint laneActive) => + SubgroupAny(_context.Load(_context._boolType, laneActive)); + + public uint CurrentExecMask() => + BooleanToWaveMask(_context.Load(_context._boolType, _context._exec)); + + public uint NormalizeExecMask(uint mask) => + _context.BitwiseAnd64(mask, _context._module.Constant64(_context._ulongType, 0xFFFF_FFFFUL)); + + public void StoreWaveMask(uint register, uint condition) => + _context.StoreS(register, _context.Narrow(BooleanToWaveMask(condition))); + } + } +} diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.NativeWave64Lowering.cs b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.NativeWave64Lowering.cs new file mode 100644 index 0000000..d4daa6e --- /dev/null +++ b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.NativeWave64Lowering.cs @@ -0,0 +1,160 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.ShaderCompiler.Vulkan; + +public static partial class Gen5SpirvTranslator +{ + private sealed partial class CompilationContext + { + // One guest wave is one 64-lane host subgroup (ballot.x = lanes 0..31, ballot.y = lanes + // 32..63). It has no workgroup exchange, LDS reservation, half-mask plan or + // cross-subgroup barrier state. + private sealed class NativeWave64Lowering : IWaveLowering + { + private readonly CompilationContext _context; + + public NativeWave64Lowering(CompilationContext context) + { + _context = context; + } + + public bool RequiresCrossSubgroupSynchronization => false; + + public uint ReserveLdsDwords(uint guestDwordCount) => guestDwordCount; + + public void DeclareResources() { } + public void DeclareFunctionResources() { } + public void BeginBlock() { } + public void BeforeInstruction(Gen5ShaderInstruction instruction) { } + public void EndBlock() { } + + public void InitializeMasks() + { + var context = _context; + if (context._subgroupInvocationIdInput != 0) + { + StoreWaveMask(106, context._module.ConstantBool(false)); + StoreWaveMask(126, context._module.ConstantBool(true)); + } + else + { + context.StoreS64(106, context._module.Constant64(context._ulongType, 0)); + context.StoreS64(126, context._module.Constant64(context._ulongType, 1)); + } + } + + public uint BooleanToWaveMask(uint condition) + { + var context = _context; + if (context._subgroupInvocationIdInput == 0) + { + return context._module.AddInstruction( + SpirvOp.Select, + context._ulongType, + condition, + CurrentLaneBit(), + context._module.Constant64(context._ulongType, 0)); + } + + return BallotToMask(condition); + } + + // The host subgroup is at least 64 wide: lanes 0..31 are ballot.x and lanes 32..63 ballot.y. + private uint BallotToMask(uint condition) + { + var context = _context; + var ballot = context._module.AddInstruction( + SpirvOp.GroupNonUniformBallot, + context._uvec4Type, + context.UInt(3), + condition); + var low = context._module.AddInstruction(SpirvOp.CompositeExtract, context._uintType, ballot, 0); + var high = context._module.AddInstruction(SpirvOp.CompositeExtract, context._uintType, ballot, 1); + return context.BitwiseOr64( + context._module.AddInstruction(SpirvOp.UConvert, context._ulongType, low), + context.ShiftLeftLogical64( + context._module.AddInstruction(SpirvOp.UConvert, context._ulongType, high), + context._module.Constant64(context._ulongType, 32))); + } + + public uint SubgroupAny(uint condition) => + _context._subgroupInvocationIdInput == 0 + ? condition + : _context._module.AddInstruction( + SpirvOp.GroupNonUniformAny, + _context._boolType, + _context.UInt(3), + condition); + + public uint CurrentLaneBit() + { + var context = _context; + if (context._subgroupInvocationIdInput == 0) + { + return context._module.Constant64(context._ulongType, 1); + } + + var shifted = context.ShiftLeftLogical64( + context._module.Constant64(context._ulongType, 1), + context._module.AddInstruction( + SpirvOp.UConvert, + context._ulongType, + context.GuestWaveLane())); + return shifted; + } + + public uint BroadcastFirstActive(uint value) + { + var context = _context; + if (context._subgroupInvocationIdInput == 0) + { + return value; + } + + var activeLanes = BallotToMask(context.Load(context._boolType, context._exec)); + var activeLow = context._module.AddInstruction(SpirvOp.UConvert, context._uintType, activeLanes); + var activeHigh = context._module.AddInstruction( + SpirvOp.UConvert, + context._uintType, + context.ShiftRightLogical64(activeLanes, context._module.Constant64(context._ulongType, 32))); + var firstActiveLane = context._module.AddInstruction( + SpirvOp.Select, + context._uintType, + context._module.AddInstruction(SpirvOp.IEqual, context._boolType, activeLow, context.UInt(0)), + context.IAdd(context.Ext(73, context._uintType, activeHigh), context.UInt(32)), + context.Ext(73, context._uintType, activeLow)); + return context._module.AddInstruction( + SpirvOp.GroupNonUniformBroadcast, + context._uintType, + context.UInt(3), + value, + firstActiveLane); + } + + public uint BroadcastLane(uint value, uint lane) => + _context._subgroupInvocationIdInput == 0 + ? value + : _context._module.AddInstruction( + SpirvOp.GroupNonUniformBroadcast, + _context._uintType, + _context.UInt(3), + value, + lane); + + public uint BroadcastValue(uint writer, uint value, uint lane) => + _context.ShuffleLane(value, lane); + + public uint WaveMaskAny(uint register, uint laneActive) => + SubgroupAny(_context.Load(_context._boolType, laneActive)); + + public uint CurrentExecMask() => + BooleanToWaveMask(_context.Load(_context._boolType, _context._exec)); + + public uint NormalizeExecMask(uint mask) => mask; + + public void StoreWaveMask(uint register, uint condition) => + _context.StoreS64(register, BooleanToWaveMask(condition)); + } + } +} diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Resources.cs b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Resources.cs index 9d682a8..47c3647 100644 --- a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Resources.cs +++ b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Resources.cs @@ -116,7 +116,23 @@ public static partial class Gen5SpirvTranslator _localSizeY = Math.Max(request.LocalSizeY, 1); _localSizeZ = Math.Max(request.LocalSizeZ, 1); _physicalAxisOfLogical = ComputeWorkgroupAxisOrder(_localSizeX, _localSizeY, _localSizeZ); - _emulateWave64 = _stage == Gen5SpirvStage.Compute && _waveLaneCount == 64 && (ulong)_localSizeX * _localSizeY * _localSizeZ == 64; + // A wave64 runs as one 64-lane host subgroup when the host pins it to one + // (HostWave64Supported). Hosts without 64-lane subgroups bridge the two 32-lane + // halves through workgroup memory. + if (_waveLaneCount == 32) + { + _waveLowering = new NativeWave32Lowering(this); + } + else if (request.HostWave64Supported) + { + _waveLowering = new NativeWave64Lowering(this); + _guestLaneFromSubgroup = true; + } + else + { + _waveLowering = new EmulatedWave64Lowering(this); + } + _requiredVertexOutputCount = request.RequiredVertexOutputCount; _pixelInputEnable = request.PixelInputEnable; _pixelInputAddress = request.PixelInputAddress; @@ -1909,21 +1925,7 @@ public static partial class Gen5SpirvTranslator }); var firstValue = LoadV(destination); - uint broadcast; - if (_emulateWave64) - { - broadcast = ExchangeWave64Value(isFirstActive, firstValue); - } - else if (_stage == Gen5SpirvStage.Compute || _enableGraphicsSubgroupOperations) - { - // Every active lane receives the first lane's old counter, in graphics stages too. - broadcast = _module.AddInstruction(SpirvOp.GroupNonUniformShuffle, _uintType, UInt(3), firstValue, firstLane); - } - else - { - // Without subgroup operations a graphics invocation is its own wave of one lane. - broadcast = firstValue; - } + var broadcast = BroadcastWaveValue(isFirstActive, firstValue, firstLane); var validResult = LogicalAnd(inBounds, IsNotZero64(activeMask)); StoreV(destination, _module.AddInstruction(SpirvOp.Select, _uintType, validResult, broadcast, UInt(0))); diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.WaveLowering.cs b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.WaveLowering.cs new file mode 100644 index 0000000..a0d7296 --- /dev/null +++ b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.WaveLowering.cs @@ -0,0 +1,34 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.ShaderCompiler.Vulkan; + +public static partial class Gen5SpirvTranslator +{ + private sealed partial class CompilationContext + { + private interface IWaveLowering + { + bool RequiresCrossSubgroupSynchronization { get; } + + uint ReserveLdsDwords(uint guestDwordCount); + void DeclareResources(); + void DeclareFunctionResources(); + void InitializeMasks(); + void BeginBlock(); + void BeforeInstruction(Gen5ShaderInstruction instruction); + void EndBlock(); + uint BooleanToWaveMask(uint condition); + uint SubgroupAny(uint condition); + uint CurrentLaneBit(); + uint BroadcastFirstActive(uint value); + uint BroadcastLane(uint value, uint lane); + uint BroadcastValue(uint writer, uint value, uint lane); + uint WaveMaskAny(uint register, uint laneActive); + uint CurrentExecMask(); + uint NormalizeExecMask(uint mask); + void StoreWaveMask(uint register, uint condition); + } + } +} + diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.cs b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.cs index 73a8a16..bc27667 100644 --- a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.cs +++ b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.cs @@ -17,8 +17,6 @@ public static partial class Gen5SpirvTranslator // so a smaller array is safe. private const uint PrivateLdsDwordCount = 2048; private const uint RdnaWaveLaneCount = 32; - private const uint Wave64ExchangeSlotCount = 4; - private const uint Wave64ExchangeDwordCount = Wave64ExchangeSlotCount * 2; internal static SpirvImageFormat DecodeStorageImageFormat( uint dataFormat, @@ -71,7 +69,8 @@ public static partial class Gen5SpirvTranslator private readonly bool _usesPixelValidMask; private readonly bool _enableGraphicsSubgroupOperations; private readonly uint _waveLaneCount; - private readonly bool _emulateWave64; + private readonly IWaveLowering _waveLowering; + private bool _guestLaneFromSubgroup; // Safety valve for the PC-dispatcher loop. Each iteration executes one // GCN basic block; a correctly-translated shader always reaches its @@ -195,12 +194,6 @@ public static partial class Gen5SpirvTranslator private uint _workGroupIdInput; private uint _pushConstantUintPointer; private uint _subgroupInvocationIdInput; - private uint _wave64Exchange; - private uint _wave64ExchangeElementPointer; - private uint _wave64ExchangeOffset; - private uint _wave64ExchangeParity; - private Ir.Gen5Wave64HalfMaskPlan? _halfMaskPlan; - private uint? _emittingPc; private uint _glsl; private enum ImageComponentKind @@ -421,8 +414,8 @@ public static partial class Gen5SpirvTranslator var active = Load(_boolType, _programActive); if (_maxDispatcherSteps > 0) { - var steps = IAdd(Load(_uintType, _iterationGuard), UInt(1)); - Store(_iterationGuard, steps); + var steps = IAdd(Load(_uintType, IterationGuard()), UInt(1)); + Store(IterationGuard(), steps); var withinLimit = _module.AddInstruction( SpirvOp.ULessThan, _boolType, @@ -625,7 +618,13 @@ public static partial class Gen5SpirvTranslator _scc = Flag(false); _vcc = Flag(false); _exec = Flag(true); - _reachedPixelExport = Flag(false); + // Only the title-shader trace reads it; elsewhere it would be a write-only Private + // global, and AMD's driver crashes linking stages that carry an unused one. + if (TraceTitleShaderState) + { + _reachedPixelExport = Flag(false); + } + if (_usesPixelValidMask) { _pixelValidMaskActive = Flag(true); @@ -635,22 +634,7 @@ public static partial class Gen5SpirvTranslator SpirvStorageClass.Private, _module.Constant(_uintType, 0)); _programActive = Flag(true); - if (UsesWave64Exchange()) - { - _wave64ExchangeParity = _module.AddGlobalVariable(_privateUintPointer, SpirvStorageClass.Private, UInt(0)); - _interfaces.Add(_wave64ExchangeParity); - _module.AddName(_wave64ExchangeParity, "wave64ExchangeParity"); - } - if (_maxDispatcherSteps > 0) - { - _iterationGuard = _module.AddGlobalVariable( - _privateUintPointer, - SpirvStorageClass.Private, - _module.Constant(_uintType, 0)); - _interfaces.Add(_iterationGuard); - _module.AddName(_iterationGuard, "pcGuard"); - } - + _waveLowering.DeclareFunctionResources(); foreach (var slot in FindLaneSpillSlots()) { var variable = _module.AddGlobalVariable(_privateUintPointer, SpirvStorageClass.Private, _module.Constant(_uintType, 0)); @@ -668,7 +652,11 @@ public static partial class Gen5SpirvTranslator _interfaces.Add(_scc); _interfaces.Add(_vcc); _interfaces.Add(_exec); - _interfaces.Add(_reachedPixelExport); + if (_reachedPixelExport != 0) + { + _interfaces.Add(_reachedPixelExport); + } + if (_pixelValidMaskActive != 0) { _interfaces.Add(_pixelValidMaskActive); @@ -681,7 +669,7 @@ public static partial class Gen5SpirvTranslator DeclareLayoutBindings(); DeclareScratch(); DeclareLds(); - DeclareWave64Scratch(); + _waveLowering.DeclareResources(); DeclareStageInterface(); return; } @@ -709,43 +697,6 @@ public static partial class Gen5SpirvTranslator _module.AddName(_scratch, "scratch"); } - private void DeclareWave64Scratch() - { - if (!UsesWave64Exchange()) - { - return; - } - - // Metal exposes 32 KiB of threadgroup memory on the Apple GPUs we - // target. Some PS5 compute shaders legitimately request all of it, - // so allocating another workgroup variable for the wave64 bridge - // makes pipeline creation fail. Reuse the final dwords of the - // existing LDS allocation in that case. The translator already - // bounds guest LDS accesses to this fixed allocation; keeping the - // bridge inside it preserves the host limit and still provides the - // cross-subgroup rendezvous needed to model one 64-lane guest wave. - if (_lds != 0) - { - _wave64Exchange = _lds; - _wave64ExchangeElementPointer = _ldsElementPointer; - _wave64ExchangeOffset = _ldsDwordMask + 1 < LdsDwordCount - ? _ldsDwordMask + 1 - : LdsDwordCount - Wave64ExchangeDwordCount; - return; - } - - var exchangeArrayType = _module.TypeArray(_uintType, Wave64ExchangeDwordCount); - _wave64ExchangeElementPointer = _module.TypePointer(SpirvStorageClass.Workgroup, _uintType); - _wave64Exchange = _module.AddGlobalVariable( - _module.TypePointer(SpirvStorageClass.Workgroup, exchangeArrayType), - SpirvStorageClass.Workgroup); - _wave64ExchangeOffset = 0; - _module.AddName(_wave64Exchange, "wave64Exchange"); - _interfaces.Add(_wave64Exchange); - } - - private bool UsesWave64Exchange() => _emulateWave64 && UsesSubgroupOperations(); - private uint ComputeLdsGuestDwordCount() { var declared = _request.LocalDataShareDwords; @@ -776,9 +727,7 @@ public static partial class Gen5SpirvTranslator ? ComputeLdsGuestDwordCount() : PrivateLdsDwordCount; _ldsDwordMask = dwordCount - 1; - var arrayDwordCount = UsesWave64Exchange() && dwordCount < LdsDwordCount - ? dwordCount + Wave64ExchangeDwordCount - : dwordCount; + var arrayDwordCount = _waveLowering.ReserveLdsDwords(dwordCount); var ldsArrayType = _module.TypeArray(_uintType, arrayDwordCount); var ldsPointer = _module.TypePointer(storageClass, ldsArrayType); @@ -1176,32 +1125,16 @@ public static partial class Gen5SpirvTranslator } Store(_scc, _module.ConstantBool(false)); - Store(_reachedPixelExport, _module.ConstantBool(false)); + if (_reachedPixelExport != 0) + { + Store(_reachedPixelExport, _module.ConstantBool(false)); + } + if (_pixelValidMaskActive != 0) { Store(_pixelValidMaskActive, _module.ConstantBool(true)); } - if (_subgroupInvocationIdInput != 0 && _emulateWave64) - { - StoreS64(106, BothHalvesOfOwnBallot(_module.ConstantBool(false))); - StoreS64(126, BothHalvesOfOwnBallot(_module.ConstantBool(true))); - } - else if (_subgroupInvocationIdInput != 0) - { - StoreWaveMask(106, _module.ConstantBool(false)); - StoreWaveMask(126, _module.ConstantBool(true)); - } - else - { - // Graphics stages emulate one logical wave lane. Keep the - // guest-visible VCC/EXEC scalar pairs synchronized with the - // internal booleans: shaders commonly save EXEC from s126:s127 - // and restore it after divergent work. Initializing only _exec - // left those registers at zero, so the first restore disabled - // every fragment before its color export. - StoreS64(106, _module.Constant64(_ulongType, 0)); - StoreS64(126, _module.Constant64(_ulongType, 1)); - } + _waveLowering.InitializeMasks(); // Primitive (NGG) vertex programs rebuild EXEC from the merged wave info in s3: // vertex count in bits 0-7, primitive count in bits 8-15. Left at zero, EXEC @@ -1440,8 +1373,6 @@ public static partial class Gen5SpirvTranslator StoreS(register.Value, value); } - private enum SharedMemoryPhase { None, Read, Write } - private bool TryEmitBlock( IReadOnlyList blocks, int blockIndex, @@ -1463,51 +1394,18 @@ public static partial class Gen5SpirvTranslator { error = string.Empty; var block = blocks[blockIndex]; - var halfMaskPlan = HalfMaskPlan(); - // One guest wave can span two host subgroups. Keep its shared-memory phases ordered. - // The half-mask plan inserts its own barriers; other wave64 programs also - // need LDS phase ordering when reads and writes span basic blocks. - var synchronizeSharedMemory = _emulateWave64 && halfMaskPlan is null; - var sharedMemoryPhase = SharedMemoryPhase.None; + _waveLowering.BeginBlock(); for (var index = block.StartIndex; index < block.EndIndex; index++) { var instruction = _request.Program.Instructions[index]; - if (halfMaskPlan is not null) - { - if (halfMaskPlan.ExactPairsBefore.TryGetValue(instruction.Pc, out var exactPairs)) - { - EmitExactWaveMasks(exactPairs); - } - - if (halfMaskPlan.SharedMemoryBarriersBefore.Contains(instruction.Pc)) - { - EmitWave64Barrier(); - } - } - + _waveLowering.BeforeInstruction(instruction); if (IsBranch(instruction.Opcode) || instruction.Opcode == "SEndpgm") { continue; } - if (synchronizeSharedMemory) - { - var nextPhase = instruction.Control is Gen5DataShareControl { Gds: false } - ? instruction.Opcode.StartsWith("DsRead", StringComparison.Ordinal) ? SharedMemoryPhase.Read - : instruction.Opcode.StartsWith("DsWrite", StringComparison.Ordinal) ? SharedMemoryPhase.Write : SharedMemoryPhase.None - : SharedMemoryPhase.None; - if (instruction.Opcode == "SBarrier") sharedMemoryPhase = SharedMemoryPhase.None; - if (nextPhase != SharedMemoryPhase.None) - { - if (sharedMemoryPhase != SharedMemoryPhase.None && sharedMemoryPhase != nextPhase) EmitWave64Barrier(); - sharedMemoryPhase = nextPhase; - } - } - _execKnownFull = IsExecKnownFull(instruction.Pc); - _emittingPc = instruction.Pc; var emitted = TryEmitInstruction(instruction, out error); - _emittingPc = null; _execKnownFull = false; if (!emitted) { @@ -1521,7 +1419,7 @@ public static partial class Gen5SpirvTranslator CapturePixelExec(instruction); } - if (synchronizeSharedMemory && sharedMemoryPhase != SharedMemoryPhase.None) EmitWave64Barrier(); + _waveLowering.EndBlock(); return true; } @@ -1908,8 +1806,8 @@ public static partial class Gen5SpirvTranslator if (_maxDispatcherSteps > 0) { - var steps = IAdd(Load(_uintType, _iterationGuard), UInt(1)); - Store(_iterationGuard, steps); + var steps = IAdd(Load(_uintType, IterationGuard()), UInt(1)); + Store(IterationGuard(), steps); var withinLimit = _module.AddInstruction(SpirvOp.ULessThan, _boolType, steps, UInt((uint)_maxDispatcherSteps)); again = _module.AddInstruction(SpirvOp.LogicalAnd, _boolType, again, withinLimit); } @@ -2063,8 +1961,8 @@ public static partial class Gen5SpirvTranslator { // The dispatcher's safety valve, counted per loop iteration: a mistranslated // exit ends the invocation instead of wedging the GPU queue. - var steps = IAdd(Load(_uintType, _iterationGuard), UInt(1)); - Store(_iterationGuard, steps); + var steps = IAdd(Load(_uintType, IterationGuard()), UInt(1)); + Store(IterationGuard(), steps); var withinLimit = _module.AddInstruction(SpirvOp.ULessThan, _boolType, steps, UInt((uint)_maxDispatcherSteps)); again = _module.AddInstruction(SpirvOp.LogicalAnd, _boolType, again, withinLimit); } @@ -2079,7 +1977,7 @@ public static partial class Gen5SpirvTranslator // the block was skipped. Preserve that diagnostic safety budget. if (_maxDispatcherSteps > 0) { - Store(_iterationGuard, IAdd(Load(_uintType, _iterationGuard), UInt(1))); + Store(IterationGuard(), IAdd(Load(_uintType, IterationGuard()), UInt(1))); } _module.AddStatement(SpirvOp.Branch, entryMerge); _module.AddLabel(entryMerge); @@ -2657,15 +2555,7 @@ public static partial class Gen5SpirvTranslator }); var firstValue = LoadV(destination); - uint broadcast; - if (_emulateWave64) - { - broadcast = ExchangeWave64Value(isFirstActive, firstValue); - } - else - { - broadcast = ShuffleLane(firstValue, firstLane); - } + var broadcast = BroadcastWaveValue(isFirstActive, firstValue, firstLane); var validResult = _module.AddInstruction( SpirvOp.LogicalAnd, @@ -6347,7 +6237,10 @@ public static partial class Gen5SpirvTranslator return true; } - Store(_reachedPixelExport, _module.ConstantBool(true)); + if (_reachedPixelExport != 0) + { + Store(_reachedPixelExport, _module.ConstantBool(true)); + } var values = new uint[4]; for (var component = 0; component < 4; component++) @@ -7469,6 +7362,13 @@ public static partial class Gen5SpirvTranslator UInt(0), dwordAddress); + private uint DeclareFunctionUint(uint initializer) + { + var variable = _module.AddGlobalVariable(_privateUintPointer, SpirvStorageClass.Private, initializer); + _interfaces.Add(variable); + return variable; + } + // Without subgroup operations each invocation is a one-lane wave, so a V_WRITELANE to // lane N > 0 used to be dropped and V_READLANE returned lane 0. Compilers spill SGPRs // (EXEC included) to VGPR lanes like this; restoring the wrong EXEC made waterfall @@ -7476,7 +7376,7 @@ public static partial class Gen5SpirvTranslator private List<(uint Register, uint Lane)> FindLaneSpillSlots() { var slots = new List<(uint Register, uint Lane)>(); - if (_emulateWave64) + if (_stage == Gen5SpirvStage.Compute && _waveLowering is EmulatedWave64Lowering) { return slots; } @@ -7686,6 +7586,24 @@ public static partial class Gen5SpirvTranslator // Declared on first use when Private. AMD's compiler keeps an unused 4 KiB private // array as a named .bss global: two stages then fail to link, and the driver copies // the NOBITS section as file data and reads past the end of the ELF. + private uint IterationGuard() + { + if (_iterationGuard == 0) + { + _iterationGuard = _module.AddGlobalVariable( + _privateUintPointer, + SpirvStorageClass.Private, + _module.Constant(_uintType, 0)); + _interfaces.Add(_iterationGuard); + _module.AddName(_iterationGuard, "pcGuard"); + } + + return _iterationGuard; + } + + private static readonly bool TraceTitleShaderState = + Environment.GetEnvironmentVariable("SHARPEMU_TRACE_TITLE_SHADER_STATE") == "1"; + private uint PackedHalfRegisters() { if (_packedHalfRegisters != 0) @@ -7956,18 +7874,19 @@ public static partial class Gen5SpirvTranslator _module.AddInstruction(SpirvOp.LogicalNot, _boolType, value); private uint SubgroupAny(uint condition) => - _subgroupInvocationIdInput == 0 - ? condition - : _emulateWave64 - ? IsNotZero64(BooleanToWaveMask(condition)) - : _module.AddInstruction( - SpirvOp.GroupNonUniformAny, - _boolType, - UInt(3), - condition); + _waveLowering.SubgroupAny(condition); private uint GuestWaveLane() { + // A native wave64 is one host subgroup: its lane is the subgroup lane, the same one + // ballot and shuffle address, whatever order the driver gave the invocations. + if (_waveLaneCount == 64 && _guestLaneFromSubgroup && _subgroupInvocationIdInput != 0) + { + return BitwiseAnd( + Load(_uintType, _subgroupInvocationIdInput), + UInt(63)); + } + if (_waveLaneCount == 64 && _localInvocationIndexInput != 0) { return BitwiseAnd( @@ -8017,29 +7936,8 @@ public static partial class Gen5SpirvTranslator ? value : _module.AddInstruction(SpirvOp.GroupNonUniformShuffle, _uintType, UInt(3), value, lane); - private uint CurrentLaneBit() - { - if (_subgroupInvocationIdInput == 0) - { - return _module.Constant64(_ulongType, 1); - } - - var maskedLane = GuestWaveLane(); - var shifted = ShiftLeftLogical64( - _module.Constant64(_ulongType, 1), - _module.AddInstruction( - SpirvOp.UConvert, - _ulongType, - maskedLane)); - return _emulateWave64 - ? shifted - : _module.AddInstruction( - SpirvOp.Select, - _ulongType, - IsCurrentLaneInRdnaWave(), - shifted, - _module.Constant64(_ulongType, 0)); - } + private uint CurrentLaneBit() => + _waveLowering.CurrentLaneBit(); private uint IsCurrentLaneInRdnaWave() => _module.AddInstruction( @@ -8056,193 +7954,11 @@ public static partial class Gen5SpirvTranslator CurrentLaneBit(), _module.Constant64(_ulongType, 0)); - private uint BooleanToWaveMask(uint condition) - { - if (_subgroupInvocationIdInput == 0) - { - return BooleanToLaneMask(condition); - } - - if (_emulateWave64) - { - var lane = GuestWaveLane(); - var owned = OwnHalfBallot(condition); - var exchange = BeginWave64Exchange(); - EmitConditional(IsHalfWaveLeader(lane), () => - Store(Wave64ExchangePointer(exchange, ShiftRightLogical(lane, UInt(5))), owned)); - EmitWave64Barrier(); - return Pair64( - Load(_uintType, Wave64ExchangePointer(exchange, UInt(0))), - Load(_uintType, Wave64ExchangePointer(exchange, UInt(1)))); - } - - var ballot = _module.AddInstruction( - SpirvOp.GroupNonUniformBallot, - _uvec4Type, - UInt(3), - condition); - var low = _module.AddInstruction( - SpirvOp.CompositeExtract, - _uintType, - ballot, - 0); - var widened = _module.AddInstruction(SpirvOp.UConvert, _ulongType, low); - if (_waveLaneCount != 64) - { - return widened; - } - - return _module.AddInstruction( - SpirvOp.Select, - _ulongType, - _module.AddInstruction( - SpirvOp.UGreaterThanEqual, - _boolType, - GuestWaveLane(), - UInt(32)), - ShiftLeftLogical64( - widened, - _module.Constant64(_ulongType, 32)), - widened); - } - - private uint OwnHalfBallot(uint condition) - { - var ballot = _module.AddInstruction( - SpirvOp.GroupNonUniformBallot, - _uvec4Type, - UInt(3), - condition); - var component = ShiftRightLogical(Load(_uintType, _subgroupInvocationIdInput), UInt(5)); - // Avoid dynamic extraction of subgroup ballot words: on the NVIDIA - // device tests it produced zero masks. Select the same word explicitly. - var result = _module.AddInstruction(SpirvOp.CompositeExtract, _uintType, ballot, 0); - for (uint index = 1; index < 4; index++) - { - var word = _module.AddInstruction(SpirvOp.CompositeExtract, _uintType, ballot, index); - result = _module.AddInstruction(SpirvOp.Select, _uintType, - _module.AddInstruction(SpirvOp.IEqual, _boolType, component, UInt(index)), word, result); - } - return result; - } - - private uint BooleanToHalfWaveMask(uint condition) - { - var owned = _module.AddInstruction(SpirvOp.UConvert, _ulongType, OwnHalfBallot(condition)); - return _module.AddInstruction( - SpirvOp.Select, - _ulongType, - _module.AddInstruction(SpirvOp.UGreaterThanEqual, _boolType, GuestWaveLane(), UInt(32)), - ShiftLeftLogical64(owned, _module.Constant64(_ulongType, 32)), - owned); - } - - private uint BothHalvesOfOwnBallot(uint condition) - { - var owned = _module.AddInstruction(SpirvOp.UConvert, _ulongType, OwnHalfBallot(condition)); - return BitwiseOr64(owned, ShiftLeftLogical64(owned, _module.Constant64(_ulongType, 32))); - } - - private uint IsHalfWaveLeader(uint lane) => - _module.AddInstruction(SpirvOp.IEqual, _boolType, BitwiseAnd(lane, UInt(31)), UInt(0)); - - private uint BeginWave64Exchange() - { - var parity = Load(_uintType, _wave64ExchangeParity); - Store(_wave64ExchangeParity, BitwiseXor(parity, UInt(1))); - return IAdd( - UInt(_wave64ExchangeOffset), - _module.AddInstruction(SpirvOp.IMul, _uintType, parity, UInt(Wave64ExchangeSlotCount))); - } - - private uint Wave64ExchangePointer(uint exchange, uint slot) => - _module.AddInstruction( - SpirvOp.AccessChain, - _wave64ExchangeElementPointer, - _wave64Exchange, - IAdd(exchange, slot)); - - private uint ExchangeWave64Value(uint writer, uint value) - { - var exchange = BeginWave64Exchange(); - EmitConditional(writer, () => Store(Wave64ExchangePointer(exchange, UInt(0)), value)); - EmitWave64Barrier(); - return Load(_uintType, Wave64ExchangePointer(exchange, UInt(0))); - } - - private void EmitExactWaveMasks(IReadOnlyList pairs) - { - var lane = GuestWaveLane(); - var upperHalf = ShiftRightLogical(lane, UInt(5)); - var lowerHalf = _module.AddInstruction(SpirvOp.IEqual, _boolType, upperHalf, UInt(0)); - const int pairsPerExchange = (int)Wave64ExchangeSlotCount / 2; - for (var start = 0; start < pairs.Count; start += pairsPerExchange) - { - var count = Math.Min(pairsPerExchange, pairs.Count - start); - var owned = new uint[count]; - for (var index = 0; index < count; index++) - { - var pair = pairs[start + index]; - owned[index] = _module.AddInstruction(SpirvOp.Select, _uintType, lowerHalf, LoadS(pair), LoadS(pair + 1)); - } - - var exchange = BeginWave64Exchange(); - EmitConditional(IsHalfWaveLeader(lane), () => - { - for (var index = 0; index < count; index++) - { - Store(Wave64ExchangePointer(exchange, IAdd(UInt((uint)index * 2), upperHalf)), owned[index]); - } - }); - EmitWave64Barrier(); - for (var index = 0; index < count; index++) - { - var pair = pairs[start + index]; - StoreS(pair, Load(_uintType, Wave64ExchangePointer(exchange, UInt((uint)index * 2)))); - StoreS(pair + 1, Load(_uintType, Wave64ExchangePointer(exchange, UInt((uint)index * 2 + 1)))); - } - } - } + private uint BooleanToWaveMask(uint condition) => + _waveLowering.BooleanToWaveMask(condition); private uint WaveMaskAny(uint register, uint laneActive) => - _emulateWave64 && _subgroupInvocationIdInput != 0 - ? IsNotZero64(LoadS64(register)) - : SubgroupAny(Load(_boolType, laneActive)); - - private static readonly bool HalfWaveMasks = !string.Equals( - Environment.GetEnvironmentVariable("SHARPEMU_WAVE64_HALF_MASKS"), "0", StringComparison.Ordinal); - - private bool _halfMaskPlanBuilt; - - private Ir.Gen5Wave64HalfMaskPlan? HalfMaskPlan() - { - if (_halfMaskPlanBuilt) - { - return _halfMaskPlan; - } - - _halfMaskPlanBuilt = true; - if (HalfWaveMasks && UsesWave64Exchange() && _subgroupInvocationIdInput != 0) - { - var sharedFlatPcs = _request.Memory.Entries - .Where(static memory => memory.AddressSpace is FlatAddressSpace.Shared or FlatAddressSpace.SharedOrPrivate) - .Select(static memory => memory.Pc) - .ToHashSet(); - _halfMaskPlan = Ir.Gen5Wave64HalfMaskAnalysis.Analyze(_request.Program, sharedFlatPcs); - } - - return _halfMaskPlan; - } - - private void EmitWave64Barrier() - { - var workgroup = UInt(2); - _module.AddStatement( - SpirvOp.ControlBarrier, - workgroup, - workgroup, - UInt(0x108)); - } + _waveLowering.WaveMaskAny(register, laneActive); // A wave-mask SGPR (VCC/EXEC) consumed as a per-lane predicate — the // condition of VCndmask, a VCC/EXEC branch, or the derived _vcc/_exec @@ -8269,27 +7985,8 @@ public static partial class Gen5SpirvTranslator // In wave32 a lane mask fills its register alone and the next one keeps its value. // That includes VCC and EXEC: compilers use VCC_HI (s107) as an ordinary SGPR. - private void StoreWaveMask(uint register, uint condition) - { - if (_waveLaneCount == 32) - { - StoreS(register, Narrow(BooleanToWaveMask(condition))); - return; - } - - if (_emulateWave64 && - _subgroupInvocationIdInput != 0 && - _emittingPc is { } pc && - HalfMaskPlan() is { } plan && - plan.HalfMaskWrites.TryGetValue(pc, out var halfRegister) && - halfRegister == register) - { - StoreS64(register, BooleanToHalfWaveMask(condition)); - return; - } - - StoreS64(register, BooleanToWaveMask(condition)); - } + private void StoreWaveMask(uint register, uint condition) => + _waveLowering.StoreWaveMask(register, condition); private void EmitExecConditional(Action emit) { diff --git a/src/SharpEmu.ShaderCompiler/Gen5PixelInputMapping.cs b/src/SharpEmu.ShaderCompiler/Gen5PixelInputMapping.cs index fc2260b..1605aa7 100644 --- a/src/SharpEmu.ShaderCompiler/Gen5PixelInputMapping.cs +++ b/src/SharpEmu.ShaderCompiler/Gen5PixelInputMapping.cs @@ -9,8 +9,18 @@ public static class Gen5PixelInputMapping ReadOnlySpan controls, ReadOnlySpan activeInputs) { - Span usedLocations = stackalloc bool[32]; var locations = new uint[activeInputs.Length]; + ResolveLocations(controls, activeInputs, locations); + return locations; + } + + // Fills `locations` (one slot per active input) without allocating. + public static void ResolveLocations( + ReadOnlySpan controls, + ReadOnlySpan activeInputs, + Span locations) + { + Span usedLocations = stackalloc bool[32]; for (var index = 0; index < activeInputs.Length; index++) { @@ -39,7 +49,5 @@ public static class Gen5PixelInputMapping usedLocations[(int)location] = true; } } - - return locations; } } diff --git a/src/SharpEmu.ShaderCompiler/Resources/BindingLayout.cs b/src/SharpEmu.ShaderCompiler/Resources/BindingLayout.cs index 9829b63..6335b5f 100644 --- a/src/SharpEmu.ShaderCompiler/Resources/BindingLayout.cs +++ b/src/SharpEmu.ShaderCompiler/Resources/BindingLayout.cs @@ -359,7 +359,20 @@ public sealed class BindingLayout : IEquatable public static uint NativeBindingIndex(ShaderStage stage, DescriptorBindingKind kind) => (uint)kind + (stage == ShaderStage.Pixel ? (uint)DescriptorBindingKind.Count : 0); - public DescriptorBinding? Find(DescriptorBindingKind kind) => Descriptors.FirstOrDefault(binding => binding.Kind == kind); + // Called for every bound stage of every draw: an indexed loop allocates no closure or enumerator. + public DescriptorBinding? Find(DescriptorBindingKind kind) + { + var descriptors = Descriptors; + for (var index = 0; index < descriptors.Count; index++) + { + if (descriptors[index].Kind == kind) + { + return descriptors[index]; + } + } + + return null; + } // The user-data registers live at program entry: those some path reads before it // writes them, found by liveness over the control-flow graph, in ascending order. diff --git a/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializationCache.cs b/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializationCache.cs index 759e5bc..a34a89e 100644 --- a/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializationCache.cs +++ b/src/SharpEmu.ShaderCompiler/Resources/ResourceMaterializationCache.cs @@ -10,11 +10,14 @@ namespace SharpEmu.ShaderCompiler.Resources; // returns false so the caller falls back to a full materialization. public delegate bool ResidentGuestBytesReader(ulong address, Span destination, bool clean); -// Materialization is a pure function of the plan, the draw's user data, the shader -// base, the compute state and the guest words it reads. This cache records exactly -// those words and reuses the result while every one of them is unchanged, so a draw -// that re-binds the same resources skips the descriptor walk. Any failed read, any -// GPU-owned range and any byte difference make the entry miss; nothing is guessed. +// Materialization is a pure function of the plan, the user data words the evaluator +// reads, the shader base, the compute state and the guest words it reads. This cache +// records exactly those words and reuses the result while every one of them is +// unchanged, so a draw that re-binds the same resources skips the descriptor walk. A user +// data word the evaluator never read (a per-draw constant for the shader's own ALU, say) +// does not distinguish two draws; the snapshot a hit returns carries the current user +// data. Any failed read, any GPU-owned range and any byte difference make the entry +// miss; nothing is guessed. public sealed class ResourceMaterializationCache { // Two generations approximate LRU: a full young generation becomes the old one, @@ -24,6 +27,761 @@ public sealed class ResourceMaterializationCache private Dictionary _old = new(); private byte[] _scratch = new byte[256]; private ReadRecorder? _spareRecorder; + // The user data words each (plan, base, compute state) has read so far. They only index the + // entries; Entry.Matches is what decides a hit. + private readonly Dictionary _learnedUserData = new(); + private readonly Dictionary _rebases = new(); + + private AddressRebase RebaseOf(ulong drawKey, ShaderResourcePlan plan) + { + if (!_rebases.TryGetValue(drawKey, out var rebase)) + _rebases.Add(drawKey, rebase = new AddressRebase(plan)); + return rebase; + } + + // Titles allocate per-draw constants linearly, so a buffer's base address changes on every + // draw while everything else the draw binds stays the same; keyed on that address, the cache + // misses nearly every draw. A user data word the plan uses verbatim as a buffer's base dword + // is a candidate. It is rebased only once misses have shown it repeatedly that two draws + // differing in nothing but candidate words materialize identically, except for those base + // dwords, which equal the words: the same descriptors, table, ranges, specialization and + // recorded guest bytes. One counterexample rejects the word for good. A rebased word leaves + // the key and the match, and a hit patches the base dwords from the draw's own user data. + private sealed class AddressRebase + { + private const int Confirmations = 3; + + // A word that is a base address. A buffer candidate is the base dword of a buffer descriptor + // (Buffer >= 0): the hit patches that dword, and reads inside the descriptor's range move. + // A raw candidate is the low dword of the handle of scalar loads; its reads lie at constant + // byte offsets [MinOffset, MaxOffset] from the base whose high dword is user data word High + // (or HighConstant when High < 0). + private readonly record struct Candidate(int UserData, int Buffer, int High = -1, uint HighConstant = 0, + long MinOffset = 0, long MaxOffset = 0); + + private const ulong RawAddressMask = 0x0000_FFFF_FFFF_FFFFul; + + private readonly Candidate[] _candidates; + private readonly ulong[] _candidateMask; + private ulong[] _rejected = []; + private readonly Dictionary _confirmed = new(); + private const int RecentCapacity = 512; + private readonly Dictionary _recent = new(); + + private ulong StructureOf(Entry entry) + { + var hash = new HashCode(); + hash.Add(entry.UserData.Length); + for (var index = 0; index < entry.UserData.Length; index++) + { + if (!IsUsed(entry.UsedUserData, index) || IsUsed(_candidateMask, index)) + continue; + hash.Add(index); + hash.Add(entry.UserData[index]); + } + + return (uint)hash.ToHashCode(); + } + + private readonly ShaderResourcePlan _plan; + + public AddressRebase(ShaderResourcePlan plan) + { + _plan = plan; + var candidates = new List(); + + for (var buffer = 0; buffer < plan.Info.Buffers.Count; buffer++) + { + var source = plan.DescriptorSources[(int)plan.Info.Buffers[buffer].Source]; + if (source.Dwords.Length != 0 && UserDataIndex(source.Dwords[0]) is { } index) + candidates.Add(new Candidate(index, buffer)); + } + + var values = PlanValues(plan); + var buffered = candidates.Select(candidate => candidate.UserData).ToHashSet(); + foreach (var raw in RawCandidates(plan, values)) + if (!buffered.Contains(raw.UserData)) + candidates.Add(raw); + + _candidates = candidates.ToArray(); + _candidateMask = MaskOf(_candidates.Select(candidate => candidate.UserData)); + var memoryBases = MemoryBaseCandidates(plan); + MemoryBases = memoryBases.Select(candidate => candidate.Buffer).ToArray(); + _memoryState = new int[plan.Info.Buffers.Count]; + _memorySlots = new int[plan.Info.Buffers.Count]; + _memorySlots.AsSpan().Fill(-1); + foreach (var (buffer, slot) in memoryBases) + _memorySlots[buffer] = slot; + _memoryLearnable = memoryBases.Count; + (TableSlotBuffers, TableSlotOffsets, TableSlotsThroughBases) = TableSlotSources(plan, memoryBases); + DirectSlots = DirectSlotsOf(plan, TableSlotBuffers); + } + + // A table slot that is one dword loaded through a user data pointer at constant offsets: its + // word is that dword. High is the user data index of the pointer's high dword, or -1 for + // the constant HighConstant. + public readonly record struct DirectLoad(int Low, int High, ulong HighConstant, long Immediate, uint Offset); + + // Per table slot: its direct load (default for a slot through a memory-based buffer), or + // null when some other slot is not a direct load. + public DirectLoad[]? DirectSlots { get; } + + private DirectLoad[]? DirectSlotsOf(ShaderResourcePlan plan, int[] slotBuffers) + { + var loads = new DirectLoad[plan.TableReads.Count]; + for (var slot = 0; slot < loads.Length; slot++) + { + if (slotBuffers[slot] >= 0) + continue; + var value = plan.TableReads[slot].Value; + if (value.Kind != ScalarValueKind.ScalarAddressWord || value.Operands.Length < 2 || + value.Operands[1].Kind != ScalarValueKind.Constant || (uint)value.MemoryIndex >= (uint)plan.Memory.Count) + return null; + var handle = value.Operands[0]; + if (handle.Operands.Length < 2 || UserDataIndex(handle.Operands[0]) is not { } low) + return null; + var high = UserDataIndex(handle.Operands[1]); + if (high is null && handle.Operands[1].Kind != ScalarValueKind.Constant) + return null; + loads[slot] = new DirectLoad(low, high ?? -1, high is null ? handle.Operands[1].Payload : 0, + (int)plan.Memory[value.MemoryIndex].Offset, (uint)value.Operands[1].Payload); + } + + return loads; + } + + // Per table slot: the memory-based buffer it reads a dword of at a constant byte offset from + // the base, or -1. TableSlotsThroughBases: some slot reads through such a buffer otherwise. + public int[] TableSlotBuffers { get; } = []; + public long[] TableSlotOffsets { get; } = []; + public bool TableSlotsThroughBases { get; } + + private static (int[] Buffers, long[] Offsets, bool Through) TableSlotSources(ShaderResourcePlan plan, + List<(int Buffer, int Slot)> memoryBases) + { + var buffers = new int[plan.TableReads.Count]; + var offsets = new long[plan.TableReads.Count]; + buffers.AsSpan().Fill(-1); + if (memoryBases.Count == 0) + return (buffers, offsets, false); + + int BufferOf(ScalarValue handleWord) + { + foreach (var (buffer, slot) in memoryBases) + { + var baseWord = plan.DescriptorSources[(int)plan.Info.Buffers[buffer].Source].Dwords[0]; + if (ReferenceEquals(handleWord, baseWord) || + (slot >= 0 && handleWord.Kind == ScalarValueKind.ResourceTableWord && (int)handleWord.Payload == slot)) + return buffer; + } + + return -1; + } + + var through = false; + for (var slot = 0; slot < plan.TableReads.Count; slot++) + { + var value = plan.TableReads[slot].Value; + if (value.Kind == ScalarValueKind.ScalarBufferWord && value.Operands.Length >= 2 && + value.Operands[0].Operands.Length != 0 && value.Operands[1].Kind == ScalarValueKind.Constant && + value.MemoryIndex >= 0 && value.MemoryIndex < plan.Memory.Count && + BufferOf(value.Operands[0].Operands[0]) is var direct and >= 0) + { + buffers[slot] = direct; + offsets[slot] = ((long)plan.Memory[value.MemoryIndex].Offset + (long)(uint)value.Operands[1].Payload) & ~3L; + continue; + } + + var visited = new HashSet(); + var pending = new Stack(); + pending.Push(value); + while (!through && pending.Count != 0) + { + var current = pending.Pop(); + if (!visited.Add(current)) + continue; + if (current.Kind == ScalarValueKind.ScalarBufferWord && current.Operands.Length != 0 && + current.Operands[0].Operands.Length != 0 && BufferOf(current.Operands[0].Operands[0]) >= 0) + through = true; + foreach (var operand in current.Operands) + pending.Push(operand); + } + } + + return (buffers, offsets, through); + } + + private readonly int[] _memorySlots; + + // Memory-loaded bases, learned per buffer like the user data words: confirmations count up + // to MemoryActive; MemoryRejected is final. + private const int MemoryActive = Confirmations; + private const int MemoryRejected = -1; + private readonly int[] _memoryState; + private int _memoryLearnable; + + public IReadOnlyList MemoryBases { get; } + + public bool HasLearnableMemoryBases => _memoryLearnable != 0; + + + public bool IsRejectedMemoryBase(int buffer) => _memoryState[buffer] == MemoryRejected; + + public bool AreActiveMemoryBases(List<(int Buffer, uint Word)> patches) + { + foreach (var (buffer, _) in patches) + if (_memoryState[buffer] < MemoryActive) + return false; + return true; + } + + // A rebased refresh computed for a stale entry, compared with the full materialization of + // the same draw: agreement counts towards activating its bases, one disagreement rejects + // them for good. + public void ConfirmMemoryBases(List<(int Buffer, uint Word)> patches, bool agrees) + { + foreach (var (buffer, _) in patches) + { + if (_memoryState[buffer] == MemoryRejected) + continue; + if (!agrees) + { + _memoryState[buffer] = MemoryRejected; + _memoryLearnable--; + } + else if (_memoryState[buffer] < MemoryActive) + { + _memoryState[buffer]++; + } + } + } + + // The window of buffer reads a patched base covers for this draw. + public static bool TryMovedWindow(uint[] descriptor, uint baseWord, out ulong start, out ulong size) + { + start = size = 0; + if (descriptor.Length < 3) + return false; + start = baseWord | ((ulong)(descriptor[1] & 0xFFFF) << 32); + var stride = (descriptor[1] >> 16) & 0x3FFF; + size = stride == 0 ? descriptor[2] : (ulong)descriptor[2] * stride; + return true; + } + + // The base dword moves, and so does the table slot that holds the same word. + public ResourceSnapshot PatchMemoryBases(ResourceSnapshot snapshot, List<(int Buffer, uint Word)> patches) + { + var buffers = (uint[][])snapshot.Buffers.Clone(); + uint[]? table = null; + foreach (var (buffer, word) in patches) + { + if (ReferenceEquals(buffers[buffer], snapshot.Buffers[buffer])) + buffers[buffer] = (uint[])buffers[buffer].Clone(); + var previous = buffers[buffer][0]; + buffers[buffer][0] = word; + var slot = _memorySlots[buffer]; + var flat = slot >= 0 && slot < _plan.TableReads.Count ? (int)_plan.TableReads[slot].FlatOffset : -1; + if (flat >= 0 && flat < snapshot.FlattenedResourceTable.Length && snapshot.FlattenedResourceTable[flat] == previous) + { + table ??= (uint[])snapshot.FlattenedResourceTable.Clone(); + table[flat] = word; + } + } + + return new ResourceSnapshot + { + Buffers = buffers, + Images = snapshot.Images, + Samplers = snapshot.Samplers, + FlattenedResourceTable = table ?? snapshot.FlattenedResourceTable, + UserData = snapshot.UserData, + DeviceAddressRanges = snapshot.DeviceAddressRanges, + }; + } + + private int? UserDataIndex(ScalarValue value) + { + if (value.Kind != ScalarValueKind.UserData) + return null; + var register = value.UserDataRegister; + return register >= _plan.UserDataBase && register - _plan.UserDataBase < _plan.UserDataCount + ? (int)(register - _plan.UserDataBase) + : null; + } + + // Every scalar load of the plan whose handle takes its low dword verbatim from user data. + // A word qualifies only when all its loads use constant offsets and one high dword. + // Every value the plan evaluates, operands included. + private static HashSet PlanValues(ShaderResourcePlan plan) + { + var roots = new List(); + foreach (var source in plan.DescriptorSources) + roots.AddRange(source.Dwords); + roots.AddRange(plan.TableReads.Select(read => read.Value)); + roots.AddRange(plan.DynamicReads); + foreach (var access in plan.Accesses) + { + if (access is null) + continue; + foreach (var value in new[] { access.Handle, access.SamplerHandle, access.Read, access.Offset, access.Active }) + if (value is not null) + roots.Add(value); + } + + roots.AddRange(plan.Graph.Values); + var visited = new HashSet(); + var pending = new Stack(roots); + while (pending.Count != 0) + { + var value = pending.Pop(); + if (!visited.Add(value)) + continue; + foreach (var operand in value.Operands) + pending.Push(operand); + } + + return visited; + } + + // Buffers whose base dword is read from guest memory, directly or through a slot of the + // flattened table: descriptor tables the title rewrites per frame or per draw point them at + // freshly allocated constants. The slot (or -1) is where the table holds the same word. + // Recorded reads inside such a buffer's window move with its base. + private static List<(int Buffer, int Slot)> MemoryBaseCandidates(ShaderResourcePlan plan) + { + var bases = new List<(int Buffer, int Slot)>(); + for (var buffer = 0; buffer < plan.Info.Buffers.Count; buffer++) + { + var source = plan.DescriptorSources[(int)plan.Info.Buffers[buffer].Source]; + if (source.Dwords.Length < 2) + continue; + var baseWord = source.Dwords[0]; + var slot = baseWord.Kind == ScalarValueKind.ResourceTableWord ? (int)baseWord.Payload : -1; + if (slot < 0 && baseWord.Kind is not (ScalarValueKind.ScalarAddressWord or ScalarValueKind.ScalarBufferWord)) + continue; + bases.Add((buffer, slot)); + } + + return bases; + } + + private IEnumerable RawCandidates(ShaderResourcePlan plan, HashSet values) + { + var found = new Dictionary(); + var disqualified = new HashSet(); + foreach (var value in values) + { + if (value.Kind is not (ScalarValueKind.ScalarAddressWord or ScalarValueKind.ScalarBufferWord) || + value.Operands.Length < 2 || value.Operands[0].Operands.Length < 2 || + UserDataIndex(value.Operands[0].Operands[0]) is not { } index) + continue; + + var high = value.Operands[0].Operands[1]; + var highIndex = UserDataIndex(high); + var offset = value.Operands[1]; + if ((highIndex is null && high.Kind != ScalarValueKind.Constant) || offset.Kind != ScalarValueKind.Constant || + value.MemoryIndex >= plan.Memory.Count) + { + disqualified.Add(index); + continue; + } + + var immediate = (long)plan.Memory[value.MemoryIndex].Offset; + var relative = value.Kind == ScalarValueKind.ScalarBufferWord + ? (immediate + (long)(uint)offset.Payload) & ~3L + : (immediate & ~3L) + (long)((uint)offset.Payload & ~3u); + var candidate = new Candidate(index, -1, highIndex ?? -1, highIndex is null ? (uint)high.Payload : 0, relative, relative); + if (found.TryGetValue(index, out var existing)) + { + if (existing.High != candidate.High || existing.HighConstant != candidate.HighConstant) + { + disqualified.Add(index); + continue; + } + + candidate = existing with + { + MinOffset = Math.Min(existing.MinOffset, relative), + MaxOffset = Math.Max(existing.MaxOffset, relative), + }; + } + + found[index] = candidate; + } + + return found.Values.Where(candidate => !disqualified.Contains(candidate.UserData)); + } + + // The window [start, start + size) of reads a candidate's base covers in this entry. + private bool TryWindow(Entry entry, in Candidate candidate, out ulong start, out ulong size, out ulong addressBase) + { + start = size = addressBase = 0; + if (candidate.Buffer >= 0) + { + if (candidate.Buffer >= entry.Snapshot.Buffers.Length) + return false; + var descriptor = entry.Snapshot.Buffers[candidate.Buffer]; + if (descriptor.Length < 3) + return false; + start = addressBase = descriptor[0] | ((ulong)(descriptor[1] & 0xFFFF) << 32); + var stride = (descriptor[1] >> 16) & 0x3FFF; + size = stride == 0 ? descriptor[2] : (ulong)descriptor[2] * stride; + return true; + } + + // The window starts at the base itself: the read prefetch fetches a pointer's block from + // its base, ahead of the lowest offset a load uses. + addressBase = RawBase(entry.UserData, candidate, entry.UserData[candidate.UserData]); + var lowest = Math.Min(0, candidate.MinOffset); + var low = (long)addressBase + lowest; + if (low < 0) + return false; + start = (ulong)low; + size = (ulong)(candidate.MaxOffset - lowest) + sizeof(uint); + return true; + } + + private static ulong RawBase(IReadOnlyList userData, in Candidate candidate, uint low) + { + var high = candidate.High >= 0 ? userData[candidate.High] : candidate.HighConstant; + return (((ulong)high << 32) | low) & RawAddressMask & ~3UL; + } + + // The rebased words; replaced, never mutated, so a caller can compare references. + public ulong[] Active { get; private set; } = []; + + // Every rebased base dword of the entry equals the word it came from. + public bool AppliesTo(Entry entry) + { + if (Active.Length == 0) + return true; + foreach (var (userData, buffer, _, _, _, _) in _candidates) + { + if (buffer >= 0 && IsUsed(Active, userData) && + (buffer >= entry.Snapshot.Buffers.Length || entry.Snapshot.Buffers[buffer].Length == 0 || + userData >= entry.UserData.Length || entry.Snapshot.Buffers[buffer][0] != entry.UserData[userData])) + return false; + } + + return true; + } + + public ResourceSnapshot Apply(ResourceSnapshot snapshot, IReadOnlyList userData) + { + if (Active.Length == 0) + return snapshot; + uint[][]? buffers = null; + foreach (var (index, buffer, _, _, _, _) in _candidates) + { + if (buffer < 0 || !IsUsed(Active, index) || snapshot.Buffers[buffer][0] == userData[index]) + continue; + buffers ??= (uint[][])snapshot.Buffers.Clone(); + if (ReferenceEquals(buffers[buffer], snapshot.Buffers[buffer])) + buffers[buffer] = (uint[])buffers[buffer].Clone(); + buffers[buffer][0] = userData[index]; + } + + return buffers is null + ? snapshot + : new ResourceSnapshot + { + Buffers = buffers, + Images = snapshot.Images, + Samplers = snapshot.Samplers, + FlattenedResourceTable = snapshot.FlattenedResourceTable, + UserData = snapshot.UserData, + DeviceAddressRanges = snapshot.DeviceAddressRanges, + }; + } + + // Compares a fresh materialization with the latest one that agrees on every word but the + // candidates: the same object drawn again, usually a frame earlier, with new constants. + // Comparing with the plan's previous draw instead finds another object's textures in + // the way and never concludes. + public void Learn(Entry built, + Func? Patches)?>? predict = null) + { + if (_candidates.Length == 0) + return; + var structure = StructureOf(built); + _recent.TryGetValue(structure, out var previous); + if (_recent.Count >= RecentCapacity) + _recent.Clear(); + _recent[structure] = built; + if (previous is null || previous.UserData.Length != built.UserData.Length || + !previous.UsedUserData.AsSpan().SequenceEqual(built.UsedUserData)) + return; + + var differing = new List(); + for (var index = 0; index < built.UserData.Length; index++) + { + if (!IsUsed(built.UsedUserData, index) || previous.UserData[index] == built.UserData[index]) + continue; + if (!IsUsed(_candidateMask, index)) + return; + differing.Add(index); + } + + // A word already rejected explains any mismatch; blaming the others with it would reject + // sound bases that merely change together with it. + if (differing.Count == 0 || differing.Any(index => IsUsed(_rejected, index))) + return; + + // Reads must land where the previous draw's did, moved with a rebased buffer when they + // fall inside its window. Guest bytes that changed between the two draws make the + // comparison say nothing about the words. + var reads = CompareMovedReads(this, previous, built, differing); + if (reads == ReadComparison.Inconclusive) + { + // Changed guest bytes hide whether the words are bases. Predict this draw from the + // previous one as a hit with the words rebased would, moved memory-loaded bases + // included, and judge the words by the prediction instead. + if (predict?.Invoke(previous, Or(Active, MaskOf(differing))) is not { } prediction) + return; + + var agrees = Difference(prediction.Snapshot, previous.Specialization, built.Snapshot, built.Specialization) is null; + if (prediction.Patches is { } patches) + { + // A wrong prediction that moved bases blames those bases, not the words. + ConfirmMemoryBases(patches, agrees); + if (!agrees) + return; + } + else if (!agrees) + { + _rejected = Or(_rejected, MaskOf(differing)); + if (Overlaps(Active, _rejected)) + Active = AndNot(Active, _rejected); + return; + } + + Confirm(differing); + return; + } + + if (reads == ReadComparison.Mismatch || + !SameExceptBases(previous, built, differing, compareTable: reads == ReadComparison.SameBytes)) + { + _rejected = Or(_rejected, MaskOf(differing)); + if (Overlaps(Active, _rejected)) + Active = AndNot(Active, _rejected); + return; + } + + Confirm(differing); + } + + private void Confirm(List differing) + { + var activated = new List(); + foreach (var index in differing) + { + var count = _confirmed.GetValueOrDefault(index) + 1; + _confirmed[index] = count; + if (count >= Confirmations && !IsUsed(_rejected, index) && !IsUsed(Active, index)) + activated.Add(index); + } + + if (activated.Count != 0) + Active = Or(Active, MaskOf(activated)); + } + + // Runs a prediction as though the words in probe were rebased already. + public T WithActive(ulong[] probe, Func run) + { + var saved = Active; + Active = probe; + try + { + return run(); + } + finally + { + Active = saved; + } + } + + // The flattened table is left out when only table-only bytes changed: it holds the words read + // from memory, whose addresses the read comparison already checked. + private bool SameExceptBases(Entry previous, Entry built, List differing, bool compareTable) + { + if (!previous.Specialization.Equals(built.Specialization)) + return false; + + var left = previous.Snapshot; + var right = built.Snapshot; + if (!SameWords(left.Images, right.Images) || !SameWords(left.Samplers, right.Samplers) || + (compareTable + ? !left.FlattenedResourceTable.AsSpan().SequenceEqual(right.FlattenedResourceTable) + : left.FlattenedResourceTable.Length != right.FlattenedResourceTable.Length) || + !left.DeviceAddressRanges.AsSpan().SequenceEqual(right.DeviceAddressRanges) || + left.Buffers.Length != right.Buffers.Length) + return false; + + for (var buffer = 0; buffer < left.Buffers.Length; buffer++) + { + var before = left.Buffers[buffer]; + var after = right.Buffers[buffer]; + if (before.Length != after.Length) + return false; + for (var dword = 0; dword < before.Length; dword++) + { + if (dword == 0 && BaseWord(buffer, differing) is { } index) + { + if (before[0] != previous.UserData[index] || after[0] != built.UserData[index]) + return false; + } + else if (before[dword] != after[dword]) + { + return false; + } + } + } + + return true; + } + + // Every read of the built draw is the previous draw's read, moved by the rebase when it lies + // in a moved buffer's window; bytesDiffer tells whether the guest bytes read there changed. + private enum ReadComparison + { + // A read moved where the words cannot explain it while every byte read alike matched. + Mismatch, + // Guest bytes changed where the descriptors depend on them, or a changed pointer in + // guest memory redirected later reads: the pair says nothing about the words. + Inconclusive, + SameBytes, + // Only words the flattened table alone reads changed: per-draw constants. + TableBytesDiffer, + } + + private static ReadComparison CompareMovedReads(AddressRebase rebase, Entry previous, Entry built, List differing) + { + var matched = previous.RangeAddresses.Length == built.RangeAddresses.Length; + var bytesDiffer = false; + var descriptorBytesDiffer = false; + var common = Math.Min(previous.RangeAddresses.Length, built.RangeAddresses.Length); + for (var range = 0; range < common; range++) + { + var length = previous.RangeLengths[range]; + if (rebase.Move(previous, built.UserData, previous.RangeAddresses[range], length, differing) != built.RangeAddresses[range] || + length != built.RangeLengths[range] || previous.RangeClean[range] != built.RangeClean[range]) + { + matched = false; + continue; + } + + for (var offset = 0; offset < length; offset += sizeof(uint)) + { + var before = previous.RangeOffsets[range] + offset; + var after = built.RangeOffsets[range] + offset; + if (previous.Bytes.AsSpan(before, sizeof(uint)).SequenceEqual(built.Bytes.AsSpan(after, sizeof(uint)))) + continue; + bytesDiffer = true; + descriptorBytesDiffer |= !previous.WordTableOnly[before / sizeof(uint)] || !built.WordTableOnly[after / sizeof(uint)]; + } + } + + if (!matched) + return bytesDiffer ? ReadComparison.Inconclusive : ReadComparison.Mismatch; + if (descriptorBytesDiffer || (bytesDiffer && !(previous.TableRefreshable && built.TableRefreshable))) + return ReadComparison.Inconclusive; + return bytesDiffer ? ReadComparison.TableBytesDiffer : ReadComparison.SameBytes; + } + + // Where a read the entry recorded lies for a draw with this user data: a read inside the + // window of a candidate whose base word moved moves with it. Only the words in moving count + // (the differing words, or the rebased ones on a hit). + private ulong Move(Entry entry, IReadOnlyList userData, ulong address, int length, IReadOnlyCollection? moving) + { + foreach (var candidate in _candidates) + { + var index = candidate.UserData; + if (moving is null ? !IsUsed(Active, index) : !moving.Contains(index)) + continue; + if (userData[index] == entry.UserData[index] || !TryWindow(entry, candidate, out var start, out var size, out var addressBase)) + continue; + // Only a range wholly inside the window moves: a range that merely overlaps it can hold + // another block the title allocated next to it, which must not move. + if (address < start || address - start + (ulong)length > size) + continue; + var movedBase = candidate.Buffer >= 0 + ? (addressBase & ~0xFFFF_FFFFUL) | userData[index] + : RawBase(entry.UserData, candidate, userData[index]); + return address - addressBase + movedBase; + } + + return address; + } + + // A hit reads the entry's recorded ranges where they lie for this draw. + public ulong MoveRead(Entry entry, IReadOnlyList userData, ulong address, int length) => + Active.Length == 0 ? address : Move(entry, userData, address, length, null); + + // True when a rebased base of the entry differs from this draw's word. + public bool Moves(Entry entry, IReadOnlyList userData) + { + foreach (var candidate in _candidates) + if (IsUsed(Active, candidate.UserData) && entry.UserData[candidate.UserData] != userData[candidate.UserData]) + return true; + return false; + } + + private int? BaseWord(int buffer, List differing) + { + foreach (var candidate in _candidates) + if (candidate.Buffer == buffer && differing.Contains(candidate.UserData)) + return candidate.UserData; + return null; + } + + private static bool SameWords(uint[][] left, uint[][] right) + { + if (left.Length != right.Length) + return false; + for (var index = 0; index < left.Length; index++) + if (!left[index].AsSpan().SequenceEqual(right[index])) + return false; + return true; + } + + private static ulong[] MaskOf(IEnumerable indices) + { + var mask = Array.Empty(); + foreach (var index in indices) + { + if (mask.Length <= index >> 6) + Array.Resize(ref mask, (index >> 6) + 1); + mask[index >> 6] |= 1UL << (index & 63); + } + + return mask; + } + + private static ulong[] Or(ulong[] left, ulong[] right) + { + var result = new ulong[Math.Max(left.Length, right.Length)]; + for (var index = 0; index < result.Length; index++) + result[index] = (index < left.Length ? left[index] : 0) | (index < right.Length ? right[index] : 0); + return result; + } + + private static ulong[] AndNot(ulong[] left, ulong[] right) + { + var result = (ulong[])left.Clone(); + for (var index = 0; index < result.Length && index < right.Length; index++) + result[index] &= ~right[index]; + return result.All(word => word == 0) ? [] : result; + } + + private static bool Overlaps(ulong[] left, ulong[] right) + { + for (var index = 0; index < left.Length && index < right.Length; index++) + if ((left[index] & right[index]) != 0) + return true; + return false; + } + } public ResourceMaterializationCache(int generationCapacity = 16384) { @@ -33,11 +791,12 @@ public sealed class ResourceMaterializationCache private static long _totalHits; private static long _totalMisses; private static long _totalUncacheable; - private const int MaxVariants = 4; + private const int MaxVariants = 8; private static long _totalStale; private static long _totalStaleUnreadable; private static long _totalRefreshes; + [ThreadStatic] private static bool _readingTable; @@ -66,6 +825,46 @@ public sealed class ResourceMaterializationCache $"[PERF][RESOURCE_CACHE] hits={hits} misses={misses} stale={stale} stale_unreadable={staleUnreadable} refreshes={refreshes} uncacheable={uncacheable} hit_rate={(total == 0 ? 0 : hits * 100.0 / total):F1}% {RawReadPrefetch.TakeReport()}"); } + // The first part a cached result differs from a fresh one in, or null when they are the same. + private static string? Difference(ResourceSnapshot cached, ResourceSpecialization cachedSpecialization, + ResourceSnapshot fresh, ResourceSpecialization freshSpecialization) + { + static string? Words(string name, uint[][] left, uint[][] right) + { + if (left.Length != right.Length) + return $"{name}.count {left.Length}!={right.Length}"; + for (var index = 0; index < left.Length; index++) + { + if (left[index].Length != right[index].Length) + return $"{name}[{index}].length"; + for (var word = 0; word < left[index].Length; word++) + if (left[index][word] != right[index][word]) + return $"{name}[{index}][{word}] cached=0x{left[index][word]:X} fresh=0x{right[index][word]:X}"; + } + + return null; + } + + if (!cachedSpecialization.Equals(freshSpecialization)) + return "specialization"; + if (Words("buffer", cached.Buffers, fresh.Buffers) is { } buffers) + return buffers; + if (Words("image", cached.Images, fresh.Images) is { } images) + return images; + if (Words("sampler", cached.Samplers, fresh.Samplers) is { } samplers) + return samplers; + if (cached.FlattenedResourceTable.Length != fresh.FlattenedResourceTable.Length) + return "table.length"; + for (var slot = 0; slot < cached.FlattenedResourceTable.Length; slot++) + if (cached.FlattenedResourceTable[slot] != fresh.FlattenedResourceTable[slot]) + return $"table[{slot}] cached=0x{cached.FlattenedResourceTable[slot]:X} fresh=0x{fresh.FlattenedResourceTable[slot]:X}"; + if (!cached.DeviceAddressRanges.AsSpan().SequenceEqual(fresh.DeviceAddressRanges)) + return "ranges"; + if (!cached.UserData.AsSpan().SequenceEqual(fresh.UserData)) + return "user_data"; + return null; + } + public bool Materialize( ShaderResourcePlan plan, ResourceRuntimeInputs inputs, @@ -74,17 +873,23 @@ public sealed class ResourceMaterializationCache ref ResourceSpecialization specialization, out ResourceMaterializationFailure failure) { - var key = KeyOf(plan, inputs); - var found = TryFind(key, plan, inputs, out var cached); + var drawKey = DrawKeyOf(plan, inputs); + _learnedUserData.TryGetValue(drawKey, out var learned); + var rebase = RebaseOf(drawKey, plan); + var key = KeyOf(drawKey, inputs.UserData, learned, rebase.Active); + var found = TryFind(key, plan, inputs, rebase.Active, out var cached); + Entry? stale = null; + (List<(int Buffer, uint Word)> Patches, ResourceSnapshot Snapshot, Entry From)? learning = null; if (found) { var unreadable = false; Entry? previous = null; for (var variant = cached; variant is not null; previous = variant, variant = variant.Next) { - if (!variant.Matches(plan, inputs)) + if (!variant.Matches(plan, inputs, rebase.Active) || !rebase.AppliesTo(variant)) continue; - if (Validate(variant, residentReader, out var variantUnreadable)) + stale ??= variant; + if (Validate(variant, residentReader, rebase, inputs.UserData, out var variantUnreadable)) { if (previous is not null) { @@ -95,7 +900,7 @@ public sealed class ResourceMaterializationCache Hits++; Interlocked.Increment(ref _totalHits); - snapshot = variant.Snapshot; + snapshot = rebase.Apply(WithCurrentUserData(variant.Snapshot, inputs.UserData), inputs.UserData); specialization = variant.Specialization; failure = default; return true; @@ -104,11 +909,55 @@ public sealed class ResourceMaterializationCache unreadable |= variantUnreadable; } - if (TryRefreshTable(key, cached, plan, inputs, residentReader, out var refreshed)) + // Moved bases first: a changed base word is a descriptor word, which a table refresh + // would read in full only to refuse. + ResourceSnapshot rebased = null!; + List<(int Buffer, uint Word)> patches = null!; + // Each variant is tried: a key's variants are often the same draw with different + // descriptors (materials), and only the one this draw binds can be rebased. + Entry? refreshedFrom = null; + Entry? beforeRefreshed = null; + if (stale is not null && rebase.HasLearnableMemoryBases) + { + for (Entry? previousVariant = null, variant = cached; variant is not null; previousVariant = variant, variant = variant.Next) + { + if (!variant.Matches(plan, inputs, rebase.Active) || !rebase.AppliesTo(variant)) + continue; + if (TryRebaseRefresh(variant, plan, inputs, residentReader, rebase, out rebased, out patches)) + { + refreshedFrom = variant; + beforeRefreshed = previousVariant; + break; + } + } + } + + if (refreshedFrom is not null) + { + if (rebase.AreActiveMemoryBases(patches)) + { + if (beforeRefreshed is not null) + { + beforeRefreshed.Next = refreshedFrom.Next; + refreshedFrom.Next = cached; + Store(key, refreshedFrom); + } + + Hits++; + Interlocked.Increment(ref _totalHits); + snapshot = rebased; + specialization = refreshedFrom.Specialization; + failure = default; + return true; + } + + learning = (patches, rebased, refreshedFrom); + } + else if (rebase.AppliesTo(cached) && TryRefreshTable(key, cached, plan, inputs, residentReader, rebase, out var refreshed)) { TableRefreshes++; Interlocked.Increment(ref _totalRefreshes); - snapshot = refreshed.Snapshot; + snapshot = rebase.Apply(WithCurrentUserData(refreshed.Snapshot, inputs.UserData), inputs.UserData); specialization = refreshed.Specialization; failure = default; return true; @@ -126,7 +975,7 @@ public sealed class ResourceMaterializationCache { var recording = new ResourceRuntimeInputs { - UserData = inputs.UserData, + UserData = recorder.WrapUserData(inputs.UserData), ShaderBase = inputs.ShaderBase, ReadMemory = recorder.Wrap(inputs.ReadMemory, clean: false), ReadCleanMemory = recorder.Wrap(inputs.ReadCleanMemory, clean: true), @@ -145,7 +994,23 @@ public sealed class ResourceMaterializationCache return true; } - var built = recorder.Build(plan, inputs, snapshot, specialization); + var built = recorder.Build(plan, inputs, snapshot, specialization, rebase.MemoryBases); + if (learning is { } pending) + { + var agrees = Difference(pending.Snapshot, pending.From.Specialization, built.Snapshot, built.Specialization) is null; + rebase.ConfirmMemoryBases(pending.Patches, agrees); + } + + var activeBefore = rebase.Active; + Learn(rebase, built, plan, inputs, residentReader); + var widened = Union(learned, built.UsedUserData); + if (!ReferenceEquals(widened, learned) || !ReferenceEquals(activeBefore, rebase.Active)) + { + _learnedUserData[drawKey] = widened; + key = KeyOf(drawKey, inputs.UserData, widened, rebase.Active); + found = false; + } + if (found) { built.Next = cached; @@ -175,17 +1040,29 @@ public sealed class ResourceMaterializationCache // flattened table, only the table is evaluated again. It is refused when the plan's // specialization reads or extends the table (indirect or candidate tables), when the table // now reads a word the entry did not validate, or when a word moved between the two reads. + // + // With rebased words the entry's ranges are read where they lie for this draw. The refreshed + // entry keeps the entry's own addresses and base words with the bytes read there, so a later + // draw moves its reads the same way and compares against those bytes. private bool TryRefreshTable(ulong key, Entry cached, ShaderResourcePlan plan, ResourceRuntimeInputs inputs, - ResidentGuestBytesReader residentReader, out Entry refreshed) + ResidentGuestBytesReader residentReader, AddressRebase rebase, out Entry refreshed) { refreshed = null!; if (!cached.TableRefreshable) return false; var current = new byte[cached.Bytes.Length]; + var moved = cached.RangeAddresses; + if (rebase.Moves(cached, inputs.UserData)) + { + moved = new ulong[cached.RangeAddresses.Length]; + for (var index = 0; index < moved.Length; index++) + moved[index] = rebase.MoveRead(cached, inputs.UserData, cached.RangeAddresses[index], cached.RangeLengths[index]); + } + for (var index = 0; index < cached.RangeAddresses.Length; index++) { - if (!residentReader(cached.RangeAddresses[index], current.AsSpan(cached.RangeOffsets[index], cached.RangeLengths[index]), cached.RangeClean[index])) + if (!residentReader(moved[index], current.AsSpan(cached.RangeOffsets[index], cached.RangeLengths[index]), cached.RangeClean[index])) return false; } @@ -233,8 +1110,10 @@ public sealed class ResourceMaterializationCache foreach (var (address, word, _, _) in recorder.Reads) { - if (!TryFindWord(cached, address, out var offset) || - System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(current.AsSpan(offset, sizeof(uint))) != word) + var found = ReferenceEquals(moved, cached.RangeAddresses) + ? TryFindWord(cached, address, out var offset) + : TryFindMovedWord(cached, moved, address, out offset); + if (!found || System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(current.AsSpan(offset, sizeof(uint))) != word) return false; } @@ -247,6 +1126,7 @@ public sealed class ResourceMaterializationCache { Plan = cached.Plan, UserData = cached.UserData, + UsedUserData = cached.UsedUserData, ShaderBase = cached.ShaderBase, ComputeState = cached.ComputeState, RangeAddresses = cached.RangeAddresses, @@ -254,6 +1134,7 @@ public sealed class ResourceMaterializationCache RangeLengths = cached.RangeLengths, RangeClean = cached.RangeClean, WordTableOnly = cached.WordTableOnly, + BaseWordBuffers = cached.BaseWordBuffers, TableRefreshable = true, Bytes = current, Snapshot = new ResourceSnapshot @@ -290,6 +1171,26 @@ public sealed class ResourceMaterializationCache _spareRecorder = recorder; } + // The byte offset of a dword read at address when the entry's ranges lie at moved addresses. + private static bool TryFindMovedWord(Entry entry, ulong[] moved, ulong address, out int offset) + { + // moved may be a reused buffer longer than the entry's ranges. + for (var index = 0; index < entry.RangeAddresses.Length; index++) + { + if (address < moved[index]) + continue; + var delta = address - moved[index]; + if (delta % sizeof(uint) == 0 && delta < (ulong)entry.RangeLengths[index]) + { + offset = entry.RangeOffsets[index] + (int)delta; + return true; + } + } + + offset = 0; + return false; + } + // The byte offset of a recorded dword in the entry's bytes, found by its address. private static bool TryFindWord(Entry entry, ulong address, out int offset) { @@ -319,29 +1220,85 @@ public sealed class ResourceMaterializationCache return true; } - private static ulong KeyOf(ShaderResourcePlan plan, ResourceRuntimeInputs inputs) + private static ulong DrawKeyOf(ShaderResourcePlan plan, ResourceRuntimeInputs inputs) { var hash = new HashCode(); hash.Add(RuntimeHelpers.GetHashCode(plan)); hash.Add(inputs.ShaderBase); hash.Add(inputs.ComputeState); - var userData = inputs.UserData; - hash.Add(userData.Count); - for (var index = 0; index < userData.Count; index++) - hash.Add(userData[index]); - var low = (uint)hash.ToHashCode(); - // A second, independent mix keeps accidental collisions out of the 64-bit key. - var high = 0x9E3779B9u; - for (var index = 0; index < userData.Count; index++) - high = (high ^ userData[index]) * 0x01000193u; + hash.Add(inputs.UserData.Count); + return (ulong)(uint)hash.ToHashCode() | ((ulong)(uint)RuntimeHelpers.GetHashCode(plan) << 32); + } + + // Only the words the draw's shader has been seen to read take part, less the rebased ones: the + // key is an index, and Entry.Matches decides whether an entry is reusable. + private static ulong KeyOf(ulong drawKey, IReadOnlyList userData, ulong[]? learned, ulong[]? rebased = null) + { + var low = (uint)drawKey; + var high = (uint)(drawKey >> 32) ^ 0x9E3779B9u; + if (learned is not null) + { + for (var index = 0; index < userData.Count; index++) + { + if (!IsUsed(learned, index) || (rebased is not null && IsUsed(rebased, index))) + continue; + var word = userData[index]; + low = (low ^ word) * 0x01000193u; + high = ((high + word) * 0x85EBCA6Bu) ^ (high >> 13); + } + } + return ((ulong)high << 32) | low; } - private bool TryFind(ulong key, ShaderResourcePlan plan, ResourceRuntimeInputs inputs, out Entry entry) + private static bool IsUsed(ulong[] mask, int index) => + (uint)(index >> 6) < (uint)mask.Length && (mask[index >> 6] & (1UL << (index & 63))) != 0; + + // The same array when nothing new was learned, so the caller can tell. + private static ulong[] Union(ulong[]? learned, ulong[] used) + { + if (learned is null) + return used; + var changed = learned.Length < used.Length; + for (var index = 0; !changed && index < used.Length; index++) + changed = (used[index] & ~learned[index]) != 0; + if (!changed) + return learned; + var merged = new ulong[Math.Max(learned.Length, used.Length)]; + for (var index = 0; index < merged.Length; index++) + merged[index] = (index < learned.Length ? learned[index] : 0) | (index < used.Length ? used[index] : 0); + return merged; + } + + // A cached snapshot is shared by every draw that hits it, so it keeps the user data of the draw + // that built it; a draw whose unread words differ gets its own copy that carries its own. + private static ResourceSnapshot WithCurrentUserData(ResourceSnapshot cached, IReadOnlyList userData) + { + var words = cached.UserData; + var same = words.Length == userData.Count; + for (var index = 0; same && index < words.Length; index++) + same = words[index] == userData[index]; + if (same) + return cached; + var current = new uint[userData.Count]; + for (var index = 0; index < current.Length; index++) + current[index] = userData[index]; + return new ResourceSnapshot + { + Buffers = cached.Buffers, + Images = cached.Images, + Samplers = cached.Samplers, + FlattenedResourceTable = cached.FlattenedResourceTable, + UserData = current, + DeviceAddressRanges = cached.DeviceAddressRanges, + }; + } + + private bool TryFind(ulong key, ShaderResourcePlan plan, ResourceRuntimeInputs inputs, ulong[] rebased, out Entry entry) { if (_young.TryGetValue(key, out entry!)) - return entry.Matches(plan, inputs); - if (!_old.TryGetValue(key, out entry!) || !entry.Matches(plan, inputs)) + return entry.Matches(plan, inputs, rebased); + if (!_old.TryGetValue(key, out entry!) || !entry.Matches(plan, inputs, rebased)) return false; _old.Remove(key); Store(key, entry); @@ -359,16 +1316,390 @@ public sealed class ResourceMaterializationCache _young[key] = entry; } - private bool Validate(Entry entry, ResidentGuestBytesReader residentReader, out bool unreadable) + // What a hit on the previous entry would give this draw with the words in probe rebased: its + // reads moved and its bytes unchanged, or its memory-loaded bases moved as well. Null when + // neither path applies, which says nothing about the words. + private (ResourceSnapshot Snapshot, List<(int Buffer, uint Word)>? Patches)? Predict(Entry previous, ShaderResourcePlan plan, + ResourceRuntimeInputs inputs, ResidentGuestBytesReader residentReader, AddressRebase rebase, ulong[] probe) => + rebase.WithActive<(ResourceSnapshot, List<(int Buffer, uint Word)>?)?>(probe, () => + { + if (!previous.Matches(plan, inputs, probe) || !rebase.AppliesTo(previous)) + return null; + if (Validate(previous, residentReader, rebase, inputs.UserData, out _)) + return (rebase.Apply(WithCurrentUserData(previous.Snapshot, inputs.UserData), inputs.UserData), null); + if (rebase.HasLearnableMemoryBases && + TryRebaseRefresh(previous, plan, inputs, residentReader, rebase, out var snapshot, out var patches)) + return (snapshot, patches); + return null; + }); + + // Its own method: a lambda in MaterializeCore that captured the parameters made every call, + // hits included, allocate the closure on entry. + private void Learn(AddressRebase rebase, Entry built, ShaderResourcePlan plan, ResourceRuntimeInputs inputs, + ResidentGuestBytesReader residentReader) => + rebase.Learn(built, (previous, probe) => Predict(previous, plan, inputs, residentReader, rebase, probe)); + + // The table for this draw without evaluating it: a slot that reads a memory-based buffer at a + // constant offset takes the word now at that offset, which the moved window has just read; the + // other slots read nothing that changed, so they keep the entry's words, except a base slot, + // which the patch has already moved. current holds this draw's words in the entry's layout. + // The table is patched in place: the caller passes a copy it owns. + private static bool TryCopyTable(Entry entry, ShaderResourcePlan plan, AddressRebase rebase, byte[] current, uint[] table) + { + var sources = entry.TableCopySources ??= TableCopySourcesOf(entry, plan, rebase, table.Length); + if (sources.Length == 0) + return false; + for (var slot = 0; slot < sources.Length; slot++) + { + var offset = sources[slot]; + if (offset >= 0) + table[(int)plan.TableReads[slot].FlatOffset] = System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(current.AsSpan(offset)); + } + + return true; + } + + // A table whose slots are all direct loads or reads through memory-based buffers. + private static bool CanCopyDirect(AddressRebase rebase) + { + if (rebase.DirectSlots is not null) + return true; + return false; + } + + + // Every slot takes the dword it loads for this draw. A direct slot's address is evaluated from + // the draw's own user data, as the table evaluation would, and its dword is taken from the range + // this draw read around it; a slot through a memory-based buffer copies as in TryCopyTable. + // Nothing is written unless every slot is placed. + private static bool TryCopyDirectTable(Entry entry, ShaderResourcePlan plan, AddressRebase rebase, byte[] current, + ulong[] moved, IReadOnlyList userData, uint[] table) + { + var through = entry.TableCopySources ??= TableCopySourcesOf(entry, plan, rebase, table.Length); + var direct = rebase.DirectSlots!; + if (through.Length != direct.Length) + return Unplaced(); + Span sources = direct.Length <= 256 ? stackalloc int[direct.Length] : new int[direct.Length]; + for (var slot = 0; slot < direct.Length; slot++) + { + if (plan.TableReads[slot].FlatOffset >= table.Length) + return Unplaced(); + if (rebase.TableSlotBuffers[slot] >= 0) + { + if (through[slot] < 0) + return Unplaced(); + sources[slot] = through[slot]; + continue; + } + + var load = direct[slot]; + if (load.Low >= userData.Count || load.High >= userData.Count) + return Unplaced(); + var high = load.High >= 0 ? userData[load.High] : load.HighConstant; + var baseAddress = ((high << 32) | userData[load.Low]) & 0x0000_FFFF_FFFF_FFFFul; + if (RuntimeValueEvaluator.ResolveRawAddress(ScalarValueKind.ScalarAddressWord, baseAddress, high, 0, + load.Immediate, load.Offset, out var address) != RuntimeValueEvaluator.RawAddress.Read || + !TryFindMovedWord(entry, moved, address, out sources[slot])) + return Unplaced(); + } + + for (var slot = 0; slot < sources.Length; slot++) + table[(int)plan.TableReads[slot].FlatOffset] = System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(current.AsSpan(sources[slot])); + return true; + + static bool Unplaced() + { + return false; + } + } + + // Per table slot: the byte offset in the entry's words of the word the slot copies, or -1 for a + // slot that keeps its word. Empty when some slot cannot be placed. + private static int[] TableCopySourcesOf(Entry entry, ShaderResourcePlan plan, AddressRebase rebase, int tableLength) + { + var sources = new int[plan.TableReads.Count]; + for (var slot = 0; slot < sources.Length; slot++) + { + sources[slot] = -1; + var buffer = rebase.TableSlotBuffers[slot]; + if (buffer < 0) + continue; + var descriptor = entry.Snapshot.Buffers[buffer]; + if (!AddressRebase.TryMovedWindow(descriptor, descriptor[0], out var start, out var size) || + rebase.TableSlotOffsets[slot] < 0 || (ulong)rebase.TableSlotOffsets[slot] + sizeof(uint) > size || + !TryFindWord(entry, start + (ulong)rebase.TableSlotOffsets[slot], out var offset) || + plan.TableReads[slot].FlatOffset >= tableLength) + return []; + sources[slot] = offset; + } + + return sources; + } + + // Where each memory-loaded base word of the entry lies: its range and byte offset in it. + private static (int Range, int Offset)[] BaseWordProbesOf(Entry entry) + { + var probes = new List<(int Range, int Offset)>(); + for (var range = 0; range < entry.RangeAddresses.Length; range++) + { + for (var offset = 0; offset < entry.RangeLengths[range]; offset += sizeof(uint)) + { + if (entry.BaseWordBuffers![(entry.RangeOffsets[range] + offset) / sizeof(uint)] >= 0) + probes.Add((range, offset)); + } + } + + return probes.ToArray(); + } + + private byte[] _rebaseCurrent = []; + private ulong[] _rebaseMoved = []; + private readonly List<(int Buffer, uint Word)> _rebaseBases = new(); + + // Per recorded range: the memory-based buffer whose window wholly holds it, or NoBase. Such a + // range is read only once the buffer's base is known: it moves with the base. + private static int[] RangeWindowsOf(Entry entry, AddressRebase rebase) + { + var windows = new int[entry.RangeAddresses.Length]; + windows.AsSpan().Fill(NoBase); + for (var index = 0; index < windows.Length; index++) + { + foreach (var buffer in rebase.MemoryBases) + { + var descriptor = entry.Snapshot.Buffers[buffer]; + if (descriptor.Length >= 3 && AddressRebase.TryMovedWindow(descriptor, descriptor[0], out var start, out var size) && + entry.RangeAddresses[index] >= start && entry.RangeAddresses[index] - start + (ulong)entry.RangeLengths[index] <= size) + { + windows[index] = buffer; + break; + } + } + } + + return windows; + } + + // A stale entry whose only changed words are memory-loaded buffer bases, and constants only the + // table reads: the bases are patched into a copy of the snapshot and the table is evaluated + // again for this draw, which reads the constants where the moved bases now point. Every read of + // that evaluation is a word the entry validated unchanged, or lies in the window a patched base + // now covers. The descriptors and specialization are the entry's; learning checks that. + private bool TryRebaseRefresh(Entry entry, ShaderResourcePlan plan, ResourceRuntimeInputs inputs, + ResidentGuestBytesReader residentReader, AddressRebase rebase, out ResourceSnapshot snapshot, + out List<(int Buffer, uint Word)> patches) + { + snapshot = null!; + patches = null!; + if (entry.BaseWordBuffers is not { } map) + return false; + var userData = inputs.UserData; + // The words read here are compared and copied, never kept, so the buffers are reused. + if (_rebaseCurrent.Length < entry.Bytes.Length) + _rebaseCurrent = new byte[Math.Max(entry.Bytes.Length, _rebaseCurrent.Length * 2)]; + if (_rebaseMoved.Length < entry.RangeAddresses.Length) + _rebaseMoved = new ulong[Math.Max(entry.RangeAddresses.Length, _rebaseMoved.Length * 2)]; + var current = _rebaseCurrent; + var moved = _rebaseMoved; + var windowed = entry.RangeWindows ??= RangeWindowsOf(entry, rebase); + for (var index = 0; index < entry.RangeAddresses.Length; index++) + { + if (windowed[index] != NoBase) + continue; + moved[index] = rebase.MoveRead(entry, userData, entry.RangeAddresses[index], entry.RangeLengths[index]); + if (!residentReader(moved[index], current.AsSpan(entry.RangeOffsets[index], entry.RangeLengths[index]), entry.RangeClean[index])) + return false; + } + + // The new base of each buffer whose base word changed, from the ranges read so far. A base + // word inside a moving window would need its own base first; such an entry is not rebased. + var rangeCount = entry.RangeAddresses.Length; + var bases = _rebaseBases; + bases.Clear(); + foreach (var (range, rangeOffset) in entry.BaseWordProbes ??= BaseWordProbesOf(entry)) + { + if (windowed[range] != NoBase) + return false; + var offset = entry.RangeOffsets[range] + rangeOffset; + var now = System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(current.AsSpan(offset)); + if (now != System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(entry.Bytes.AsSpan(offset))) + bases.Add((map[offset / sizeof(uint)], now)); + } + + for (var index = 0; index < rangeCount; index++) + { + var buffer = windowed[index]; + if (buffer == NoBase) + continue; + var oldBase = entry.Snapshot.Buffers[buffer][0]; + var newBase = oldBase; + foreach (var (changed, word) in bases) + if (changed == buffer) + newBase = word; + moved[index] = entry.RangeAddresses[index] - oldBase + newBase; + if (!residentReader(moved[index], current.AsSpan(entry.RangeOffsets[index], entry.RangeLengths[index]), entry.RangeClean[index])) + return false; + } + + List<(int Buffer, uint Word)>? found = null; + // A changed constant outside the moved windows is read by a slot the copy cannot place. + var constantsOutsideWindows = false; + for (var index = 0; index < rangeCount; index++) + { + // Most ranges did not change at all; only a changed one is looked at word by word. + if (current.AsSpan(entry.RangeOffsets[index], entry.RangeLengths[index]) + .SequenceEqual(entry.Bytes.AsSpan(entry.RangeOffsets[index], entry.RangeLengths[index]))) + continue; + for (var offset = entry.RangeOffsets[index]; offset < entry.RangeOffsets[index] + entry.RangeLengths[index]; offset += sizeof(uint)) + { + var word = offset / sizeof(uint); + var now = System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(current.AsSpan(offset)); + if (now == System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(entry.Bytes.AsSpan(offset))) + continue; + var buffer = map[word]; + if (buffer >= 0) + { + if (rebase.IsRejectedMemoryBase(buffer)) + return false; + (found ??= new()).Add((buffer, now)); + } + else if (!entry.WordTableOnly[word]) + return false; + else + constantsOutsideWindows |= windowed[index] == NoBase; + } + } + + if (found is null) + return false; + var patched = rebase.PatchMemoryBases(rebase.Apply(WithCurrentUserData(entry.Snapshot, userData), userData), found); + // A changed constant outside the windows is still copied when every slot is a direct load: + // each slot's word is then the dword at its own address, which this draw has just read. + var copyDirect = constantsOutsideWindows && CanCopyDirect(rebase); + if (plan.TableReads.Count != 0 && (!constantsOutsideWindows || copyDirect) && !rebase.TableSlotsThroughBases) + { + // The patch cloned the table when it moved a base slot; otherwise it is the entry's. + var table = ReferenceEquals(patched.FlattenedResourceTable, entry.Snapshot.FlattenedResourceTable) + ? (uint[])patched.FlattenedResourceTable.Clone() + : patched.FlattenedResourceTable; + // The copy writes nothing unless every slot can be placed. + if (copyDirect + ? TryCopyDirectTable(entry, plan, rebase, current, moved, userData, table) + : TryCopyTable(entry, plan, rebase, current, table)) + { + snapshot = new ResourceSnapshot + { + Buffers = patched.Buffers, + Images = patched.Images, + Samplers = patched.Samplers, + FlattenedResourceTable = table, + UserData = patched.UserData, + DeviceAddressRanges = patched.DeviceAddressRanges, + }; + patches = found; + return true; + } + } + + if (plan.TableReads.Count != 0) + { + if (!entry.TableRefreshable) + return false; + var windows = new List<(ulong Start, ulong Size)>(found.Count); + foreach (var (buffer, baseWord) in found) + if (AddressRebase.TryMovedWindow(patched.Buffers[buffer], baseWord, out var start, out var size)) + windows.Add((start, size)); + + var recorder = TakeRecorder(); + try + { + var recording = new ResourceRuntimeInputs + { + UserData = userData, + ShaderBase = inputs.ShaderBase, + ReadMemory = recorder.Wrap(inputs.ReadMemory, clean: false), + ReadCleanMemory = recorder.Wrap(inputs.ReadCleanMemory, clean: true), + ReadResidentMemory = recorder.WrapResident(inputs.ReadResidentMemory), + ReadsClean = inputs.ReadsClean, + ComputeState = inputs.ComputeState, + }; + var cachedTable = entry.Snapshot.FlattenedResourceTable; + ReadingTable = true; + bool evaluated; + uint[] table; + try + { + evaluated = ResourceMaterializer.TryEvaluateTable(plan, recording, out table); + } + finally + { + ReadingTable = false; + } + + if (!evaluated || recorder.Failed || table.Length != cachedTable.Length) + return false; + + foreach (var (address, word, _, _) in recorder.Reads) + { + var inWindow = false; + foreach (var (start, size) in windows) + inWindow |= address >= start && address - start < size; + if (inWindow) + continue; + if (!TryFindMovedWord(entry, moved, address, out var offset) || + System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(current.AsSpan(offset, sizeof(uint))) != word) + return false; + } + + // The written device-address slots come from reads validated unchanged above. + foreach (var slot in plan.WrittenRangeSlotByHandle.Values) + cachedTable.AsSpan((int)slot, ShaderResourcePlan.WrittenRangeDwordCount).CopyTo(table.AsSpan((int)slot)); + patched = new ResourceSnapshot + { + Buffers = patched.Buffers, + Images = patched.Images, + Samplers = patched.Samplers, + FlattenedResourceTable = table, + UserData = patched.UserData, + DeviceAddressRanges = patched.DeviceAddressRanges, + }; + } + finally + { + ReturnRecorder(recorder); + } + } + + snapshot = patched; + patches = found; + return true; + } + + private bool Validate(Entry entry, ResidentGuestBytesReader residentReader, AddressRebase rebase, IReadOnlyList userData, + out bool unreadable) { unreadable = false; + // A memory-loaded base usually moved when anything did: its few bytes are checked before + // ranges that can span kilobytes. The full comparison below still decides a match. + if (entry.BaseWordBuffers is not null) + { + Span word = stackalloc byte[sizeof(uint)]; + foreach (var (range, offset) in entry.BaseWordProbes ??= BaseWordProbesOf(entry)) + { + var address = rebase.MoveRead(entry, userData, entry.RangeAddresses[range], entry.RangeLengths[range]) + (ulong)offset; + if (residentReader(address, word, entry.RangeClean[range]) && + !word.SequenceEqual(entry.Bytes.AsSpan(entry.RangeOffsets[range] + offset, sizeof(uint)))) + return false; + } + } + for (var index = 0; index < entry.RangeAddresses.Length; index++) { var length = entry.RangeLengths[index]; if (_scratch.Length < length) _scratch = new byte[Math.Max(length, _scratch.Length * 2)]; var current = _scratch.AsSpan(0, length); - if (!residentReader(entry.RangeAddresses[index], current, entry.RangeClean[index])) + var address = rebase.MoveRead(entry, userData, entry.RangeAddresses[index], length); + if (!residentReader(address, current, entry.RangeClean[index])) { unreadable = true; return false; @@ -385,6 +1716,8 @@ public sealed class ResourceMaterializationCache { public required ShaderResourcePlan Plan { get; init; } public required uint[] UserData { get; init; } + // The user data words the evaluator read, one bit per word. + public required ulong[] UsedUserData { get; init; } public required ulong ShaderBase { get; init; } public required ComputeSelectorState? ComputeState { get; init; } public required ulong[] RangeAddresses { get; init; } @@ -398,20 +1731,59 @@ public sealed class ResourceMaterializationCache public required byte[] Bytes { get; init; } public required ResourceSnapshot Snapshot { get; init; } public required ResourceSpecialization Specialization { get; init; } + // Per recorded dword: the buffer whose memory-loaded base dword it is, or a negative value. + public int[]? BaseWordBuffers { get; init; } + // Filled on the first rebased refresh of the entry; see RangeWindowsOf and TableCopySourcesOf. + public int[]? RangeWindows { get; set; } + public int[]? TableCopySources { get; set; } + public (int Range, int Offset)[]? BaseWordProbes { get; set; } public Entry? Next { get; set; } - public bool Matches(ShaderResourcePlan plan, ResourceRuntimeInputs inputs) + // A rebased word is an address the hit patches in, so it does not distinguish entries. + public bool Matches(ShaderResourcePlan plan, ResourceRuntimeInputs inputs, ulong[] rebased) { if (!ReferenceEquals(Plan, plan) || ShaderBase != inputs.ShaderBase || !Nullable.Equals(ComputeState, inputs.ComputeState) || UserData.Length != inputs.UserData.Count) return false; for (var index = 0; index < UserData.Length; index++) - if (UserData[index] != inputs.UserData[index]) + if (IsUsed(UsedUserData, index) && !IsUsed(rebased, index) && UserData[index] != inputs.UserData[index]) return false; return true; } } + // A user data list that notes which words are read by index. Copying it out (the snapshot's + // own copy) goes through the enumerator and is not a read of the evaluator. + private sealed class UserDataRecorder : IReadOnlyList + { + private IReadOnlyList _inner = []; + private ulong[] _used = []; + + public void Begin(IReadOnlyList inner) + { + _inner = inner; + _used = new ulong[(inner.Count + 63) >> 6]; + } + + // The recorded set; the next Begin allocates a fresh one, so the entry keeps this array. + public ulong[] TakeUsed() => _used; + + public int Count => _inner.Count; + + public uint this[int index] + { + get + { + _used[index >> 6] |= 1UL << (index & 63); + return _inner[index]; + } + } + + public IEnumerator GetEnumerator() => _inner.GetEnumerator(); + + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator(); + } + private sealed class ReadRecorder { private readonly List<(ulong Address, uint Word, bool Clean, bool Table)> _reads = new(); @@ -451,6 +1823,14 @@ public sealed class ResourceMaterializationCache public bool Failed { get; private set; } + private readonly UserDataRecorder _userData = new(); + + public IReadOnlyList WrapUserData(IReadOnlyList inner) + { + _userData.Begin(inner); + return _userData; + } + public List<(ulong Address, uint Word, bool Clean, bool Table)> Reads => _reads; public Action TablePhase { get; } @@ -499,9 +1879,14 @@ public sealed class ResourceMaterializationCache } public Entry Build(ShaderResourcePlan plan, ResourceRuntimeInputs inputs, ResourceSnapshot snapshot, - ResourceSpecialization specialization) + ResourceSpecialization specialization, IReadOnlyList memoryBases) { - _reads.Sort((left, right) => left.Address.CompareTo(right.Address)); + // Reads mostly arrive in address order; sorting a sorted list was ~5% of a draw-heavy render thread. + var ordered = true; + for (var index = 1; ordered && index < _reads.Count; index++) + ordered = _reads[index - 1].Address <= _reads[index].Address; + if (!ordered) + _reads.Sort((left, right) => left.Address.CompareTo(right.Address)); // Count unique words and contiguous ranges first, then fill the // retained arrays directly. Temporary lists previously duplicated // every recorded byte and range on each cache miss. @@ -558,8 +1943,10 @@ public sealed class ResourceMaterializationCache userData[index] = inputs.UserData[index]; return new Entry { + BaseWordBuffers = memoryBases.Count == 0 ? null : MapBaseWords(snapshot, memoryBases, bytes), Plan = plan, UserData = userData, + UsedUserData = _userData.TakeUsed(), ShaderBase = inputs.ShaderBase, ComputeState = inputs.ComputeState, RangeAddresses = addresses, @@ -574,5 +1961,46 @@ public sealed class ResourceMaterializationCache Specialization = specialization, }; } + + // The recorded word each memory-loaded base dword came from: the only recorded word holding + // that value. A word two bases claim names neither. Learning checks the guess. + private static int[]? MapBaseWords(ResourceSnapshot snapshot, IReadOnlyList memoryBases, byte[] bytes) + { + int[]? map = null; + var wordCount = bytes.Length / sizeof(uint); + foreach (var buffer in memoryBases) + { + if (buffer >= snapshot.Buffers.Length || snapshot.Buffers[buffer].Length < 2) + continue; + var value = snapshot.Buffers[buffer][0]; + var match = -1; + for (var word = 0; word < wordCount; word++) + { + if (System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(bytes.AsSpan(word * sizeof(uint))) != value) + continue; + if (match >= 0) + { + match = -1; + break; + } + match = word; + } + + if (match < 0) + continue; + if (map is null) + { + map = new int[wordCount]; + map.AsSpan().Fill(NoBase); + } + + map[match] = map[match] == NoBase ? buffer : SharedBase; + } + + return map; + } } + + private const int NoBase = -1; + private const int SharedBase = -2; } diff --git a/src/SharpEmu.ShaderCompiler/ShaderCompileRequest.cs b/src/SharpEmu.ShaderCompiler/ShaderCompileRequest.cs index 2e64c02..c6c97e2 100644 --- a/src/SharpEmu.ShaderCompiler/ShaderCompileRequest.cs +++ b/src/SharpEmu.ShaderCompiler/ShaderCompileRequest.cs @@ -162,6 +162,8 @@ public sealed class ShaderCompileRequest public IReadOnlyDictionary WrittenRangeSlotByMemoryIndex { get; } public uint WaveSize { get; init; } = 32; + // True when the pipeline runs on a 64-lane host subgroup, so one subgroup is one guest Wave64. + public bool HostWave64Supported { get; init; } public bool EnableExecGuardElision { get; init; } = true; public uint ScratchDwords { get; init; } public bool EnableGraphicsSubgroupOperations { get; init; } = true; diff --git a/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/Registers/RegisterBanksSnapshotTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/Registers/RegisterBanksSnapshotTests.cs new file mode 100644 index 0000000..18ccd44 --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Gpu/GpuCommands/Registers/RegisterBanksSnapshotTests.cs @@ -0,0 +1,236 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Libs.Gpu.GpuCommands; +using SharpEmu.Libs.Gpu.GpuCommands.Registers; +using Xunit; +using static SharpEmu.Libs.Gpu.GpuCommands.Registers.ContextRegisterOffset; +using static SharpEmu.Libs.Gpu.GpuCommands.Registers.ShaderRegisterOffset; +using static SharpEmu.Libs.Gpu.GpuCommands.Registers.UserConfigRegisterOffset; + +namespace SharpEmu.Libs.Tests.Gpu.GpuCommands.Registers; + +// A snapshot shares the live banks' parts until the live banks write them; it never changes. +public sealed class RegisterBanksSnapshotTests +{ + private const ulong PacketAddress = 0x1_0000_1000; + + private static RegisterBanks NewBanks() => new(static message => new InvalidOperationException(message)); + + private static PacketContext Packet(uint opcode, int valueCount) => + new(PacketHeader.Make((uint)valueCount + 2, opcode), PacketAddress, 0, 100, 100); + + private static void WriteContext(RegisterBanks banks, uint offset, params uint[] values) => + RegisterWriteTable.WriteContextPacket(banks, Packet(PacketOpcode.SetContextRegister, values.Length), offset, values); + + private static void WriteShader(RegisterBanks banks, uint offset, params uint[] values) => + RegisterWriteTable.WriteShaderPacket(banks, Packet(PacketOpcode.SetShaderRegister, values.Length), offset, values); + + private static void WriteUserConfig(RegisterBanks banks, uint offset, params uint[] values) => + RegisterWriteTable.WriteUserConfigPacket(banks, Packet(PacketOpcode.SetUserConfigRegister, values.Length), offset, values); + + [Fact] + public void WritesAfterASnapshotLeaveItUnchanged() + { + var banks = NewBanks(); + WriteContext(banks, DbDepthControl, 0x0000_0006); + WriteShader(banks, SpiShaderPgmLoPs, 0x1000); + WriteShader(banks, SpiShaderUserDataPs0, 11, 12); + WriteShader(banks, SpiShaderPgmLoEs, 0x2000); + WriteShader(banks, ComputePgmLo, 0x3000); + WriteUserConfig(banks, VgtPrimitiveType, 4); + var snapshot = banks.Snapshot(); + + WriteContext(banks, DbDepthControl, 0x0000_0000); + WriteShader(banks, SpiShaderPgmLoPs, 0x5000); + WriteShader(banks, SpiShaderUserDataPs0, 21, 22); + WriteShader(banks, SpiShaderPgmLoEs, 0x6000); + WriteShader(banks, ComputePgmLo, 0x7000); + WriteUserConfig(banks, VgtPrimitiveType, 6); + + Assert.True(snapshot.Context.DepthTarget.DepthTestEnabled); + Assert.False(banks.Context.DepthTarget.DepthTestEnabled); + Assert.Equal(0x1000ul << 8, snapshot.Shader.Pixel.Address); + Assert.Equal(0x5000ul << 8, banks.Shader.Pixel.Address); + Assert.Equal([11u, 12u], snapshot.Shader.Pixel.UserScalars.Values[..2]); + Assert.Equal([21u, 22u], banks.Shader.Pixel.UserScalars.Values[..2]); + Assert.Equal(0x2000ul << 8, snapshot.Shader.Vertex.ExportAddress); + Assert.Equal(0x6000ul << 8, banks.Shader.Vertex.ExportAddress); + Assert.Equal(0x3000ul << 8, snapshot.Shader.Compute.Address); + Assert.Equal(0x7000ul << 8, banks.Shader.Compute.Address); + Assert.Equal(4u, snapshot.UserConfig.PrimitiveType); + Assert.Equal(6u, banks.UserConfig.PrimitiveType); + } + + [Fact] + public void AWriteCopiesOnlyThePartItStoresInto() + { + var banks = NewBanks(); + var snapshot = banks.Snapshot(); + + WriteShader(banks, SpiShaderUserDataPs0, 7); + + Assert.NotSame(snapshot.Shader.Pixel, banks.Shader.Pixel); + Assert.Same(snapshot.Shader.Vertex, banks.Shader.Vertex); + Assert.Same(snapshot.Shader.Compute, banks.Shader.Compute); + Assert.Same(snapshot.Context, banks.Context); + Assert.Same(snapshot.UserConfig, banks.UserConfig); + } + + [Fact] + public void AUserScalarWriteCopiesOnlyItsSet() + { + var banks = NewBanks(); + WriteShader(banks, SpiShaderUserDataGs0, 1); + WriteShader(banks, SpiShaderUserDataHs0, 2); + var snapshot = banks.Snapshot(); + + WriteShader(banks, SpiShaderUserDataGs0, 3); + WriteShader(banks, SpiShaderPgmLoPs, 0x4000); + + Assert.NotSame(snapshot.Shader.Vertex, banks.Shader.Vertex); + Assert.NotSame(snapshot.Shader.Vertex.GeometryUserScalars, banks.Shader.Vertex.GeometryUserScalars); + Assert.Same(snapshot.Shader.Vertex.HullUserScalars, banks.Shader.Vertex.HullUserScalars); + Assert.Equal(1u, snapshot.Shader.Vertex.GeometryUserScalars.Values[0]); + Assert.Equal(3u, banks.Shader.Vertex.GeometryUserScalars.Values[0]); + Assert.NotSame(snapshot.Shader.Pixel, banks.Shader.Pixel); + Assert.Same(snapshot.Shader.Pixel.UserScalars, banks.Shader.Pixel.UserScalars); + + // The set copied after the snapshot is the live banks' own: a later write does not copy it again. + var geometry = banks.Shader.Vertex.GeometryUserScalars; + WriteShader(banks, SpiShaderUserDataGs0, 4); + Assert.Same(geometry, banks.Shader.Vertex.GeometryUserScalars); + Assert.Equal(1u, snapshot.Shader.Vertex.GeometryUserScalars.Values[0]); + } + + [Fact] + public void ContextStateOperationsAfterASnapshotLeaveItUnchanged() + { + var banks = NewBanks(); + WriteContext(banks, DbDepthControl, 0x0000_0006); + var snapshot = banks.Snapshot(); + + banks.ApplyContextState(ContextStateOperation.PushClear); + WriteContext(banks, DbDepthControl, 0x0000_0000); + banks.ApplyContextState(ContextStateOperation.Pop); + WriteContext(banks, DbDepthControl, 0x0000_0000); + + Assert.True(snapshot.Context.DepthTarget.DepthTestEnabled); + Assert.False(banks.Context.DepthTarget.DepthTestEnabled); + } + + // Every context register, through the table path and the packet path, against a snapshot + // taken after a first write: the context's nested parts are copied only by their writers. + [Fact] + public void EveryContextRegisterWriteLeavesASnapshotUnchanged() + { + var changed = 0; + for (var offset = 0u; offset < RegisterBankLayout.ContextRegisterCount; offset++) + { + foreach (var packet in new[] { false, true }) + { + var banks = NewBanks(); + WriteContext(banks, DbDepthControl, 0x0000_0006); + var snapshot = banks.Snapshot(); + var before = Describe(snapshot.Context); + foreach (var value in new[] { 0xFFFF_FFFFu, 0x1234_5678u }) + { + try + { + if (packet) + WriteContext(banks, offset, value); + else + RegisterWriteTable.WriteContextEntry(banks, offset, value, PacketAddress); + } + catch (InvalidOperationException) + { + // A register whose writer refuses a single value. + } + } + + Assert.True(before == Describe(snapshot.Context), $"offset=0x{offset:X} packet={packet}"); + if (Describe(banks.Context) != before) + changed++; + } + } + + // The description sees the writes: most registers store something. + Assert.True(changed > 400, $"changed={changed}"); + } + + private static string Describe(object? value) + { + var text = new System.Text.StringBuilder(); + Describe(value, text); + return text.ToString(); + } + + private static void Describe(object? value, System.Text.StringBuilder text) + { + switch (value) + { + case null: + text.Append("null;"); + return; + case System.Collections.IDictionary dictionary: + foreach (var key in dictionary.Keys.Cast().OrderBy(key => key)) + text.Append(key).Append('=').Append(dictionary[key]).Append(','); + text.Append(';'); + return; + case Array array: + text.Append('['); + foreach (var element in array) + Describe(element, text); + text.Append(']'); + return; + } + + var type = value.GetType(); + if (type.IsPrimitive || type.IsEnum || value is string) + { + text.Append(value).Append(';'); + return; + } + + text.Append('{'); + foreach (var field in type.GetFields(System.Reflection.BindingFlags.Instance | + System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.NonPublic)) + { + text.Append(field.Name).Append(':'); + Describe(field.GetValue(value), text); + } + + text.Append('}'); + } + + [Fact] + public void AContextWriteCopiesOnlyTheNestedPartsItStoresInto() + { + var banks = NewBanks(); + var snapshot = banks.Snapshot(); + + WriteContext(banks, DbDepthControl, 0x0000_0006); + WriteContext(banks, PaClVteCntl, 0x43F); + + Assert.NotSame(snapshot.Context, banks.Context); + Assert.NotSame(snapshot.Context.ScreenViewport, banks.Context.ScreenViewport); + Assert.Same(snapshot.Context.ScreenViewport.Viewports, banks.Context.ScreenViewport.Viewports); + Assert.Same(snapshot.Context.ShaderInterface, banks.Context.ShaderInterface); + Assert.Same(snapshot.Context.ColorTargets, banks.Context.ColorTargets); + Assert.Same(snapshot.Context.UnmodeledTableRegisters, banks.Context.UnmodeledTableRegisters); + } + + [Fact] + public void ASecondSnapshotSharesAgainUntilTheNextWrite() + { + var banks = NewBanks(); + var first = banks.Snapshot(); + WriteContext(banks, DbDepthControl, 0x0000_0006); + var second = banks.Snapshot(); + WriteContext(banks, DbDepthControl, 0x0000_0000); + + Assert.False(first.Context.DepthTarget.DepthTestEnabled); + Assert.True(second.Context.DepthTarget.DepthTestEnabled); + Assert.False(banks.Context.DepthTarget.DepthTestEnabled); + } +} diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Images/GuestImageCacheTests.Metadata.cs b/tests/SharpEmu.Libs.Tests/Gpu/Images/GuestImageCacheTests.Metadata.cs index 2eb5a41..233a83e 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Images/GuestImageCacheTests.Metadata.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Images/GuestImageCacheTests.Metadata.cs @@ -99,6 +99,24 @@ public sealed partial class GuestImageCacheTests harness.Shutdown(); } + // A fill seen before its target leaves its range known as DCC metadata, so the next fill of it + // runs on the host and the range stays CPU-readable. + [Fact] + public void APendingDccFill_MarksItsRangeAsDccMetadata() + { + if (!GatePrerequisites.Ready(_vulkan)) return; + using var fatal = new FatalScope(); + using var harness = new CacheHarness(_vulkan); + var address = harness.MapBacked(0x20000, ReadWrite); + var metadata = address + 0x10000; + Assert.False(harness.Images.OverlapsDccMetadata(metadata, 0x100)); + + Assert.False(harness.Images.TryAbsorbDccFill(metadata, 0x100, 0x00000000u)); + + Assert.True(harness.Images.OverlapsDccMetadata(metadata + 0x80, 0x10)); + Assert.False(harness.Images.OverlapsDccMetadata(metadata + 0x100, 0x10)); + } + [Fact] public void DccFills_RemainGuestProducersUntilDescriptorDiscovery() { @@ -451,6 +469,35 @@ public sealed partial class GuestImageCacheTests harness.Shutdown(); } + // The pages inside a filled range are clean after it; a page it shares with another GPU-written + // range keeps that range's state. + [Fact] + public void DirtyDccFill_KeepsASharedBoundaryPageDirty() + { + if (!GatePrerequisites.Ready(_vulkan)) return; + using var harness = new CacheHarness(_vulkan); + const ulong Page = SharpEmu.HLE.GpuMemory.TrackerLayout.PageBytes; + var memory = harness.MapBacked(8 * Page, ReadWrite); + var metadata = memory + Page + Page / 2; + var size = 3 * Page; + var neighbour = metadata - 0x100; + harness.Images.RegisterDccMetadataForTest(metadata, size); + harness.Worker.Run(() => + { + _ = harness.Cache.ObtainBuffer(neighbour, 0x100, isWritten: true); + _ = harness.Cache.ObtainBuffer(metadata, size, isWritten: true); + }); + + harness.Worker.Run(() => harness.Cache.FillBuffer(metadata, size, 0x00000000, isGds: false)); + + Assert.False(harness.Cache.HasGpuDirtyBytes(metadata, size)); + Assert.False(harness.Cache.HasGpuDirtyPages(memory + 2 * Page, 2 * Page)); + Assert.False(harness.Cache.HasGpuDirtyPages(memory + 4 * Page, Page)); + Assert.True(harness.Cache.HasGpuDirtyPages(memory + Page, Page)); + Assert.True(harness.Cache.HasGpuDirtyBytes(neighbour, 0x100)); + harness.Shutdown(); + } + [Fact] public void CleanDccFill_WritesBothCopiesWithoutMarkingTheGpuCopyNewer() { diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Pipelines/ShaderPipelineCacheTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Pipelines/ShaderPipelineCacheTests.cs index 4d2a98b..e16ad15 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Pipelines/ShaderPipelineCacheTests.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Pipelines/ShaderPipelineCacheTests.cs @@ -266,6 +266,9 @@ public sealed class ShaderPipelineCacheTests : IDisposable executor.DrawAuto(1, Banks(), Auto(3)); executor.DrawAuto(2, Banks(), Auto(3)); executor.DrawAuto(3, Banks(PrimitiveTriangleStrip), Auto(4)); + // The lookup key is reused across draws: after another pipeline, the first is found again. + executor.DrawAuto(4, Banks(), Auto(3)); + executor.DrawAuto(5, Banks(PrimitiveTriangleStrip), Auto(4)); Assert.Equal(2, guest.Host.GraphicsPipelines.Count); Assert.Equal(2, cache.GraphicsPipelineCount); diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorComputeTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorComputeTests.cs index b05928c..9d09711 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorComputeTests.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorComputeTests.cs @@ -279,9 +279,25 @@ public sealed class RenderExecutorComputeTests : IDisposable Assert.Contains($"clear_image {MetadataAddress:X} 1000 {ClearValue:X8}", _host.Calls); Assert.Contains($"absorb_dcc {MetadataAddress:X} 1000 {ClearValue:X8}", _host.Calls); + Assert.Contains($"fill_dcc {MetadataAddress:X} 1000 {ClearValue:X8}", _host.Calls); AssertDispatched(4, 1, 1); } + // A pending surface's metadata the host knows is filled there: as a dispatch it would be GPU-written + // and read back at every later bind. + [Fact] + public void PendingMetadataTheHostKnows_IsFilledOnTheHost() + { + _host.HostFillableDcc.Add(MetadataAddress); + ConfigureClearKernel(256); + _executor.Dispatch(1, Banks(), 256, 1, 1, ClearInitiator); + + Assert.Contains($"absorb_dcc {MetadataAddress:X} 1000 {ClearValue:X8}", _host.Calls); + Assert.Contains($"fill_dcc {MetadataAddress:X} 1000 {ClearValue:X8}", _host.Calls); + AssertNotDispatched(); + Assert.Contains("reset_bindings", _host.Calls); + } + [Fact] public void RegisteredDccAddress_ConsumesTheFillAndRecordsNothing() { diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorDrawTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorDrawTests.cs index aa772f4..66e3d5d 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorDrawTests.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorDrawTests.cs @@ -22,9 +22,21 @@ public sealed class RenderExecutorDrawTests : IDisposable private readonly RenderExecutor _executor; private readonly FatalScope _fatal = new(); - public RenderExecutorDrawTests() => _executor = new RenderExecutor(_host, _pipelines); + private readonly InputsCheckingPrefetch _prefetch; - public void Dispose() => _fatal.Dispose(); + public RenderExecutorDrawTests() + { + _executor = new RenderExecutor(_host, _pipelines); + _prefetch = new InputsCheckingPrefetch(_host); + _executor.Prefetch = _prefetch; + } + + // Every draw of these tests also checks that its program inputs follow from its registers. + public void Dispose() + { + _fatal.Dispose(); + Assert.Empty(_prefetch.Mismatches); + } private void AssertOrder(params string[] prefixes) { diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorPrefetchTests.cs b/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorPrefetchTests.cs new file mode 100644 index 0000000..28377d1 --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorPrefetchTests.cs @@ -0,0 +1,87 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.Libs.Gpu.GpuCommands.Registers; +using SharpEmu.Libs.Gpu.Rendering; +using SharpEmu.Libs.Tests.Gpu.Scheduling; +using Xunit; +using static SharpEmu.Libs.Tests.Gpu.Rendering.RenderExecutorFixtures; + +namespace SharpEmu.Libs.Tests.Gpu.Rendering; + +// A draw takes the programs a prefetch resolved from its banks, or resolves them itself. +[Collection(SchedulingStateCollection.Name)] +public sealed class RenderExecutorPrefetchTests : IDisposable +{ + private readonly RecordingRenderHost _host = new(); + private readonly FakePipelineProvider _pipelines = new(); + private readonly RenderExecutor _executor; + private readonly FatalScope _fatal = new(); + + public RenderExecutorPrefetchTests() => _executor = new RenderExecutor(_host, _pipelines); + + public void Dispose() => _fatal.Dispose(); + + // Resolves ahead with the executor, as a worker would, and hands the result back once. + private sealed class AheadPrefetch(RenderExecutor executor, IRenderHost host) : IProgramPrefetch + { + private RegisterBanks? _banks; + private GraphicsProgramInputs _inputs; + private GraphicsPrograms? _programs; + + public int Taken { get; private set; } + + public int Declined { get; private set; } + + public void Resolve(RegisterBanks banks) + { + _banks = banks; + _inputs = RenderExecutor.ProgramInputsOf(banks, host.FormatSupport, host.Fatal); + _programs = executor.ResolveGraphicsPrograms(banks, in _inputs); + } + + public bool TryTakeGraphics(RegisterBanks banks, in GraphicsProgramInputs inputs, out GraphicsPrograms programs) + { + if (_programs is not null && ReferenceEquals(banks, _banks) && inputs.Equals(_inputs)) + { + programs = _programs; + _programs = null; + Taken++; + return true; + } + + programs = null!; + Declined++; + return false; + } + } + + [Fact] + public void ADrawTakesTheProgramsResolvedAheadFromItsBanks() + { + var prefetch = new AheadPrefetch(_executor, _host); + _executor.Prefetch = prefetch; + var banks = Banks(); + prefetch.Resolve(banks); + var resolutions = _pipelines.Calls.Count(call => call.StartsWith("get_graphics_programs", StringComparison.Ordinal)); + + _executor.DrawAuto(1, banks, Auto(3)); + + Assert.Equal(1, prefetch.Taken); + Assert.Equal(resolutions, _pipelines.Calls.Count(call => call.StartsWith("get_graphics_programs", StringComparison.Ordinal))); + Assert.Contains(_host.Calls, call => call.StartsWith("draw ", StringComparison.Ordinal)); + } + + [Fact] + public void ADrawWithOtherBanksResolvesItsOwnPrograms() + { + var prefetch = new AheadPrefetch(_executor, _host); + _executor.Prefetch = prefetch; + prefetch.Resolve(Banks()); + + _executor.DrawAuto(1, Banks(), Auto(3)); + + Assert.Equal((0, 1), (prefetch.Taken, prefetch.Declined)); + Assert.Equal(2, _pipelines.Calls.Count(call => call.StartsWith("get_graphics_programs", StringComparison.Ordinal))); + } +} diff --git a/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorTestSupport.cs b/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorTestSupport.cs index c50241a..cfd6432 100644 --- a/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorTestSupport.cs +++ b/tests/SharpEmu.Libs.Tests/Gpu/Rendering/RenderExecutorTestSupport.cs @@ -108,6 +108,9 @@ internal sealed class RecordingRenderHost : IRenderHost public HashSet RegisteredDcc { get; } = new(); + // Metadata the host knows and fills itself although its surface is not registered yet. + public HashSet HostFillableDcc { get; } = new(); + public HashSet ClearableImages { get; } = new(); public Func? ClampOverride { get; set; } @@ -391,7 +394,7 @@ internal sealed class RecordingRenderHost : IRenderHost public bool TryFillDccMetadata(ulong address, ulong size, uint fillValue) { Calls.Add($"fill_dcc {address:X} {size:X} {fillValue:X8}"); - return RegisteredDcc.Contains(address); + return RegisteredDcc.Contains(address) || HostFillableDcc.Contains(address); } public bool HostCopyAccepted { get; set; } = true; @@ -405,6 +408,33 @@ internal sealed class RecordingRenderHost : IRenderHost public Exception Fatal(string message) => new RenderExecutorFatalException(message); } +// Takes nothing, but checks that the inputs a draw resolves its programs with are the ones its +// register banks alone give: a prefetch computed ahead of the draw would use those. +internal sealed class InputsCheckingPrefetch(IRenderHost host) : IProgramPrefetch +{ + public List Mismatches { get; } = new(); + + public int Checks { get; private set; } + + public bool TryTakeGraphics(RegisterBanks banks, in GraphicsProgramInputs inputs, out GraphicsPrograms programs) + { + Checks++; + var expected = RenderExecutor.ProgramInputsOf(banks, host.FormatSupport, host.Fatal); + if (!expected.Equals(inputs)) + { + Mismatches.Add( + $"draw pixelActive={inputs.PixelActive}/{expected.PixelActive} depthBound={inputs.DepthBound}/{expected.DepthBound} " + + $"mapping=[{Describe(inputs.Mapping)}]/[{Describe(expected.Mapping)}]"); + } + + programs = null!; + return false; + } + + private static string Describe(ReadOnlySpan mapping) => + string.Join(",", mapping.ToArray().Select(map => map.Packed)); +} + // Hands back the configured programs and records every pipeline request. internal sealed class FakePipelineProvider : IShaderPipelineProvider { diff --git a/tests/SharpEmu.Libs.Tests/VideoOut/PresenterImageBindingTests.cs b/tests/SharpEmu.Libs.Tests/VideoOut/PresenterImageBindingTests.cs index 5592b49..c604ec1 100644 --- a/tests/SharpEmu.Libs.Tests/VideoOut/PresenterImageBindingTests.cs +++ b/tests/SharpEmu.Libs.Tests/VideoOut/PresenterImageBindingTests.cs @@ -92,6 +92,41 @@ public sealed class PresenterImageBindingTests : IClassFixture + { + var resource = new ImageResource + { + ResourceClass = PlanImageResourceClass.Sampled, + NumericClass = ImageNumericClass.Float, + Dimension = ImageDimension.Dim2D, + Read = true, + }; + var program = new ShaderProgramInfo(); + var words = RegisterWords.Texture(address, GuestPixelFormat.Bits32Float, 48, 24); + var first = (ResourceSlotIdentifier)GetFieldValue(presenter.InvokeMethod("ResolveImageBinding", resource, words, program, 0)!, "ImageIdentifier"); + var second = (ResourceSlotIdentifier)GetFieldValue(presenter.InvokeMethod("ResolveImageBinding", resource, words, program, 0)!, "ImageIdentifier"); + Assert.Equal(first, second); + + // A removed image is not served again: the binding looks the descriptor up afresh. + var generation = harness.Images.LookupGeneration; + typeof(GuestImageCache).GetMethod("DeleteImage", InstanceMembers)!.Invoke(harness.Images, [first]); + Assert.NotEqual(generation, harness.Images.LookupGeneration); + var third = (ResourceSlotIdentifier)GetFieldValue(presenter.InvokeMethod("ResolveImageBinding", resource, words, program, 0)!, "ImageIdentifier"); + Assert.True(harness.Images.TryGetImage(third, out var image) && image.Registered); + presenter.RenderHost.ResetBindings(); + }); + harness.Shutdown(); + } + [Theory] [InlineData(true, 1u, false)] [InlineData(true, 2u, true)] diff --git a/tests/SharpEmu.ShaderCompiler.Tests/Gen5WaveLoweringTests.cs b/tests/SharpEmu.ShaderCompiler.Tests/Gen5WaveLoweringTests.cs new file mode 100644 index 0000000..d1f598e --- /dev/null +++ b/tests/SharpEmu.ShaderCompiler.Tests/Gen5WaveLoweringTests.cs @@ -0,0 +1,250 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Buffers.Binary; +using System.Text; +using SharpEmu.ShaderCompiler.Vulkan; +using Xunit; +using static SharpEmu.ShaderCompiler.Tests.Resources.ResourceTestProgram; + +namespace SharpEmu.ShaderCompiler.Tests; + +// Checks the SPIR-V each wave lowering (NativeWave32, NativeWave64, EmulatedWave64) produces for the +// same guest programs: which subgroup instructions appear, how a ballot is split into a 64-bit mask, +// and whether the workgroup exchange / rendezvous machinery exists. +public sealed class Gen5WaveLoweringTests +{ + public enum Lowering { Wave32, NativeWave64, EmulatedWave64 } + + private sealed record Module( + IReadOnlyList<(SpirvOp Op, uint[] Operands)> Instructions, + IReadOnlyList Names) + { + public int Count(SpirvOp op) => Instructions.Count(instruction => instruction.Op == op); + + // CompositeExtract of a ballot result: its literal index is the last operand. + public IReadOnlyList BallotExtractIndices() + { + var ballots = Instructions.Where(i => i.Op == SpirvOp.GroupNonUniformBallot).Select(i => i.Operands[1]).ToHashSet(); + return Instructions + .Where(i => i.Op == SpirvOp.CompositeExtract && ballots.Contains(i.Operands[2])) + .Select(i => i.Operands[^1]) + .ToList(); + } + } + + private static Module Parse(byte[] spirv) + { + var instructions = new List<(SpirvOp, uint[])>(); + var names = new List(); + for (var offset = 20; offset < spirv.Length;) + { + var word = BinaryPrimitives.ReadUInt32LittleEndian(spirv.AsSpan(offset)); + var length = (int)(word >> 16); + var op = (SpirvOp)(word & 0xFFFF); + var operands = new uint[length - 1]; + for (var index = 0; index < operands.Length; index++) + { + operands[index] = BinaryPrimitives.ReadUInt32LittleEndian(spirv.AsSpan(offset + 4 + index * 4)); + } + + instructions.Add((op, operands)); + if (op == SpirvOp.Name) + { + var text = new List(); + for (var index = 1; index < operands.Length; index++) + { + text.AddRange(BitConverter.GetBytes(operands[index])); + } + + names.Add(Encoding.UTF8.GetString(text.ToArray()).TrimEnd('\0')); + } + + offset += length * 4; + } + + return new Module(instructions, names); + } + + private static Module Compile(Gen5ShaderProgram program, Lowering lowering, uint threads = 64) + { + var (plan, resources, layout) = Prepare(program); + var configured = new ShaderCompileRequest(plan, resources, layout) + { + WaveSize = lowering == Lowering.Wave32 ? 32u : 64u, + HostWave64Supported = lowering == Lowering.NativeWave64, + LocalSizeX = threads, + }; + Assert.True(Gen5SpirvTranslator.TryCompileProgram(configured, out var shader, out var error), error); + return Parse(shader.Spirv); + } + + // v_cmp_lt_u32 vcc, v0, v1 : a per-lane compare whose result becomes a wave mask. + private static Gen5ShaderProgram CompareProgram() => Program( + Vopc(0, "VCmpLtU32", Operand(5), 1), + EndProgram(4)); + + private static Gen5ShaderProgram ReadFirstLaneProgram() => Program( + ReadFirstLane(0, 4, 0), + EndProgram(4)); + + private static Gen5ShaderProgram ReadLaneProgram(uint lane) => Program( + ReadLane(0, 4, 0, lane), + EndProgram(4)); + + private static Gen5ShaderProgram DataShareProgram(bool barrierBetween) => + Program( + new[] + { + DataShare(0, "DsWriteB32", false, [Gen5Operand.Vector(0), Gen5Operand.Vector(1)], [], 0, 0), + } + .Concat(barrierBetween + ? new[] { new Gen5ShaderInstruction(4, Gen5ShaderEncoding.Sopp, "SBarrier", [0u], [], [], null) } + : []) + .Concat(new[] + { + DataShare(8, "DsReadB32", false, [Gen5Operand.Vector(0)], [2u], 0, 0), + EndProgram(16), + }) + .ToArray()); + + [Theory] + [InlineData(Lowering.Wave32)] + [InlineData(Lowering.NativeWave64)] + [InlineData(Lowering.EmulatedWave64)] + public void EveryLoweringCompilesTheBasicWavePrograms(Lowering lowering) + { + foreach (var program in new[] { CompareProgram(), ReadFirstLaneProgram(), ReadLaneProgram(0), ReadLaneProgram(33), DataShareProgram(false), DataShareProgram(true) }) + { + Compile(program, lowering); + } + } + + [Fact] + public void Wave32_BallotUsesOnlyTheLowWord() + { + var module = Compile(CompareProgram(), Lowering.Wave32); + Assert.True(module.Count(SpirvOp.GroupNonUniformBallot) > 0); + Assert.All(module.BallotExtractIndices(), index => Assert.Equal(0u, index)); + } + + [Fact] + public void NativeWave64_BallotCombinesBothWords() + { + var module = Compile(CompareProgram(), Lowering.NativeWave64); + var indices = module.BallotExtractIndices(); + Assert.Contains(0u, indices); + Assert.Contains(1u, indices); + } + + [Fact] + public void OnlyTheEmulatedLoweringDeclaresTheExchange() + { + foreach (var lowering in new[] { Lowering.Wave32, Lowering.NativeWave64 }) + { + var module = Compile(ReadFirstLaneProgram(), lowering); + Assert.DoesNotContain("wave64Exchange", module.Names); + Assert.DoesNotContain("wave64ExchangeParity", module.Names); + } + + var emulated = Compile(ReadFirstLaneProgram(), Lowering.EmulatedWave64); + Assert.Contains("wave64ExchangeParity", emulated.Names); + Assert.Contains("wave64Exchange", emulated.Names); + } + + [Fact] + public void EmulatedWave64_NeedsNoExchangeWithoutSubgroupOperations() + { + var module = Compile(Program(Vop2(0, "VAddI32", 1, Operand(1), Gen5Operand.Vector(0)), EndProgram(4)), Lowering.EmulatedWave64); + Assert.DoesNotContain("wave64ExchangeParity", module.Names); + } + + [Theory] + [InlineData(Lowering.Wave32, 64u)] + [InlineData(Lowering.NativeWave64, 64u)] + [InlineData(Lowering.NativeWave64, 256u)] + public void NativeLoweringsAddNoBarriersAroundSharedMemory(Lowering lowering, uint threads) + { + var module = Compile(DataShareProgram(false), lowering, threads); + Assert.Equal(0, module.Count(SpirvOp.ControlBarrier)); + } + + [Theory] + [InlineData(false, 2)] + [InlineData(true, 2)] + public void EmulatedWave64_OrdersSharedMemoryPhasesWithinOneWave(bool explicitBarrier, int expectedBarriers) + { + var module = Compile(DataShareProgram(explicitBarrier), Lowering.EmulatedWave64, threads: 64); + Assert.Equal(expectedBarriers, module.Count(SpirvOp.ControlBarrier)); + } + + [Fact] + public void EmulatedWave64_MultiWaveGroupsUseNoWorkgroupBarrierForSharedMemoryPhases() + { + var module = Compile(DataShareProgram(false), Lowering.EmulatedWave64, threads: 128); + Assert.Equal(0, module.Count(SpirvOp.ControlBarrier)); + } + + [Fact] + public void EmulatedWave64_MultiWaveExchangeMeetsAtAnArrivalCounter() + { + var single = Compile(ReadFirstLaneProgram(), Lowering.EmulatedWave64, threads: 64); + var multi = Compile(ReadFirstLaneProgram(), Lowering.EmulatedWave64, threads: 256); + Assert.DoesNotContain("wave64RendezvousEpoch", single.Names); + Assert.Contains("wave64RendezvousEpoch", multi.Names); + Assert.True(multi.Count(SpirvOp.AtomicIAdd) > single.Count(SpirvOp.AtomicIAdd)); + } + + [Theory] + [InlineData(Lowering.Wave32)] + [InlineData(Lowering.NativeWave64)] + public void NativeLoweringsNeverDeclareARendezvous(Lowering lowering) + { + var module = Compile(ReadFirstLaneProgram(), lowering, threads: 256); + Assert.DoesNotContain("wave64RendezvousEpoch", module.Names); + Assert.Equal(0, module.Count(SpirvOp.AtomicIAdd)); + } + + [Fact] + public void HostWave64SupportDoesNotChangeAWave32Shader() + { + var program = ReadFirstLaneProgram(); + var (plan, resources, layout) = Prepare(program); + byte[] Compile32(bool hostWave64) + { + var configured = new ShaderCompileRequest(plan, resources, layout) + { + WaveSize = 32, + HostWave64Supported = hostWave64, + LocalSizeX = 64, + }; + Assert.True(Gen5SpirvTranslator.TryCompileProgram(configured, out var shader, out var error), error); + return shader.Spirv.ToArray(); + } + + Assert.Equal(Compile32(false), Compile32(true)); + } + + [Theory] + [InlineData(Lowering.Wave32)] + [InlineData(Lowering.NativeWave64)] + [InlineData(Lowering.EmulatedWave64)] + public void ReadLaneBroadcastsFromTheRequestedLane(Lowering lowering) + { + var module = Compile(ReadLaneProgram(33), lowering); + Assert.True( + module.Count(SpirvOp.GroupNonUniformBroadcast) > 0 || module.Count(SpirvOp.GroupNonUniformShuffle) > 0 || + module.Count(SpirvOp.Store) > 0); + } + + [Theory] + [InlineData(Lowering.Wave32)] + [InlineData(Lowering.NativeWave64)] + [InlineData(Lowering.EmulatedWave64)] + public void LoweringsAreDeterministic(Lowering lowering) + { + var first = Compile(CompareProgram(), lowering); + var second = Compile(CompareProgram(), lowering); + Assert.Equal(first.Instructions.Count, second.Instructions.Count); + } +} diff --git a/tests/SharpEmu.ShaderCompiler.Tests/Resources/ResourceMaterializationCacheTests.cs b/tests/SharpEmu.ShaderCompiler.Tests/Resources/ResourceMaterializationCacheTests.cs index e1f7354..a7b71ae 100644 --- a/tests/SharpEmu.ShaderCompiler.Tests/Resources/ResourceMaterializationCacheTests.cs +++ b/tests/SharpEmu.ShaderCompiler.Tests/Resources/ResourceMaterializationCacheTests.cs @@ -153,6 +153,26 @@ public sealed class ResourceMaterializationCacheTests Assert.Equal(0, cache.Hits); } + private static ShaderResourcePlan PlanWithSpareUserData() => + ShaderResourcePlan.Extract(DirectImageTableTests.CreateWaveIndexedDescriptorProgram(), ShaderStage.Compute, Hash, 0, 4); + + [Fact] + public void AUserDataWordTheProgramNeverReadsDoesNotSplitTheEntry() + { + var plan = PlanWithSpareUserData(); + var heap = new Heap(); + var cache = new ResourceMaterializationCache(); + Assert.True(Run(cache, plan, heap, [0x1000, 0, 5, 6], out var first, out _)); + Assert.True(Run(cache, plan, heap, [0x1000, 0, 7, 8], out var second, out _)); + Assert.Equal((1, 1), (cache.Hits, cache.Misses)); + // The hit carries the second draw's user data and leaves the first draw's snapshot alone. + Assert.Equal(new uint[] { 0x1000, 0, 7, 8 }, second.UserData); + Assert.Equal(new uint[] { 0x1000, 0, 5, 6 }, first.UserData); + Assert.Same(first.Images, second.Images); + Assert.True(Run(cache, plan, heap, [0x1000, 0, 5, 6], out var third, out _)); + Assert.Same(first, third); + } + [Fact] public void ADifferentDrawKeyIsADifferentEntry() { @@ -264,6 +284,276 @@ public sealed class ResourceMaterializationCacheTests Assert.Equal(0, cache.TableRefreshes); } + // A V# taken verbatim from s[4:7], the way titles pass per-draw constants. + private static ShaderResourcePlan DirectBufferPlan() => + Extract(Program(BufferLoad(0, 4), EndProgram(8)), userDataCount: 8); + + // The same V#, whose base pair s[4:5] is also the address of a scalar load. + private static ShaderResourcePlan SteeringBufferPlan() => + Extract(Program(ScalarLoad(0, 4, 16), Vop1(8, "VMovB32", 0, Gen5Operand.Scalar(16)), BufferStore(12, 4), EndProgram(20)), + userDataCount: 8); + + private static bool RunWords(ResourceMaterializationCache cache, ShaderResourcePlan plan, TestWordMemory memory, uint[] userData, + out ResourceSnapshot snapshot) + { + snapshot = new ResourceSnapshot(); + var specialization = new ResourceSpecialization(); + return cache.Materialize(plan, Inputs(userData, memory.Read, memory.Read), (address, destination, _) => + { + for (var offset = 0; offset < destination.Length; offset += 4) + { + if (!memory.Read(address + (ulong)offset, out var word)) + return false; + BitConverter.TryWriteBytes(destination[offset..], word); + } + + return true; + }, ref snapshot, ref specialization, out _); + } + + private static uint[] Reference(ShaderResourcePlan plan, TestWordMemory memory, uint[] userData) + { + var snapshot = new ResourceSnapshot(); + var specialization = new ResourceSpecialization(); + Assert.True(ResourceMaterializer.Materialize(plan, Inputs(userData, memory.Read, memory.Read), ref snapshot, ref specialization)); + return snapshot.Buffers[0]; + } + + private static uint[] LinearConstants(int draw) => [0, 0, 0, 0, 0x10000u + (uint)draw * 0x100u, 0, 256, 0x2_0000]; + + [Fact] + public void ALinearlyAllocatedBufferBaseStopsMissingOnceConfirmed() + { + var plan = DirectBufferPlan(); + var memory = new TestWordMemory(); + var cache = new ResourceMaterializationCache(); + for (var draw = 0; draw < 16; draw++) + { + var userData = LinearConstants(draw); + Assert.True(RunWords(cache, plan, memory, userData, out var snapshot)); + Assert.Equal(Reference(plan, memory, userData), snapshot.Buffers[0]); + Assert.Equal(userData, snapshot.UserData); + } + + // One miss to build, three to confirm the word; every later draw is a hit. + Assert.Equal((12, 4), (cache.Hits, cache.Misses)); + } + + [Fact] + public void InterleavedObjectsConfirmTheirOwnBases() + { + // Constants V# in s[4:7], a second buffer in s[8:11] whose size tells the two objects apart. + var plan = Extract(Program(BufferLoad(0, 4), BufferLoad(8, 8), EndProgram(16)), userDataCount: 12); + var memory = new TestWordMemory(); + var cache = new ResourceMaterializationCache(); + for (var draw = 0; draw < 32; draw++) + { + uint[] userData = [0, 0, 0, 0, 0x10000u + (uint)draw * 0x100u, 0, 256, 0x2_0000, 0x40000, 0, draw % 2 == 0 ? 256u : 512u, 0x2_0000]; + Assert.True(RunWords(cache, plan, memory, userData, out var snapshot)); + var expected = new ResourceSnapshot(); + var expectedSpecialization = new ResourceSpecialization(); + Assert.True(ResourceMaterializer.Materialize(plan, Inputs(userData, memory.Read, memory.Read), ref expected, ref expectedSpecialization)); + Assert.Equal(expected.Buffers.Select(buffer => buffer.ToArray()), snapshot.Buffers.Select(buffer => buffer.ToArray())); + } + + Assert.True(cache.Hits >= 20, $"hits={cache.Hits} misses={cache.Misses}"); + } + + // Pointer s[0:1] moves on every draw; the program loads four constants through it. + private static uint[] PointerConstants(int draw) => [0x10000u + (uint)draw * 0x100u, 0, 0, 0, 0x30000, 0, 256, 0x2_0000, 0]; + + [Fact] + public void AMovingScalarLoadPointerStopsMissingOnceConfirmed() + { + var (plan, _, _) = Prepare(FlattenedReadReuseTests.RepeatedReadProgram(4), userDataCount: 9); + // The same four words repeat every 0x100 bytes, so every pointer finds the same constants. + var memory = new TestWordMemory { Base = 0x10000, Words = Enumerable.Range(0, 0x1000).Select(index => (uint)(index % 64) * 3 + 1).ToArray() }; + var cache = new ResourceMaterializationCache(); + for (var draw = 0; draw < 16; draw++) + { + var userData = PointerConstants(draw); + Assert.True(RunWords(cache, plan, memory, userData, out var snapshot)); + var expected = new ResourceSnapshot(); + var expectedSpecialization = new ResourceSpecialization(); + Assert.True(ResourceMaterializer.Materialize(plan, Inputs(userData, memory.Read, memory.Read), ref expected, ref expectedSpecialization)); + Assert.Equal(expected.FlattenedResourceTable, snapshot.FlattenedResourceTable); + Assert.Equal(expected.Buffers.Select(buffer => buffer.ToArray()), snapshot.Buffers.Select(buffer => buffer.ToArray())); + } + + Assert.Equal((12, 4), (cache.Hits, cache.Misses)); + + // Other constants behind the next pointer: the draw reads them instead of reusing the table. + var moved = PointerConstants(30); + memory.At(moved[0] + 8) = 0xBEEF; + Assert.True(RunWords(cache, plan, memory, moved, out var reread)); + Assert.Equal(0xBEEFu, reread.FlattenedResourceTable[2]); + Assert.Equal(12, cache.Hits); + } + + [Fact] + public void ARebasedHitLeavesTheCachedSnapshotAlone() + { + var plan = DirectBufferPlan(); + var memory = new TestWordMemory(); + var cache = new ResourceMaterializationCache(); + for (var draw = 0; draw < 4; draw++) + Assert.True(RunWords(cache, plan, memory, LinearConstants(draw), out _)); + + Assert.True(RunWords(cache, plan, memory, LinearConstants(3), out var built)); + Assert.True(RunWords(cache, plan, memory, LinearConstants(9), out var patched)); + Assert.Equal(0x10900u, patched.Buffers[0][0]); + Assert.Equal(0x10300u, built.Buffers[0][0]); + Assert.NotSame(built.Buffers, patched.Buffers); + Assert.True(RunWords(cache, plan, memory, LinearConstants(3), out var again)); + Assert.Equal(0x10300u, again.Buffers[0][0]); + } + + [Fact] + public void AReadThatMovesWithTheBaseAndKeepsItsBytesIsReused() + { + var plan = SteeringBufferPlan(); + var memory = new TestWordMemory { Base = 0x10000, Words = Enumerable.Repeat(0x1234u, 0x1000).ToArray() }; + var cache = new ResourceMaterializationCache(); + for (var draw = 0; draw < 16; draw++) + { + var userData = LinearConstants(draw); + Assert.True(RunWords(cache, plan, memory, userData, out var snapshot)); + var expected = new ResourceSnapshot(); + var expectedSpecialization = new ResourceSpecialization(); + Assert.True(ResourceMaterializer.Materialize(plan, Inputs(userData, memory.Read, memory.Read), ref expected, ref expectedSpecialization)); + Assert.Equal(expected.Buffers[0], snapshot.Buffers[0]); + Assert.Equal(expected.FlattenedResourceTable, snapshot.FlattenedResourceTable); + } + + Assert.Equal((12, 4), (cache.Hits, cache.Misses)); + + // The moved read now finds other bytes at the new base: the draw must not reuse the table. + var changed = LinearConstants(20); + memory.At(changed[4]) = 0x5678; + Assert.True(RunWords(cache, plan, memory, changed, out var refreshed)); + Assert.Equal(0x5678u, refreshed.FlattenedResourceTable[0]); + Assert.Equal(12, cache.Hits); + } + + [Fact] + public void MovedConstantsThatChangeEveryDrawAreReadAgain() + { + var plan = SteeringBufferPlan(); + var memory = new TestWordMemory { Base = 0x10000, Words = Enumerable.Range(0, 0x1000).Select(index => (uint)index * 7).ToArray() }; + var cache = new ResourceMaterializationCache(); + for (var draw = 0; draw < 16; draw++) + { + var userData = LinearConstants(draw); + Assert.True(RunWords(cache, plan, memory, userData, out var snapshot)); + Assert.Equal(Reference(plan, memory, userData), snapshot.Buffers[0]); + var expected = new ResourceSnapshot(); + var expectedSpecialization = new ResourceSpecialization(); + Assert.True(ResourceMaterializer.Materialize(plan, Inputs(userData, memory.Read, memory.Read), ref expected, ref expectedSpecialization)); + Assert.Equal(expected.FlattenedResourceTable, snapshot.FlattenedResourceTable); + } + + // The constants differ at every base, so nothing is reused as is; once the base is + // confirmed, the moved table is evaluated again instead of the whole walk. + Assert.Equal(0, cache.Hits); + Assert.True(cache.TableRefreshes >= 10, $"refreshes={cache.TableRefreshes} misses={cache.Misses}"); + } + + // The V# comes from a descriptor table at s[0:1]; its constants are read through it. + private static ShaderResourcePlan MemoryDescriptorPlan() => + Extract(Program(ScalarLoad(0, 0, 8, 4), ScalarBufferLoad(4, 8, 12, 4), Vop1(8, "VMovB32", 0, Gen5Operand.Scalar(12)), + BufferLoad(12, 8), EndProgram(16)), userDataCount: 2); + + // The title rewrites the table every draw: a freshly allocated base, and new constants there. + private static void WriteDraw(TestWordMemory memory, int draw) + { + var bufferBase = 0x10000u + (uint)draw * 0x100u; + memory.At(0x1000) = bufferBase; + memory.At(0x1004) = 0; + memory.At(0x1008) = 256; + memory.At(0x100C) = 0x2_0000; + for (var word = 0u; word < 4; word++) + memory.At(bufferBase + word * 4) = (uint)draw * 10 + word; + } + + [Fact] + public void ABaseRewrittenInGuestMemoryStopsMissingOnceConfirmed() + { + var plan = MemoryDescriptorPlan(); + var memory = new TestWordMemory { Base = 0x1000, Words = new uint[0x8000] }; + var cache = new ResourceMaterializationCache(); + uint[] userData = [0x1000, 0]; + for (var draw = 0; draw < 16; draw++) + { + WriteDraw(memory, draw); + Assert.True(RunWords(cache, plan, memory, userData, out var snapshot)); + var expected = new ResourceSnapshot(); + var expectedSpecialization = new ResourceSpecialization(); + Assert.True(ResourceMaterializer.Materialize(plan, Inputs(userData, memory.Read, memory.Read), ref expected, ref expectedSpecialization)); + Assert.Equal(expected.Buffers.Select(buffer => buffer.ToArray()), snapshot.Buffers.Select(buffer => buffer.ToArray())); + Assert.Equal(expected.FlattenedResourceTable, snapshot.FlattenedResourceTable); + } + + // One miss to build, three to confirm the base; the later draws patch it. + Assert.True(cache.Hits >= 10, $"hits={cache.Hits} misses={cache.Misses}"); + + // Another descriptor word changed with the base: the draw is materialized again. + WriteDraw(memory, 20); + memory.At(0x1008) = 512; + var hits = cache.Hits; + Assert.True(RunWords(cache, plan, memory, userData, out var resized)); + Assert.Equal(hits, cache.Hits); + Assert.Equal(512u, resized.Buffers[0][2]); + } + + private static void AssertMatchesFullWalk(ShaderResourcePlan plan, TestWordMemory memory, uint[] userData, ResourceSnapshot snapshot) + { + var expected = new ResourceSnapshot(); + var expectedSpecialization = new ResourceSpecialization(); + Assert.True(ResourceMaterializer.Materialize(plan, Inputs(userData, memory.Read, memory.Read), ref expected, ref expectedSpecialization)); + Assert.Equal(expected.Buffers.Select(buffer => buffer.ToArray()), snapshot.Buffers.Select(buffer => buffer.ToArray())); + Assert.Equal(expected.FlattenedResourceTable, snapshot.FlattenedResourceTable); + } + + [Fact] + public void ATableConstantBesideARewrittenBaseIsCopiedForTheDraw() + { + // The V# and a constant both come from the table at s[0:1]; the constant changes every draw. + var plan = Extract(Program(ScalarLoad(0, 0, 8, 4), ScalarLoad(4, 0, 16, 1, immediateOffset: 16), + ScalarBufferLoad(8, 8, 12, 4), Vop1(12, "VMovB32", 0, Gen5Operand.Scalar(12)), + Vop1(16, "VMovB32", 1, Gen5Operand.Scalar(16)), BufferLoad(20, 8), EndProgram(24)), userDataCount: 2); + var memory = new TestWordMemory { Base = 0x1000, Words = new uint[0x8000] }; + var cache = new ResourceMaterializationCache(); + uint[] userData = [0x1000, 0]; + for (var draw = 0; draw < 16; draw++) + { + WriteDraw(memory, draw); + memory.At(0x1010) = (uint)draw * 3 + 1; + Assert.True(RunWords(cache, plan, memory, userData, out var snapshot)); + AssertMatchesFullWalk(plan, memory, userData, snapshot); + } + + Assert.True(cache.Hits >= 10, $"hits={cache.Hits} misses={cache.Misses}"); + } + + [Fact] + public void AlternatingObjectsWithRewrittenBasesRebaseTheirOwnVariant() + { + // Two objects share the draw key and differ in their buffer size; both move their base. + var plan = MemoryDescriptorPlan(); + var memory = new TestWordMemory { Base = 0x1000, Words = new uint[0x8000] }; + var cache = new ResourceMaterializationCache(); + uint[] userData = [0x1000, 0]; + for (var draw = 0; draw < 32; draw++) + { + WriteDraw(memory, draw); + memory.At(0x1008) = draw % 2 == 0 ? 256u : 512u; + Assert.True(RunWords(cache, plan, memory, userData, out var snapshot)); + AssertMatchesFullWalk(plan, memory, userData, snapshot); + } + + Assert.True(cache.Hits >= 20, $"hits={cache.Hits} misses={cache.Misses}"); + } + [Fact] public void AFullGenerationKeepsRecentEntries() {